fix(test): F3 skip-as-pass in test_x509.py + F7 negative KAT vectors

tools/run_all_tests.py

@@ -130,7 +130,19 @@ def main():
  suites = ["entropy", "net", "sha256", "crypto", "hkdf",
  "keyschedule", "http", "tls_record", "tls_handshake",
  "x25519"]
- if not skip_slow:
+
+ # Suites deliberately not run this session, as (name, reason). --skip-slow
+ # used to drop x509 by simply never adding it to the list, so the aggregate
+ # printed a TOTAL and exited 0 with no trace that the entire X.509/ECDSA
+ # suite had not run. That is audit finding F3's shape one level up: the
+ # skipped assertions left the denominator instead of being accounted for.
+ # An explicit operator flag is a legitimate reason to skip; it is not a
+ # licence to report an unqualified clean pass.
+ skipped_suites = []
+ if skip_slow:
+ skipped_suites.append(
+ ("x509", "--skip-slow (X.509 DER parsing + ECDSA P-256 verify)"))
+ else:
  suites.insert(0, "x509")
 
  config = ViceConfig(prg_path=PRG_PATH, warp=True, ntsc=True, sound=False,
@@ -176,13 +188,25 @@ def run_suite_in_own_instance(mgr, suite_name):
  total_failed = sum(r[2] for r in results)
  total_tests = total_passed + total_failed
 
+ skipped_note = ""
+ if skipped_suites:
+ skipped_note = (f" -- {len(skipped_suites)} suite(s) SKIPPED: "
+ + ", ".join(n for n, _ in skipped_suites))
+
  print(f"\n{'='*60}")
  print(f"TOTAL: {total_passed}/{total_tests} passed, "
- f"{total_failed} failed")
+ f"{total_failed} failed{skipped_note}")
  for name, passed, failed, duration in sorted(results):
  status = "OK" if failed == 0 else "FAIL"
  print(f" {status:4s} {name:20s} {passed:3d}/{passed+failed:3d} "
  f"({duration:.1f}s)")
+ for name, reason in skipped_suites:
+ print(f" SKIP {name:20s} --- did not run: {reason}")
+ if skipped_suites:
+ print("\n WARNING: the suite(s) above did not run. This aggregate "
+ "result does not")
+ print(" certify them, and their assertions are absent from "
+ "the TOTAL.")
  print(f"{'='*60}")
 
  sys.exit(0 if total_failed == 0 else 1)

tools/test_ecdsa_kat_oracle.py

@@ -1,24 +1,51 @@
 #!/usr/bin/env python3
 """test_ecdsa_kat_oracle.py - Library-side KAT oracle for ECDSA P-256 verify.
 
-Runs additional known-VALID P-256/SHA-256 signature vectors against the
-C64's `ecdsa_verify` routine (the c64-https dispatcher over the
-libs/nistcurves sibling). Mirrors the structure of
-`tools/test_x509.py` group 3 subtest [3c] (call `setup_ecdsa_verify(...)`,
-then `jsr_with_carry(... labels["ecdsa_verify"] ...)`, assert C=0) but
-exercises 3 additional vectors so we can distinguish a primitive bug
-from a [3c]-specific test-setup bug:
-
- - [3e] CAVP SigVer P-256/SHA-256 valid #1 (Result = P record)
- - [3f] CAVP SigVer P-256/SHA-256 valid #2
- - [3g] CAVP SigVer P-256/SHA-256 valid #3
-
+Runs CAVP P-256/SHA-256 signature vectors against the C64's `ecdsa_verify`
+routine (the c64-https dispatcher over the libs/nistcurves sibling). Mirrors
+the structure of `tools/test_x509.py` group 3 subtest [3c] (call
+`setup_ecdsa_verify(...)`, then `jsr_with_carry(... labels["ecdsa_verify"] ...)`,
+assert the carry) but exercises additional vectors so we can distinguish a
+primitive bug from a [3c]-specific test-setup bug:
+
+ - [3e] CAVP SigVer P-256/SHA-256 valid #1 (Result = P) -> expect C=0
+ - [3f] CAVP SigVer P-256/SHA-256 valid #2 (Result = P) -> expect C=0
+ - [3g] CAVP SigVer P-256/SHA-256 valid #3 (Result = P) -> expect C=0
+ - [3h] CAVP SigVer P-256/SHA-256 invalid #1 (Result = F, S changed) -> C=1
+ - [3i] CAVP SigVer P-256/SHA-256 invalid #2 (Result = F, R changed) -> C=1
+ - [3j] CAVP SigVer P-256/SHA-256 invalid #3 (Result = F, Msg changed) -> C=1
+
+Negative vectors (audit finding F7)
+-----------------------------------
+This oracle originally ran three vectors, all valid, all expecting C=0.
+That cannot distinguish a working verifier from one that reports "valid"
+unconditionally: against an `ecdsa_verify` stubbed to `clc; rts` it happily
+reported 3/3. An oracle with no negative case does not test verification,
+it tests that the routine returns.
+
+The three `Result = F` records above close that. They are genuine CAVP
+records, not signatures manufactured by mutating a valid one and not
+anything produced by running this implementation.
+
+Every `Result = F` record in the file has Q on the curve and r, s in
+[1, n-1] (checked host-side, see below), so none of them can be rejected by
+a cheap range or point-validity gate — each one forces the full verify math
+and compares the recovered R.x against r. One record per CAVP modification
+class is included: 3 (S changed), 2 (R changed), 1 (Message changed).
+
+Provenance
+----------
 Vectors are extracted verbatim from
 `libs/nistcurves/tools/vectors/nist_p256_sigver.rsp` (NIST CAVP SigVer,
-P-256/SHA-256 section), specifically the records flagged `Result = P`.
-For each vector the hash is `SHA-256(Msg)`; r/s/Qx/Qy are taken straight
-from the .rsp file in big-endian wire order, matching the BE struct ABI
-of the sibling's `ecdsa_verify_256`.
+P-256/SHA-256 section). For each vector the hash is `SHA-256(Msg)`;
+r/s/Qx/Qy are taken straight from the .rsp file in big-endian wire order,
+matching the BE struct ABI of the sibling's `ecdsa_verify_256`.
+
+Every vector below — positive and negative — was independently confirmed
+host-side against OpenSSL via the `cryptography` package before being added
+here: all 15 records in the .rsp agreed with their Result column, with Q on
+curve and r, s in range. The `expect_carry` field encodes the .rsp Result
+column (P -> 0, F -> 1), never an observed C64 result.
 
 Usage:
  python3 tools/test_ecdsa_kat_oracle.py [--verbose]
@@ -71,13 +98,19 @@
 
 
 # ---------------------------------------------------------------------------
-# Hardcoded P-256 known-VALID KAT vectors (CAVP SigVer, Result = P records)
+# Hardcoded P-256 KAT vectors (CAVP SigVer, both Result = P and Result = F)
+#
+# `expect_carry` mirrors the .rsp Result column: P (valid) -> C=0,
+# F (invalid) -> C=1. See the module docstring for provenance and for why
+# the negative records are load-bearing (audit finding F7).
 # ---------------------------------------------------------------------------
 
 KAT_VECTORS = [
+ # --- Result = P (valid) --------------------------------------------
  # CAVP SigVer P-256/SHA-256 valid record #1
  dict(
  tag="CAVP SigVer P-256/SHA-256 valid #1",
+ expect_carry=0,
  hash=bytes.fromhex(
  "d1b8ef21eb4182ee270638061063a3f3"
  "c16c114e33937f69fb232cc833965a94"),
@@ -97,6 +130,7 @@
  # CAVP SigVer P-256/SHA-256 valid record #2
  dict(
  tag="CAVP SigVer P-256/SHA-256 valid #2",
+ expect_carry=0,
  hash=bytes.fromhex(
  "b9336a8d1f3e8ede001d19f41320bc76"
  "72d772a3d2cb0e435fff3c27d6804a2c"),
@@ -116,6 +150,7 @@
  # CAVP SigVer P-256/SHA-256 valid record #3
  dict(
  tag="CAVP SigVer P-256/SHA-256 valid #3",
+ expect_carry=0,
  hash=bytes.fromhex(
  "41007876926a20f821d72d9c6f2c9dae"
  "6c03954123ea6e6939d7e6e669438891"),
@@ -132,9 +167,76 @@
  "9b52672742d637a32add056dfd6d8792"
  "f2a33c2e69dafabea09b960bc61e230a"),
  ),
+
+ # --- Result = F (invalid) ------------------------------------------
+ # These are what make this file an oracle rather than a smoke test:
+ # a verify that answers "valid" unconditionally passes every vector
+ # above and fails every vector below. Q is on the curve and r, s are in
+ # [1, n-1] for all three, so none is rejectable by a cheap gate.
+ #
+ # CAVP SigVer P-256/SHA-256 invalid record, "Result = F (3 - S changed)"
+ dict(
+ tag="CAVP SigVer P-256/SHA-256 invalid #1 (F: S changed)",
+ expect_carry=1,
+ hash=bytes.fromhex(
+ "a82c31412f537135d1c418bd7136fb5f"
+ "de9426e70c70e7c2fb11f02f30fdeae2"),
+ r=bytes.fromhex(
+ "d19ff48b324915576416097d2544f7cb"
+ "df8768b1454ad20e0baac50e211f23b0"),
+ s=bytes.fromhex(
+ "a3e81e59311cdfff2d4784949f7a2cb5"
+ "0ba6c3a91fa54710568e61aca3e847c6"),
+ qx=bytes.fromhex(
+ "87f8f2b218f49845f6f10eec38771362"
+ "69f5c1a54736dbdf69f89940cad41555"),
+ qy=bytes.fromhex(
+ "e15f369036f49842fac7a86c8a2b0557"
+ "609776814448b8f5e84aa9f4395205e9"),
+ ),
+ # CAVP SigVer P-256/SHA-256 invalid record, "Result = F (2 - R changed)"
+ dict(
+ tag="CAVP SigVer P-256/SHA-256 invalid #2 (F: R changed)",
+ expect_carry=1,
+ hash=bytes.fromhex(
+ "5984eab8854d0a9aa5f0c70f96deeb51"
+ "0e5f9ff8c51befcdc3c41bac53577f22"),
+ r=bytes.fromhex(
+ "dc23d130c6117fb5751201455e99f36f"
+ "59aba1a6a21cf2d0e7481a97451d6693"),
+ s=bytes.fromhex(
+ "d6ce7708c18dbf35d4f8aa7240922dc6"
+ "823f2e7058cbc1484fcad1599db5018c"),
+ qx=bytes.fromhex(
+ "5cf02a00d205bdfee2016f7421807fc3"
+ "8ae69e6b7ccd064ee689fc1a94a9f7d2"),
+ qy=bytes.fromhex(
+ "ec530ce3cc5c9d1af463f264d685afe2"
+ "b4db4b5828d7e61b748930f3ce622a85"),
+ ),
+ # CAVP SigVer P-256/SHA-256 invalid record, "Result = F (1 - Message changed)"
+ dict(
+ tag="CAVP SigVer P-256/SHA-256 invalid #3 (F: Message changed)",
+ expect_carry=1,
+ hash=bytes.fromhex(
+ "d80e9933e86769731ec16ff31e682153"
+ "1bcf07fcbad9e2ac16ec9e6cb343a870"),
+ r=bytes.fromhex(
+ "288f7a1cd391842cce21f00e6f15471c"
+ "04dc182fe4b14d92dc18910879799790"),
+ s=bytes.fromhex(
+ "247b3c4e89a3bcadfea73c7bfd361def"
+ "43715fa382b8c3edf4ae15d6e55e9979"),
+ qx=bytes.fromhex(
+ "69b7667056e1e11d6caf6e45643f8b21"
+ "e7a4bebda463c7fdbc13bc98efbd0214"),
+ qy=bytes.fromhex(
+ "d3f9b12eb46c7c6fda0da3fc85bc1fd8"
+ "31557f9abc902a3be3cb3e8be7d1aa2f"),
+ ),
 ]
 
-SUBTEST_LABELS = ["3e", "3f", "3g"]
+SUBTEST_LABELS = ["3e", "3f", "3g", "3h", "3i", "3j"]
 
 
 # ---------------------------------------------------------------------------
@@ -209,7 +311,9 @@ def run_kat_oracle(transport, labels):
  for idx, vec in enumerate(KAT_VECTORS):
  sub = SUBTEST_LABELS[idx]
  tag = vec["tag"]
- print(f"\n [{sub}] ECDSA verify: {tag} (expected C=0)")
+ want = vec["expect_carry"]
+ want_word = "valid" if want == 0 else "INVALID"
+ print(f"\n [{sub}] ECDSA verify: {tag} (expected C={want}, {want_word})")
  if VERBOSE:
  print(f" hash = {vec['hash'][:8].hex()}... r = {vec['r'][:8].hex()}...")
  print(f" s = {vec['s'][:8].hex()}... Qx = {vec['qx'][:8].hex()}... Qy = {vec['qy'][:8].hex()}...")
@@ -240,12 +344,19 @@ def run_kat_oracle(transport, labels):
  carry = jsr_with_carry(transport, labels["ecdsa_verify"],
  timeout=2400.0, poll_interval=30.0)
  elapsed = time.time() - t0
- if carry == 0:
+ got_word = "valid" if carry == 0 else "invalid"
+ if carry == want:
  passed += 1
- print(f" PASS: ecdsa_verify returned C=0 (valid) [{elapsed:.0f}s]")
+ print(f" PASS: ecdsa_verify returned C={carry} "
+ f"({got_word}) [{elapsed:.0f}s]")
  else:
  failed += 1
- print(f" FAIL: ecdsa_verify returned C=1 (invalid) [{elapsed:.0f}s]")
+ print(f" FAIL: ecdsa_verify returned C={carry} ({got_word}), "
+ f"expected C={want} ({want_word}) [{elapsed:.0f}s]")
+ if want == 1:
+ print(" A CAVP Result=F vector was accepted. The "
+ "verifier is reporting")
+ print(" signatures valid that NIST says are not.")
  print(f" hash: {c64_hash.hex()}")
  print(f" r: {c64_r.hex()}")
  print(f" s: {c64_s.hex()}")
@@ -288,8 +399,21 @@ def main():
  print("\nFATAL: ECDSA verify labels missing; nothing to test.")
  sys.exit(1)
 
+ n_pos = sum(1 for v in KAT_VECTORS if v["expect_carry"] == 0)
+ n_neg = sum(1 for v in KAT_VECTORS if v["expect_carry"] == 1)
+
+ # Structural guard against audit finding F7 regressing: an oracle made up
+ # entirely of valid signatures cannot fail against a verifier that always
+ # answers "valid", so it is not an oracle.
+ if n_neg == 0:
+ print("\nFATAL: KAT_VECTORS contains no Result=F (expect_carry=1) vector.")
+ print(" An all-positive vector set passes against a verify stubbed")
+ print(" to 'clc; rts' and therefore proves nothing. See F7.")
+ sys.exit(1)
+
  print(f"\n Labels loaded from {LABELS_PATH}")
- print(f" Vectors to run: {len(KAT_VECTORS)} (CAVP SigVer P-256/SHA-256 valid)")
+ print(f" Vectors to run: {len(KAT_VECTORS)} CAVP SigVer P-256/SHA-256 "
+ f"({n_pos} valid / {n_neg} invalid)")
  print(f" Per-vector wallclock budget: 2400 s (VICE warp; typical ~5-16 min)")
 
  config = default_vice_config(prg_path=PRG_PATH, warp=True, ntsc=True, sound=False)
@@ -318,7 +442,8 @@ def main():
  print(f" sqtab_init FAILED: {e}")
  sys.exit(1)
 
- print(f"\n=== ECDSA P-256 KAT oracle ({len(KAT_VECTORS)} valid vectors) ===")
+ print(f"\n=== ECDSA P-256 KAT oracle "
+ f"({n_pos} valid + {n_neg} invalid vectors) ===")
  passed, failed = run_kat_oracle(transport, labels)
 
  mgr.release(inst)
@@ -329,6 +454,18 @@ def main():
  print(f"{'='*60}")
  print(f" Passed: {passed}/{total}")
  print(f" Failed: {failed}/{total}")
+ if total != len(KAT_VECTORS):
+ # Every declared vector must produce a verdict; a vector that silently
+ # dropped out is the same class of defect as F3's skipped group.
+ print(f"\n [-] ECDSA KAT oracle: only {total} of "
+ f"{len(KAT_VECTORS)} declared vectors produced a verdict")
+ sys.exit(1)
+ if failed == 0:
+ print(f"\n [+] ECDSA KAT oracle: ALL {total} VECTORS PASSED "
+ f"({n_pos} valid accepted, {n_neg} invalid rejected)")
+ else:
+ print(f"\n [-] ECDSA KAT oracle: {failed} VECTOR(S) FAILED")
+ print(f"{'='*60}")
  sys.exit(0 if failed == 0 else 1)