From c952558c9133bee10b05d50417dfb166033e6e05 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Thu, 13 Aug 2026 06:23:04 -0500 Subject: [PATCH 1/3] fix(test): a missing label is a failed group, not a silent skip (F3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dropping the single symbol `ecdsa_verify` from build/labels.txt made tools/test_x509.py skip its entire ECDSA group, report "ALL 7 TESTS PASSED" and exit 0. The skipped assertions left the denominator instead of counting against it, so the suite reported success while verifying nothing about ECDSA. The realistic trigger is a libs/nistcurves bump renaming an export — exactly the change this suite exists to catch, and live risk today with the siblings two minor versions ahead of the pin. Every group the suite claims to run is now declared in REQUIRED_LABEL_SETS and is required: - main() fails closed in a preflight, before launching VICE: exit 1 with a summary line naming the group and its missing labels. - run_tests() counts each un-runnable group as a failure and names it. This matters independently: tools/run_all_tests.py calls run_tests() directly and only sees (passed, failed), so the arity is unchanged and a vanished group now turns that runner red too. - "no tests ran" is a failure rather than a shrug. - CV_LABELS is deliberately NOT required — no group drives it, and a group that does not exist cannot be silently skipped. Also converts the one other skip-as-pass in this file: a P-384 certificate that fails to parse reported "SKIP: may not be supported yet" and returned (0, 0), which would have hidden a P-384 DER regression. It is now a failure. P-384 cert *parsing* works today (group 2 passes 2/2 — measured, see below); only P-384 ECDSA *verify* is stubbed at the TLS layer. Acceptance (VICE, ip65 build, -reu via default_vice_config): ecdsa_verify absent from the labels view before: "[+] X.509/ECDSA: ALL 7 TESTS PASSED" EXIT_CODE=0 after: "[-] X.509/ECDSA: ABORTED -- declared test group(s) cannot run: ECDSA P-256 verify (group 3) [missing: ecdsa_verify]" EXIT_CODE=1 labels intact after: "[+] X.509/ECDSA: ALL 11 TESTS PASSED" EXIT_CODE=0 The denominator moving 7 -> 11 is the fix working: the ECDSA group now contributes its 4 assertions instead of evaporating. Co-Authored-By: Claude Opus 5 (1M context) --- tools/test_x509.py | 165 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 134 insertions(+), 31 deletions(-) diff --git a/tools/test_x509.py b/tools/test_x509.py index 238ede6..b78c2c6 100644 --- a/tools/test_x509.py +++ b/tools/test_x509.py @@ -10,6 +10,22 @@ python3 tools/test_x509.py [--seed S] [--verbose] Requires: Python 3.10+, c64_test_harness, VICE x64sc, cryptography + +Skip policy (audit finding F3) +------------------------------ +This suite used to treat a missing label as a benign skip: dropping the +single symbol ``ecdsa_verify`` from ``build/labels.txt`` silently deleted the +whole ECDSA group, and the run still reported "ALL 7 TESTS PASSED" and exited +0. The skipped assertions left the denominator instead of counting against it. + +The realistic trigger is a ``libs/nistcurves`` bump renaming an export, which +is precisely the change this suite exists to catch. + +Policy now: every group listed in ``REQUIRED_LABEL_SETS`` is required. If its +labels are missing, ``main()`` aborts before launching VICE (exit 1, summary +names the group), and ``run_tests()`` — which ``tools/run_all_tests.py`` calls +directly — counts each such group as a failure. Optional/unwired label sets +(``CV_LABELS``) are deliberately not in that list. """ import datetime @@ -78,13 +94,27 @@ "sqtab_init", ] -# Labels for CertificateVerify tests +# Labels for CertificateVerify tests. +# NOTE: no group currently drives these — the CertificateVerify tests are not +# wired up (see run_tests()). They are therefore NOT in REQUIRED_LABEL_SETS: +# a group that does not exist cannot be silently skipped. CV_LABELS = [ "tls_handle_cert_verify", "tls_rec_buf", "tls_rec_len", "tls_transcript", ] +# Every test group this suite claims to run, with the labels it needs. +# +# These are REQUIRED, not optional. If a build stops exporting one of these +# symbols — the realistic trigger being a libs/nistcurves bump that renames an +# export — the affected group must be reported as a FAILURE, never quietly +# dropped from the denominator. See the module docstring's "silent skip" note. +REQUIRED_LABEL_SETS = [ + ("DER parser (groups 1-2)", DER_LABELS), + ("ECDSA P-256 verify (group 3)", ECDSA_LABELS), +] + # --------------------------------------------------------------------------- # Helpers @@ -156,20 +186,40 @@ def jsr_with_carry(transport, addr, timeout=120.0, poll_interval=0.5): return result[0] -def check_label(labels, name): - """Return True if label exists, print skip message if not.""" - if labels.address(name) is None: - print(f" SKIP: label '{name}' not found (routine not yet implemented)") - return False - return True +# Groups that could not run this session because labels were missing. +# Entries are (group_name, [missing label names]). run_tests() resets this. +SKIPPED_GROUPS = [] + +def missing_labels(labels, label_list): + """Return the subset of label_list that the build does not export.""" + return [name for name in label_list if labels.address(name) is None] -def check_labels(labels, label_list): - """Return True if all labels in the list exist.""" - for name in label_list: - if labels.address(name) is None: - print(f" SKIP: label '{name}' not found -- skipping test group") - return False + +def preflight_required_labels(labels): + """Return [(group_name, [missing labels])] for declared groups that can't run.""" + broken = [] + for group_name, label_list in REQUIRED_LABEL_SETS: + missing = missing_labels(labels, label_list) + if missing: + broken.append((group_name, missing)) + return broken + + +def check_labels(labels, label_list, group="unnamed group"): + """Return True if all labels exist; otherwise record the group as skipped. + + A missing label is NOT a benign skip. It means the build no longer exports + a symbol this suite depends on, so the group's assertions never execute. + Recording the group here is what keeps it out of the "everything passed" + denominator — run_tests() turns each recorded entry into a failure. + """ + missing = missing_labels(labels, label_list) + if missing: + print(f" SKIP: {group}: label(s) {', '.join(missing)} not found " + f"-- group CANNOT RUN (counted as a failure)") + SKIPPED_GROUPS.append((group, missing)) + return False return True @@ -269,7 +319,7 @@ def test_der_parser_p256(transport, labels): passed = 0 failed = 0 - if not check_labels(labels, DER_LABELS): + if not check_labels(labels, DER_LABELS, "Group 1: DER Parser P-256"): return 0, 0 print("\n Generating P-256 self-signed certificate...") @@ -396,7 +446,7 @@ def test_der_parser_p384(transport, labels): passed = 0 failed = 0 - if not check_labels(labels, DER_LABELS): + if not check_labels(labels, DER_LABELS, "Group 2: DER Parser P-384"): return 0, 0 print("\n Generating P-384 self-signed certificate...") @@ -408,16 +458,22 @@ def test_der_parser_p384(transport, labels): # Load certificate to C64 load_cert_to_c64(transport, labels, cert_der) - # Parse the certificate + # Parse the certificate. + # A parse failure here is a FAILURE, not a skip. This used to report + # "SKIP: may not be supported yet" and return (0, 0), which would have + # hidden a P-384 DER regression completely — same shape as F3. P-384 + # certificate *parsing* works today (group 2 passes 2/2); it is only + # P-384 ECDSA *verify* that is stubbed at the TLS layer. try: carry = jsr_with_carry(transport, labels["x509_parse_cert"], timeout=120.0, poll_interval=0.5) if carry != 0: - print(" SKIP: P-384 parse returned C=1 (may not be supported yet)") - return 0, 0 + print(" [2!] FAIL: x509_parse_cert returned C=1 on a P-384 " + "certificate") + return 0, 1 except Exception as e: - print(f" SKIP: P-384 parse raised {e}") - return 0, 0 + print(f" [2!] FAIL: x509_parse_cert raised {e}") + return 0, 1 # --- Test 1: Curve ID correct --- print("\n [2a] DER parse P-384: curve_id = 1 (P-384)") @@ -500,7 +556,7 @@ def test_ecdsa_verify_p256(transport, labels): passed = 0 failed = 0 - if not check_labels(labels, ECDSA_LABELS): + if not check_labels(labels, ECDSA_LABELS, "Group 3: ECDSA P-256 Verify"): return 0, 0 print("\n Using hardcoded P-256 test vector (pre-verified in Python)") @@ -633,9 +689,16 @@ def run_tests(transport, labels): Order: DER parser first (fast, validates VICE), then ECDSA verify. CertificateVerify tests skipped until core verify is proven. + + A declared group that cannot run because its labels are missing is counted + as ONE FAILURE, and named in SKIPPED_GROUPS. That is deliberate: this + function is called directly by tools/run_all_tests.py, which only sees + (passed, failed), so a group vanishing from the denominator would otherwise + be indistinguishable from a clean run there too. """ total_passed = 0 total_failed = 0 + SKIPPED_GROUPS.clear() # --- DER parser tests (fast, ~seconds) --- test_groups = [ @@ -663,7 +726,7 @@ def run_tests(transport, labels): traceback.print_exc() # --- ECDSA verify tests (slow, minutes each) --- - ecdsa_ok = check_labels(labels, ECDSA_LABELS) + ecdsa_ok = check_labels(labels, ECDSA_LABELS, "Group 3: ECDSA P-256 Verify") if ecdsa_ok: # One-time sqtab_init before any ECDSA tests print(f"\n{'='*60}") @@ -696,6 +759,19 @@ def run_tests(transport, labels): # CertificateVerify tests skipped for now # (re-enable after core ECDSA verify is proven) + # A group that could not run is a failure, not a hole in the denominator. + if SKIPPED_GROUPS: + print(f"\n{'='*60}") + print(" GROUPS THAT COULD NOT RUN (counted as failures)") + print(f"{'='*60}") + for group, missing in SKIPPED_GROUPS: + print(f" [-] {group}: missing label(s) {', '.join(missing)}") + print(" A missing label means the build stopped exporting a symbol " + "this suite\n depends on (e.g. a libs/nistcurves bump renamed " + "an export). The group's\n assertions never executed, so this " + "run proves nothing about it.") + total_failed += len(SKIPPED_GROUPS) + return total_passed, total_failed @@ -745,17 +821,36 @@ def main(): labels = Labels.from_file(LABELS_PATH) # Check which test groups can run - der_ok = all(labels.address(n) is not None for n in DER_LABELS) - ecdsa_ok = all(labels.address(n) is not None for n in ECDSA_LABELS) - cv_ok = all(labels.address(n) is not None for n in CV_LABELS) + der_ok = not missing_labels(labels, DER_LABELS) + ecdsa_ok = not missing_labels(labels, ECDSA_LABELS) + cv_ok = not missing_labels(labels, CV_LABELS) print(f" Labels loaded from {LABELS_PATH}") print(f" DER parser labels: {'OK' if der_ok else 'MISSING'}") print(f" ECDSA verify labels: {'OK' if ecdsa_ok else 'MISSING'}") - print(f" CertificateVerify labels: {'OK' if cv_ok else 'MISSING'}") - - if not (der_ok or ecdsa_ok or cv_ok): - print("\nFATAL: No test group has all required labels. Nothing to test.") + print(f" CertificateVerify labels: " + f"{'OK' if cv_ok else 'MISSING'} (informational; no group uses these yet)") + + # Fail closed: every group in REQUIRED_LABEL_SETS is one this suite claims + # to run. If the build no longer exports the symbols it needs, the group's + # assertions cannot execute — and a suite that cannot execute its + # assertions has not passed. Abort here, before spending a VICE session + # producing a green result that covers less than it claims. + broken = preflight_required_labels(labels) + if broken: + names = "; ".join(f"{g} [missing: {', '.join(m)}]" for g, m in broken) + print(f"\n{'='*60}") + print("RESULTS") + print(f"{'='*60}") + print(f" Passed: 0/0") + print(f" Failed: 0/0") + print(f"\n [-] X.509/ECDSA: ABORTED -- declared test group(s) " + f"cannot run: {names}") + print(" A missing label means the build stopped exporting a symbol") + print(" this suite depends on (e.g. a libs/nistcurves bump renamed") + print(" an export). Skipping the group would report success while") + print(" testing nothing, so this is a failure.") + print(f"{'='*60}") sys.exit(1) # Estimate test duration @@ -793,16 +888,24 @@ def main(): mgr.release(inst) # Summary + no_tests_ran = (passed + failed) == 0 + if no_tests_ran: + # A suite that executed no assertions has not passed. + failed = 1 total = passed + failed print(f"\n{'='*60}") print("RESULTS") print(f"{'='*60}") print(f" Passed: {passed}/{total}") print(f" Failed: {failed}/{total}") - if total == 0: - print("\n [?] No tests ran (routines not yet implemented?)") + skipped = "; ".join(f"{g} [missing: {', '.join(m)}]" for g, m in SKIPPED_GROUPS) + if no_tests_ran: + print("\n [-] X.509/ECDSA: NO TESTS RAN -- nothing was verified") elif failed == 0: print(f"\n [+] X.509/ECDSA: ALL {total} TESTS PASSED") + elif skipped: + print(f"\n [-] X.509/ECDSA: {failed} TEST(S) FAILED " + f"-- group(s) could not run: {skipped}") else: print(f"\n [-] X.509/ECDSA: {failed} TEST(S) FAILED") print(f"{'='*60}") From 0eb9a6daaef9fa258b46c0b25619e2ec0056dcba Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Thu, 13 Aug 2026 06:28:58 -0500 Subject: [PATCH 2/3] test(ecdsa): give the KAT oracle negative vectors (F7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The oracle ran three CAVP vectors, all Result=P, all expecting C=0. That cannot distinguish a working verifier from one that answers "valid" unconditionally — against `ecdsa_verify` stubbed to `clc; rts` it reported 3/3 and exited 0. An oracle with no negative case does not test verification, it tests that the routine returns. Adds three genuine CAVP SigVer P-256/SHA-256 `Result = F` records, one per modification class: [3h] F (3 - S changed) [3i] F (2 - R changed) [3j] F (1 - Message changed) Every `Result = F` record in the file has Q on the curve and r, s in [1, n-1], so none of these can be rejected by a cheap range or point-validity gate — each forces the full verify math and the recovered-R.x comparison. Provenance, established before adding anything: all 15 records in libs/nistcurves/tools/vectors/nist_p256_sigver.rsp were checked host-side against OpenSSL (via `cryptography`) plus independent raw-int on-curve and range checks. Zero disagreements with the .rsp Result column. After transcription, each of the six vectors now in the file was matched byte-for-byte back to a .rsp record and re-verified through OpenSSL *as transcribed here* (prehashed), so a typo could not slip through. No vector was produced by running this implementation. Mechanically: each vector carries `expect_carry` (mirroring the .rsp Result column, never an observed C64 result); the runner compares against it and says plainly when a NIST-invalid signature was accepted. Two structural guards keep F7 from regressing — the run aborts if the vector set ever contains zero negatives, and it fails if any declared vector does not produce a verdict. Acceptance (VICE, ip65 build, -reu via default_vice_config): pristine "[+] ECDSA KAT oracle: ALL 6 VECTORS PASSED (3 valid accepted, 3 invalid rejected)" EXIT_CODE=0 mutant: `clc; rts` at the top of ecdsa_verify (always VALID) verified present in the built binary, not just in the source — `ecdsa_verify` resolved from build/labels.txt to $6B23, mapped through the PRG load address $0801 to file offset 25378, bytes there read `18 60 ...` = CLC, RTS " Passed: 3/6 / Failed: 3/6" "[-] ECDSA KAT oracle: 3 VECTOR(S) FAILED" EXIT_CODE=1 The mutant's three passes are exactly the three vectors this file used to contain — i.e. the old oracle would have reported a clean 3/3 against a verifier that validates everything. The mutation was reverted before commit and the rebuilt PRG hashes identical to the pristine build (db31111031e2f30c52c9116576d74f21e2cc6ddee025041a881345efa7ba8f60). Co-Authored-By: Claude Opus 5 (1M context) --- tools/test_ecdsa_kat_oracle.py | 185 ++++++++++++++++++++++++++++----- 1 file changed, 161 insertions(+), 24 deletions(-) diff --git a/tools/test_ecdsa_kat_oracle.py b/tools/test_ecdsa_kat_oracle.py index 1ac6334..415899c 100644 --- a/tools/test_ecdsa_kat_oracle.py +++ b/tools/test_ecdsa_kat_oracle.py @@ -1,24 +1,51 @@ #!/usr/bin/env python3 """test_ecdsa_kat_oracle.py - Library-side KAT oracle for ECDSA P-256 verify. -Runs additional known-VALID P-256/SHA-256 signature vectors against the -C64's `ecdsa_verify` routine (the c64-https dispatcher over the -libs/nistcurves sibling). Mirrors the structure of -`tools/test_x509.py` group 3 subtest [3c] (call `setup_ecdsa_verify(...)`, -then `jsr_with_carry(... labels["ecdsa_verify"] ...)`, assert C=0) but -exercises 3 additional vectors so we can distinguish a primitive bug -from a [3c]-specific test-setup bug: - - - [3e] CAVP SigVer P-256/SHA-256 valid #1 (Result = P record) - - [3f] CAVP SigVer P-256/SHA-256 valid #2 - - [3g] CAVP SigVer P-256/SHA-256 valid #3 - +Runs CAVP P-256/SHA-256 signature vectors against the C64's `ecdsa_verify` +routine (the c64-https dispatcher over the libs/nistcurves sibling). Mirrors +the structure of `tools/test_x509.py` group 3 subtest [3c] (call +`setup_ecdsa_verify(...)`, then `jsr_with_carry(... labels["ecdsa_verify"] ...)`, +assert the carry) but exercises additional vectors so we can distinguish a +primitive bug from a [3c]-specific test-setup bug: + + - [3e] CAVP SigVer P-256/SHA-256 valid #1 (Result = P) -> expect C=0 + - [3f] CAVP SigVer P-256/SHA-256 valid #2 (Result = P) -> expect C=0 + - [3g] CAVP SigVer P-256/SHA-256 valid #3 (Result = P) -> expect C=0 + - [3h] CAVP SigVer P-256/SHA-256 invalid #1 (Result = F, S changed) -> C=1 + - [3i] CAVP SigVer P-256/SHA-256 invalid #2 (Result = F, R changed) -> C=1 + - [3j] CAVP SigVer P-256/SHA-256 invalid #3 (Result = F, Msg changed) -> C=1 + +Negative vectors (audit finding F7) +----------------------------------- +This oracle originally ran three vectors, all valid, all expecting C=0. +That cannot distinguish a working verifier from one that reports "valid" +unconditionally: against an `ecdsa_verify` stubbed to `clc; rts` it happily +reported 3/3. An oracle with no negative case does not test verification, +it tests that the routine returns. + +The three `Result = F` records above close that. They are genuine CAVP +records, not signatures manufactured by mutating a valid one and not +anything produced by running this implementation. + +Every `Result = F` record in the file has Q on the curve and r, s in +[1, n-1] (checked host-side, see below), so none of them can be rejected by +a cheap range or point-validity gate — each one forces the full verify math +and compares the recovered R.x against r. One record per CAVP modification +class is included: 3 (S changed), 2 (R changed), 1 (Message changed). + +Provenance +---------- Vectors are extracted verbatim from `libs/nistcurves/tools/vectors/nist_p256_sigver.rsp` (NIST CAVP SigVer, -P-256/SHA-256 section), specifically the records flagged `Result = P`. -For each vector the hash is `SHA-256(Msg)`; r/s/Qx/Qy are taken straight -from the .rsp file in big-endian wire order, matching the BE struct ABI -of the sibling's `ecdsa_verify_256`. +P-256/SHA-256 section). For each vector the hash is `SHA-256(Msg)`; +r/s/Qx/Qy are taken straight from the .rsp file in big-endian wire order, +matching the BE struct ABI of the sibling's `ecdsa_verify_256`. + +Every vector below — positive and negative — was independently confirmed +host-side against OpenSSL via the `cryptography` package before being added +here: all 15 records in the .rsp agreed with their Result column, with Q on +curve and r, s in range. The `expect_carry` field encodes the .rsp Result +column (P -> 0, F -> 1), never an observed C64 result. Usage: python3 tools/test_ecdsa_kat_oracle.py [--verbose] @@ -71,13 +98,19 @@ # --------------------------------------------------------------------------- -# Hardcoded P-256 known-VALID KAT vectors (CAVP SigVer, Result = P records) +# Hardcoded P-256 KAT vectors (CAVP SigVer, both Result = P and Result = F) +# +# `expect_carry` mirrors the .rsp Result column: P (valid) -> C=0, +# F (invalid) -> C=1. See the module docstring for provenance and for why +# the negative records are load-bearing (audit finding F7). # --------------------------------------------------------------------------- KAT_VECTORS = [ + # --- Result = P (valid) -------------------------------------------- # CAVP SigVer P-256/SHA-256 valid record #1 dict( tag="CAVP SigVer P-256/SHA-256 valid #1", + expect_carry=0, hash=bytes.fromhex( "d1b8ef21eb4182ee270638061063a3f3" "c16c114e33937f69fb232cc833965a94"), @@ -97,6 +130,7 @@ # CAVP SigVer P-256/SHA-256 valid record #2 dict( tag="CAVP SigVer P-256/SHA-256 valid #2", + expect_carry=0, hash=bytes.fromhex( "b9336a8d1f3e8ede001d19f41320bc76" "72d772a3d2cb0e435fff3c27d6804a2c"), @@ -116,6 +150,7 @@ # CAVP SigVer P-256/SHA-256 valid record #3 dict( tag="CAVP SigVer P-256/SHA-256 valid #3", + expect_carry=0, hash=bytes.fromhex( "41007876926a20f821d72d9c6f2c9dae" "6c03954123ea6e6939d7e6e669438891"), @@ -132,9 +167,76 @@ "9b52672742d637a32add056dfd6d8792" "f2a33c2e69dafabea09b960bc61e230a"), ), + + # --- Result = F (invalid) ------------------------------------------ + # These are what make this file an oracle rather than a smoke test: + # a verify that answers "valid" unconditionally passes every vector + # above and fails every vector below. Q is on the curve and r, s are in + # [1, n-1] for all three, so none is rejectable by a cheap gate. + # + # CAVP SigVer P-256/SHA-256 invalid record, "Result = F (3 - S changed)" + dict( + tag="CAVP SigVer P-256/SHA-256 invalid #1 (F: S changed)", + expect_carry=1, + hash=bytes.fromhex( + "a82c31412f537135d1c418bd7136fb5f" + "de9426e70c70e7c2fb11f02f30fdeae2"), + r=bytes.fromhex( + "d19ff48b324915576416097d2544f7cb" + "df8768b1454ad20e0baac50e211f23b0"), + s=bytes.fromhex( + "a3e81e59311cdfff2d4784949f7a2cb5" + "0ba6c3a91fa54710568e61aca3e847c6"), + qx=bytes.fromhex( + "87f8f2b218f49845f6f10eec38771362" + "69f5c1a54736dbdf69f89940cad41555"), + qy=bytes.fromhex( + "e15f369036f49842fac7a86c8a2b0557" + "609776814448b8f5e84aa9f4395205e9"), + ), + # CAVP SigVer P-256/SHA-256 invalid record, "Result = F (2 - R changed)" + dict( + tag="CAVP SigVer P-256/SHA-256 invalid #2 (F: R changed)", + expect_carry=1, + hash=bytes.fromhex( + "5984eab8854d0a9aa5f0c70f96deeb51" + "0e5f9ff8c51befcdc3c41bac53577f22"), + r=bytes.fromhex( + "dc23d130c6117fb5751201455e99f36f" + "59aba1a6a21cf2d0e7481a97451d6693"), + s=bytes.fromhex( + "d6ce7708c18dbf35d4f8aa7240922dc6" + "823f2e7058cbc1484fcad1599db5018c"), + qx=bytes.fromhex( + "5cf02a00d205bdfee2016f7421807fc3" + "8ae69e6b7ccd064ee689fc1a94a9f7d2"), + qy=bytes.fromhex( + "ec530ce3cc5c9d1af463f264d685afe2" + "b4db4b5828d7e61b748930f3ce622a85"), + ), + # CAVP SigVer P-256/SHA-256 invalid record, "Result = F (1 - Message changed)" + dict( + tag="CAVP SigVer P-256/SHA-256 invalid #3 (F: Message changed)", + expect_carry=1, + hash=bytes.fromhex( + "d80e9933e86769731ec16ff31e682153" + "1bcf07fcbad9e2ac16ec9e6cb343a870"), + r=bytes.fromhex( + "288f7a1cd391842cce21f00e6f15471c" + "04dc182fe4b14d92dc18910879799790"), + s=bytes.fromhex( + "247b3c4e89a3bcadfea73c7bfd361def" + "43715fa382b8c3edf4ae15d6e55e9979"), + qx=bytes.fromhex( + "69b7667056e1e11d6caf6e45643f8b21" + "e7a4bebda463c7fdbc13bc98efbd0214"), + qy=bytes.fromhex( + "d3f9b12eb46c7c6fda0da3fc85bc1fd8" + "31557f9abc902a3be3cb3e8be7d1aa2f"), + ), ] -SUBTEST_LABELS = ["3e", "3f", "3g"] +SUBTEST_LABELS = ["3e", "3f", "3g", "3h", "3i", "3j"] # --------------------------------------------------------------------------- @@ -209,7 +311,9 @@ def run_kat_oracle(transport, labels): for idx, vec in enumerate(KAT_VECTORS): sub = SUBTEST_LABELS[idx] tag = vec["tag"] - print(f"\n [{sub}] ECDSA verify: {tag} (expected C=0)") + want = vec["expect_carry"] + want_word = "valid" if want == 0 else "INVALID" + print(f"\n [{sub}] ECDSA verify: {tag} (expected C={want}, {want_word})") if VERBOSE: print(f" hash = {vec['hash'][:8].hex()}... r = {vec['r'][:8].hex()}...") print(f" s = {vec['s'][:8].hex()}... Qx = {vec['qx'][:8].hex()}... Qy = {vec['qy'][:8].hex()}...") @@ -240,12 +344,19 @@ def run_kat_oracle(transport, labels): carry = jsr_with_carry(transport, labels["ecdsa_verify"], timeout=2400.0, poll_interval=30.0) elapsed = time.time() - t0 - if carry == 0: + got_word = "valid" if carry == 0 else "invalid" + if carry == want: passed += 1 - print(f" PASS: ecdsa_verify returned C=0 (valid) [{elapsed:.0f}s]") + print(f" PASS: ecdsa_verify returned C={carry} " + f"({got_word}) [{elapsed:.0f}s]") else: failed += 1 - print(f" FAIL: ecdsa_verify returned C=1 (invalid) [{elapsed:.0f}s]") + print(f" FAIL: ecdsa_verify returned C={carry} ({got_word}), " + f"expected C={want} ({want_word}) [{elapsed:.0f}s]") + if want == 1: + print(" A CAVP Result=F vector was accepted. The " + "verifier is reporting") + print(" signatures valid that NIST says are not.") print(f" hash: {c64_hash.hex()}") print(f" r: {c64_r.hex()}") print(f" s: {c64_s.hex()}") @@ -288,8 +399,21 @@ def main(): print("\nFATAL: ECDSA verify labels missing; nothing to test.") sys.exit(1) + n_pos = sum(1 for v in KAT_VECTORS if v["expect_carry"] == 0) + n_neg = sum(1 for v in KAT_VECTORS if v["expect_carry"] == 1) + + # Structural guard against audit finding F7 regressing: an oracle made up + # entirely of valid signatures cannot fail against a verifier that always + # answers "valid", so it is not an oracle. + if n_neg == 0: + print("\nFATAL: KAT_VECTORS contains no Result=F (expect_carry=1) vector.") + print(" An all-positive vector set passes against a verify stubbed") + print(" to 'clc; rts' and therefore proves nothing. See F7.") + sys.exit(1) + print(f"\n Labels loaded from {LABELS_PATH}") - print(f" Vectors to run: {len(KAT_VECTORS)} (CAVP SigVer P-256/SHA-256 valid)") + print(f" Vectors to run: {len(KAT_VECTORS)} CAVP SigVer P-256/SHA-256 " + f"({n_pos} valid / {n_neg} invalid)") print(f" Per-vector wallclock budget: 2400 s (VICE warp; typical ~5-16 min)") config = default_vice_config(prg_path=PRG_PATH, warp=True, ntsc=True, sound=False) @@ -318,7 +442,8 @@ def main(): print(f" sqtab_init FAILED: {e}") sys.exit(1) - print(f"\n=== ECDSA P-256 KAT oracle ({len(KAT_VECTORS)} valid vectors) ===") + print(f"\n=== ECDSA P-256 KAT oracle " + f"({n_pos} valid + {n_neg} invalid vectors) ===") passed, failed = run_kat_oracle(transport, labels) mgr.release(inst) @@ -329,6 +454,18 @@ def main(): print(f"{'='*60}") print(f" Passed: {passed}/{total}") print(f" Failed: {failed}/{total}") + if total != len(KAT_VECTORS): + # Every declared vector must produce a verdict; a vector that silently + # dropped out is the same class of defect as F3's skipped group. + print(f"\n [-] ECDSA KAT oracle: only {total} of " + f"{len(KAT_VECTORS)} declared vectors produced a verdict") + sys.exit(1) + if failed == 0: + print(f"\n [+] ECDSA KAT oracle: ALL {total} VECTORS PASSED " + f"({n_pos} valid accepted, {n_neg} invalid rejected)") + else: + print(f"\n [-] ECDSA KAT oracle: {failed} VECTOR(S) FAILED") + print(f"{'='*60}") sys.exit(0 if failed == 0 else 1) From 7497e48a0b9b154f345b0f7e1e2fb2216b125c0d Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Thu, 13 Aug 2026 06:40:25 -0500 Subject: [PATCH 3/3] fix(test): --skip-slow silently dropped the entire x509 suite (F3, second door) `tools/run_all_tests.py --skip-slow` omitted "x509" from the suite list by never adding it, so the aggregate printed a TOTAL and exited 0 with no trace that the whole X.509/ECDSA suite had not run. That is F3's shape one level up: the skipped assertions left the denominator instead of being accounted for, and the dropped suite is precisely the one F3 is about. The numbers make the failure mode concrete. A full aggregate run reports TOTAL: 264/264 with `x509 11/11`; `--skip-slow` reported TOTAL: 253/253. Same "everything passed" shape, 11 assertions lighter, and nothing in the output says which 11 or why. Skipped suites are now recorded as (name, reason) and surfaced three ways: named on the TOTAL line, given their own SKIP row in the per-suite table, and followed by an explicit warning that the aggregate does not certify them. Exit code stays 0. An operator passing --skip-slow made a deliberate choice, unlike F3's missing label, which is an unrequested environmental failure. The rule the two cases share is the one that matters: an involuntary skip is a failure, an explicit skip is allowed but must never be silent. Same convention Lane G adopted for test_x25519.py's --fast gate in PR #81. Acceptance (`--skip-slow --workers 4`, real aggregate runs): before TOTAL: 253/253 passed, 0 failed (no x509 row, no mention anywhere in the output) EXIT_CODE=0 after TOTAL: 253/253 passed, 0 failed -- 1 suite(s) SKIPPED: x509 ... SKIP x509 --- did not run: --skip-slow (X.509 DER parsing + ECDSA P-256 verify) WARNING: the suite(s) above did not run. This aggregate result does not certify them, and their assertions are absent from the TOTAL. EXIT_CODE=0 Reported by Lane G while reviewing the F3 fix. Co-Authored-By: Claude Opus 5 (1M context) --- tools/run_all_tests.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tools/run_all_tests.py b/tools/run_all_tests.py index cf5403f..0e33447 100644 --- a/tools/run_all_tests.py +++ b/tools/run_all_tests.py @@ -130,7 +130,19 @@ def main(): suites = ["entropy", "net", "sha256", "crypto", "hkdf", "keyschedule", "http", "tls_record", "tls_handshake", "x25519"] - if not skip_slow: + + # Suites deliberately not run this session, as (name, reason). --skip-slow + # used to drop x509 by simply never adding it to the list, so the aggregate + # printed a TOTAL and exited 0 with no trace that the entire X.509/ECDSA + # suite had not run. That is audit finding F3's shape one level up: the + # skipped assertions left the denominator instead of being accounted for. + # An explicit operator flag is a legitimate reason to skip; it is not a + # licence to report an unqualified clean pass. + skipped_suites = [] + if skip_slow: + skipped_suites.append( + ("x509", "--skip-slow (X.509 DER parsing + ECDSA P-256 verify)")) + else: suites.insert(0, "x509") config = ViceConfig(prg_path=PRG_PATH, warp=True, ntsc=True, sound=False, @@ -176,13 +188,25 @@ def run_suite_in_own_instance(mgr, suite_name): total_failed = sum(r[2] for r in results) total_tests = total_passed + total_failed + skipped_note = "" + if skipped_suites: + skipped_note = (f" -- {len(skipped_suites)} suite(s) SKIPPED: " + + ", ".join(n for n, _ in skipped_suites)) + print(f"\n{'='*60}") print(f"TOTAL: {total_passed}/{total_tests} passed, " - f"{total_failed} failed") + f"{total_failed} failed{skipped_note}") for name, passed, failed, duration in sorted(results): status = "OK" if failed == 0 else "FAIL" print(f" {status:4s} {name:20s} {passed:3d}/{passed+failed:3d} " f"({duration:.1f}s)") + for name, reason in skipped_suites: + print(f" SKIP {name:20s} --- did not run: {reason}") + if skipped_suites: + print("\n WARNING: the suite(s) above did not run. This aggregate " + "result does not") + print(" certify them, and their assertions are absent from " + "the TOTAL.") print(f"{'='*60}") sys.exit(0 if total_failed == 0 else 1)