diff --git a/tools/run_all_tests.py b/tools/run_all_tests.py index cf5403f..0e33447 100644 --- a/tools/run_all_tests.py +++ b/tools/run_all_tests.py @@ -130,7 +130,19 @@ def main(): suites = ["entropy", "net", "sha256", "crypto", "hkdf", "keyschedule", "http", "tls_record", "tls_handshake", "x25519"] - if not skip_slow: + + # Suites deliberately not run this session, as (name, reason). --skip-slow + # used to drop x509 by simply never adding it to the list, so the aggregate + # printed a TOTAL and exited 0 with no trace that the entire X.509/ECDSA + # suite had not run. That is audit finding F3's shape one level up: the + # skipped assertions left the denominator instead of being accounted for. + # An explicit operator flag is a legitimate reason to skip; it is not a + # licence to report an unqualified clean pass. + skipped_suites = [] + if skip_slow: + skipped_suites.append( + ("x509", "--skip-slow (X.509 DER parsing + ECDSA P-256 verify)")) + else: suites.insert(0, "x509") config = ViceConfig(prg_path=PRG_PATH, warp=True, ntsc=True, sound=False, @@ -176,13 +188,25 @@ def run_suite_in_own_instance(mgr, suite_name): total_failed = sum(r[2] for r in results) total_tests = total_passed + total_failed + skipped_note = "" + if skipped_suites: + skipped_note = (f" -- {len(skipped_suites)} suite(s) SKIPPED: " + + ", ".join(n for n, _ in skipped_suites)) + print(f"\n{'='*60}") print(f"TOTAL: {total_passed}/{total_tests} passed, " - f"{total_failed} failed") + f"{total_failed} failed{skipped_note}") for name, passed, failed, duration in sorted(results): status = "OK" if failed == 0 else "FAIL" print(f" {status:4s} {name:20s} {passed:3d}/{passed+failed:3d} " f"({duration:.1f}s)") + for name, reason in skipped_suites: + print(f" SKIP {name:20s} --- did not run: {reason}") + if skipped_suites: + print("\n WARNING: the suite(s) above did not run. This aggregate " + "result does not") + print(" certify them, and their assertions are absent from " + "the TOTAL.") print(f"{'='*60}") sys.exit(0 if total_failed == 0 else 1) diff --git a/tools/test_ecdsa_kat_oracle.py b/tools/test_ecdsa_kat_oracle.py index 1ac6334..415899c 100644 --- a/tools/test_ecdsa_kat_oracle.py +++ b/tools/test_ecdsa_kat_oracle.py @@ -1,24 +1,51 @@ #!/usr/bin/env python3 """test_ecdsa_kat_oracle.py - Library-side KAT oracle for ECDSA P-256 verify. -Runs additional known-VALID P-256/SHA-256 signature vectors against the -C64's `ecdsa_verify` routine (the c64-https dispatcher over the -libs/nistcurves sibling). Mirrors the structure of -`tools/test_x509.py` group 3 subtest [3c] (call `setup_ecdsa_verify(...)`, -then `jsr_with_carry(... labels["ecdsa_verify"] ...)`, assert C=0) but -exercises 3 additional vectors so we can distinguish a primitive bug -from a [3c]-specific test-setup bug: - - - [3e] CAVP SigVer P-256/SHA-256 valid #1 (Result = P record) - - [3f] CAVP SigVer P-256/SHA-256 valid #2 - - [3g] CAVP SigVer P-256/SHA-256 valid #3 - +Runs CAVP P-256/SHA-256 signature vectors against the C64's `ecdsa_verify` +routine (the c64-https dispatcher over the libs/nistcurves sibling). Mirrors +the structure of `tools/test_x509.py` group 3 subtest [3c] (call +`setup_ecdsa_verify(...)`, then `jsr_with_carry(... labels["ecdsa_verify"] ...)`, +assert the carry) but exercises additional vectors so we can distinguish a +primitive bug from a [3c]-specific test-setup bug: + + - [3e] CAVP SigVer P-256/SHA-256 valid #1 (Result = P) -> expect C=0 + - [3f] CAVP SigVer P-256/SHA-256 valid #2 (Result = P) -> expect C=0 + - [3g] CAVP SigVer P-256/SHA-256 valid #3 (Result = P) -> expect C=0 + - [3h] CAVP SigVer P-256/SHA-256 invalid #1 (Result = F, S changed) -> C=1 + - [3i] CAVP SigVer P-256/SHA-256 invalid #2 (Result = F, R changed) -> C=1 + - [3j] CAVP SigVer P-256/SHA-256 invalid #3 (Result = F, Msg changed) -> C=1 + +Negative vectors (audit finding F7) +----------------------------------- +This oracle originally ran three vectors, all valid, all expecting C=0. +That cannot distinguish a working verifier from one that reports "valid" +unconditionally: against an `ecdsa_verify` stubbed to `clc; rts` it happily +reported 3/3. An oracle with no negative case does not test verification, +it tests that the routine returns. + +The three `Result = F` records above close that. They are genuine CAVP +records, not signatures manufactured by mutating a valid one and not +anything produced by running this implementation. + +Every `Result = F` record in the file has Q on the curve and r, s in +[1, n-1] (checked host-side, see below), so none of them can be rejected by +a cheap range or point-validity gate — each one forces the full verify math +and compares the recovered R.x against r. One record per CAVP modification +class is included: 3 (S changed), 2 (R changed), 1 (Message changed). + +Provenance +---------- Vectors are extracted verbatim from `libs/nistcurves/tools/vectors/nist_p256_sigver.rsp` (NIST CAVP SigVer, -P-256/SHA-256 section), specifically the records flagged `Result = P`. -For each vector the hash is `SHA-256(Msg)`; r/s/Qx/Qy are taken straight -from the .rsp file in big-endian wire order, matching the BE struct ABI -of the sibling's `ecdsa_verify_256`. +P-256/SHA-256 section). For each vector the hash is `SHA-256(Msg)`; +r/s/Qx/Qy are taken straight from the .rsp file in big-endian wire order, +matching the BE struct ABI of the sibling's `ecdsa_verify_256`. + +Every vector below — positive and negative — was independently confirmed +host-side against OpenSSL via the `cryptography` package before being added +here: all 15 records in the .rsp agreed with their Result column, with Q on +curve and r, s in range. The `expect_carry` field encodes the .rsp Result +column (P -> 0, F -> 1), never an observed C64 result. Usage: python3 tools/test_ecdsa_kat_oracle.py [--verbose] @@ -71,13 +98,19 @@ # --------------------------------------------------------------------------- -# Hardcoded P-256 known-VALID KAT vectors (CAVP SigVer, Result = P records) +# Hardcoded P-256 KAT vectors (CAVP SigVer, both Result = P and Result = F) +# +# `expect_carry` mirrors the .rsp Result column: P (valid) -> C=0, +# F (invalid) -> C=1. See the module docstring for provenance and for why +# the negative records are load-bearing (audit finding F7). # --------------------------------------------------------------------------- KAT_VECTORS = [ + # --- Result = P (valid) -------------------------------------------- # CAVP SigVer P-256/SHA-256 valid record #1 dict( tag="CAVP SigVer P-256/SHA-256 valid #1", + expect_carry=0, hash=bytes.fromhex( "d1b8ef21eb4182ee270638061063a3f3" "c16c114e33937f69fb232cc833965a94"), @@ -97,6 +130,7 @@ # CAVP SigVer P-256/SHA-256 valid record #2 dict( tag="CAVP SigVer P-256/SHA-256 valid #2", + expect_carry=0, hash=bytes.fromhex( "b9336a8d1f3e8ede001d19f41320bc76" "72d772a3d2cb0e435fff3c27d6804a2c"), @@ -116,6 +150,7 @@ # CAVP SigVer P-256/SHA-256 valid record #3 dict( tag="CAVP SigVer P-256/SHA-256 valid #3", + expect_carry=0, hash=bytes.fromhex( "41007876926a20f821d72d9c6f2c9dae" "6c03954123ea6e6939d7e6e669438891"), @@ -132,9 +167,76 @@ "9b52672742d637a32add056dfd6d8792" "f2a33c2e69dafabea09b960bc61e230a"), ), + + # --- Result = F (invalid) ------------------------------------------ + # These are what make this file an oracle rather than a smoke test: + # a verify that answers "valid" unconditionally passes every vector + # above and fails every vector below. Q is on the curve and r, s are in + # [1, n-1] for all three, so none is rejectable by a cheap gate. + # + # CAVP SigVer P-256/SHA-256 invalid record, "Result = F (3 - S changed)" + dict( + tag="CAVP SigVer P-256/SHA-256 invalid #1 (F: S changed)", + expect_carry=1, + hash=bytes.fromhex( + "a82c31412f537135d1c418bd7136fb5f" + "de9426e70c70e7c2fb11f02f30fdeae2"), + r=bytes.fromhex( + "d19ff48b324915576416097d2544f7cb" + "df8768b1454ad20e0baac50e211f23b0"), + s=bytes.fromhex( + "a3e81e59311cdfff2d4784949f7a2cb5" + "0ba6c3a91fa54710568e61aca3e847c6"), + qx=bytes.fromhex( + "87f8f2b218f49845f6f10eec38771362" + "69f5c1a54736dbdf69f89940cad41555"), + qy=bytes.fromhex( + "e15f369036f49842fac7a86c8a2b0557" + "609776814448b8f5e84aa9f4395205e9"), + ), + # CAVP SigVer P-256/SHA-256 invalid record, "Result = F (2 - R changed)" + dict( + tag="CAVP SigVer P-256/SHA-256 invalid #2 (F: R changed)", + expect_carry=1, + hash=bytes.fromhex( + "5984eab8854d0a9aa5f0c70f96deeb51" + "0e5f9ff8c51befcdc3c41bac53577f22"), + r=bytes.fromhex( + "dc23d130c6117fb5751201455e99f36f" + "59aba1a6a21cf2d0e7481a97451d6693"), + s=bytes.fromhex( + "d6ce7708c18dbf35d4f8aa7240922dc6" + "823f2e7058cbc1484fcad1599db5018c"), + qx=bytes.fromhex( + "5cf02a00d205bdfee2016f7421807fc3" + "8ae69e6b7ccd064ee689fc1a94a9f7d2"), + qy=bytes.fromhex( + "ec530ce3cc5c9d1af463f264d685afe2" + "b4db4b5828d7e61b748930f3ce622a85"), + ), + # CAVP SigVer P-256/SHA-256 invalid record, "Result = F (1 - Message changed)" + dict( + tag="CAVP SigVer P-256/SHA-256 invalid #3 (F: Message changed)", + expect_carry=1, + hash=bytes.fromhex( + "d80e9933e86769731ec16ff31e682153" + "1bcf07fcbad9e2ac16ec9e6cb343a870"), + r=bytes.fromhex( + "288f7a1cd391842cce21f00e6f15471c" + "04dc182fe4b14d92dc18910879799790"), + s=bytes.fromhex( + "247b3c4e89a3bcadfea73c7bfd361def" + "43715fa382b8c3edf4ae15d6e55e9979"), + qx=bytes.fromhex( + "69b7667056e1e11d6caf6e45643f8b21" + "e7a4bebda463c7fdbc13bc98efbd0214"), + qy=bytes.fromhex( + "d3f9b12eb46c7c6fda0da3fc85bc1fd8" + "31557f9abc902a3be3cb3e8be7d1aa2f"), + ), ] -SUBTEST_LABELS = ["3e", "3f", "3g"] +SUBTEST_LABELS = ["3e", "3f", "3g", "3h", "3i", "3j"] # --------------------------------------------------------------------------- @@ -209,7 +311,9 @@ def run_kat_oracle(transport, labels): for idx, vec in enumerate(KAT_VECTORS): sub = SUBTEST_LABELS[idx] tag = vec["tag"] - print(f"\n [{sub}] ECDSA verify: {tag} (expected C=0)") + want = vec["expect_carry"] + want_word = "valid" if want == 0 else "INVALID" + print(f"\n [{sub}] ECDSA verify: {tag} (expected C={want}, {want_word})") if VERBOSE: print(f" hash = {vec['hash'][:8].hex()}... r = {vec['r'][:8].hex()}...") print(f" s = {vec['s'][:8].hex()}... Qx = {vec['qx'][:8].hex()}... Qy = {vec['qy'][:8].hex()}...") @@ -240,12 +344,19 @@ def run_kat_oracle(transport, labels): carry = jsr_with_carry(transport, labels["ecdsa_verify"], timeout=2400.0, poll_interval=30.0) elapsed = time.time() - t0 - if carry == 0: + got_word = "valid" if carry == 0 else "invalid" + if carry == want: passed += 1 - print(f" PASS: ecdsa_verify returned C=0 (valid) [{elapsed:.0f}s]") + print(f" PASS: ecdsa_verify returned C={carry} " + f"({got_word}) [{elapsed:.0f}s]") else: failed += 1 - print(f" FAIL: ecdsa_verify returned C=1 (invalid) [{elapsed:.0f}s]") + print(f" FAIL: ecdsa_verify returned C={carry} ({got_word}), " + f"expected C={want} ({want_word}) [{elapsed:.0f}s]") + if want == 1: + print(" A CAVP Result=F vector was accepted. The " + "verifier is reporting") + print(" signatures valid that NIST says are not.") print(f" hash: {c64_hash.hex()}") print(f" r: {c64_r.hex()}") print(f" s: {c64_s.hex()}") @@ -288,8 +399,21 @@ def main(): print("\nFATAL: ECDSA verify labels missing; nothing to test.") sys.exit(1) + n_pos = sum(1 for v in KAT_VECTORS if v["expect_carry"] == 0) + n_neg = sum(1 for v in KAT_VECTORS if v["expect_carry"] == 1) + + # Structural guard against audit finding F7 regressing: an oracle made up + # entirely of valid signatures cannot fail against a verifier that always + # answers "valid", so it is not an oracle. + if n_neg == 0: + print("\nFATAL: KAT_VECTORS contains no Result=F (expect_carry=1) vector.") + print(" An all-positive vector set passes against a verify stubbed") + print(" to 'clc; rts' and therefore proves nothing. See F7.") + sys.exit(1) + print(f"\n Labels loaded from {LABELS_PATH}") - print(f" Vectors to run: {len(KAT_VECTORS)} (CAVP SigVer P-256/SHA-256 valid)") + print(f" Vectors to run: {len(KAT_VECTORS)} CAVP SigVer P-256/SHA-256 " + f"({n_pos} valid / {n_neg} invalid)") print(f" Per-vector wallclock budget: 2400 s (VICE warp; typical ~5-16 min)") config = default_vice_config(prg_path=PRG_PATH, warp=True, ntsc=True, sound=False) @@ -318,7 +442,8 @@ def main(): print(f" sqtab_init FAILED: {e}") sys.exit(1) - print(f"\n=== ECDSA P-256 KAT oracle ({len(KAT_VECTORS)} valid vectors) ===") + print(f"\n=== ECDSA P-256 KAT oracle " + f"({n_pos} valid + {n_neg} invalid vectors) ===") passed, failed = run_kat_oracle(transport, labels) mgr.release(inst) @@ -329,6 +454,18 @@ def main(): print(f"{'='*60}") print(f" Passed: {passed}/{total}") print(f" Failed: {failed}/{total}") + if total != len(KAT_VECTORS): + # Every declared vector must produce a verdict; a vector that silently + # dropped out is the same class of defect as F3's skipped group. + print(f"\n [-] ECDSA KAT oracle: only {total} of " + f"{len(KAT_VECTORS)} declared vectors produced a verdict") + sys.exit(1) + if failed == 0: + print(f"\n [+] ECDSA KAT oracle: ALL {total} VECTORS PASSED " + f"({n_pos} valid accepted, {n_neg} invalid rejected)") + else: + print(f"\n [-] ECDSA KAT oracle: {failed} VECTOR(S) FAILED") + print(f"{'='*60}") sys.exit(0 if failed == 0 else 1) diff --git a/tools/test_x509.py b/tools/test_x509.py index 238ede6..b78c2c6 100644 --- a/tools/test_x509.py +++ b/tools/test_x509.py @@ -10,6 +10,22 @@ python3 tools/test_x509.py [--seed S] [--verbose] Requires: Python 3.10+, c64_test_harness, VICE x64sc, cryptography + +Skip policy (audit finding F3) +------------------------------ +This suite used to treat a missing label as a benign skip: dropping the +single symbol ``ecdsa_verify`` from ``build/labels.txt`` silently deleted the +whole ECDSA group, and the run still reported "ALL 7 TESTS PASSED" and exited +0. The skipped assertions left the denominator instead of counting against it. + +The realistic trigger is a ``libs/nistcurves`` bump renaming an export, which +is precisely the change this suite exists to catch. + +Policy now: every group listed in ``REQUIRED_LABEL_SETS`` is required. If its +labels are missing, ``main()`` aborts before launching VICE (exit 1, summary +names the group), and ``run_tests()`` — which ``tools/run_all_tests.py`` calls +directly — counts each such group as a failure. Optional/unwired label sets +(``CV_LABELS``) are deliberately not in that list. """ import datetime @@ -78,13 +94,27 @@ "sqtab_init", ] -# Labels for CertificateVerify tests +# Labels for CertificateVerify tests. +# NOTE: no group currently drives these — the CertificateVerify tests are not +# wired up (see run_tests()). They are therefore NOT in REQUIRED_LABEL_SETS: +# a group that does not exist cannot be silently skipped. CV_LABELS = [ "tls_handle_cert_verify", "tls_rec_buf", "tls_rec_len", "tls_transcript", ] +# Every test group this suite claims to run, with the labels it needs. +# +# These are REQUIRED, not optional. If a build stops exporting one of these +# symbols — the realistic trigger being a libs/nistcurves bump that renames an +# export — the affected group must be reported as a FAILURE, never quietly +# dropped from the denominator. See the module docstring's "silent skip" note. +REQUIRED_LABEL_SETS = [ + ("DER parser (groups 1-2)", DER_LABELS), + ("ECDSA P-256 verify (group 3)", ECDSA_LABELS), +] + # --------------------------------------------------------------------------- # Helpers @@ -156,20 +186,40 @@ def jsr_with_carry(transport, addr, timeout=120.0, poll_interval=0.5): return result[0] -def check_label(labels, name): - """Return True if label exists, print skip message if not.""" - if labels.address(name) is None: - print(f" SKIP: label '{name}' not found (routine not yet implemented)") - return False - return True +# Groups that could not run this session because labels were missing. +# Entries are (group_name, [missing label names]). run_tests() resets this. +SKIPPED_GROUPS = [] + +def missing_labels(labels, label_list): + """Return the subset of label_list that the build does not export.""" + return [name for name in label_list if labels.address(name) is None] -def check_labels(labels, label_list): - """Return True if all labels in the list exist.""" - for name in label_list: - if labels.address(name) is None: - print(f" SKIP: label '{name}' not found -- skipping test group") - return False + +def preflight_required_labels(labels): + """Return [(group_name, [missing labels])] for declared groups that can't run.""" + broken = [] + for group_name, label_list in REQUIRED_LABEL_SETS: + missing = missing_labels(labels, label_list) + if missing: + broken.append((group_name, missing)) + return broken + + +def check_labels(labels, label_list, group="unnamed group"): + """Return True if all labels exist; otherwise record the group as skipped. + + A missing label is NOT a benign skip. It means the build no longer exports + a symbol this suite depends on, so the group's assertions never execute. + Recording the group here is what keeps it out of the "everything passed" + denominator — run_tests() turns each recorded entry into a failure. + """ + missing = missing_labels(labels, label_list) + if missing: + print(f" SKIP: {group}: label(s) {', '.join(missing)} not found " + f"-- group CANNOT RUN (counted as a failure)") + SKIPPED_GROUPS.append((group, missing)) + return False return True @@ -269,7 +319,7 @@ def test_der_parser_p256(transport, labels): passed = 0 failed = 0 - if not check_labels(labels, DER_LABELS): + if not check_labels(labels, DER_LABELS, "Group 1: DER Parser P-256"): return 0, 0 print("\n Generating P-256 self-signed certificate...") @@ -396,7 +446,7 @@ def test_der_parser_p384(transport, labels): passed = 0 failed = 0 - if not check_labels(labels, DER_LABELS): + if not check_labels(labels, DER_LABELS, "Group 2: DER Parser P-384"): return 0, 0 print("\n Generating P-384 self-signed certificate...") @@ -408,16 +458,22 @@ def test_der_parser_p384(transport, labels): # Load certificate to C64 load_cert_to_c64(transport, labels, cert_der) - # Parse the certificate + # Parse the certificate. + # A parse failure here is a FAILURE, not a skip. This used to report + # "SKIP: may not be supported yet" and return (0, 0), which would have + # hidden a P-384 DER regression completely — same shape as F3. P-384 + # certificate *parsing* works today (group 2 passes 2/2); it is only + # P-384 ECDSA *verify* that is stubbed at the TLS layer. try: carry = jsr_with_carry(transport, labels["x509_parse_cert"], timeout=120.0, poll_interval=0.5) if carry != 0: - print(" SKIP: P-384 parse returned C=1 (may not be supported yet)") - return 0, 0 + print(" [2!] FAIL: x509_parse_cert returned C=1 on a P-384 " + "certificate") + return 0, 1 except Exception as e: - print(f" SKIP: P-384 parse raised {e}") - return 0, 0 + print(f" [2!] FAIL: x509_parse_cert raised {e}") + return 0, 1 # --- Test 1: Curve ID correct --- print("\n [2a] DER parse P-384: curve_id = 1 (P-384)") @@ -500,7 +556,7 @@ def test_ecdsa_verify_p256(transport, labels): passed = 0 failed = 0 - if not check_labels(labels, ECDSA_LABELS): + if not check_labels(labels, ECDSA_LABELS, "Group 3: ECDSA P-256 Verify"): return 0, 0 print("\n Using hardcoded P-256 test vector (pre-verified in Python)") @@ -633,9 +689,16 @@ def run_tests(transport, labels): Order: DER parser first (fast, validates VICE), then ECDSA verify. CertificateVerify tests skipped until core verify is proven. + + A declared group that cannot run because its labels are missing is counted + as ONE FAILURE, and named in SKIPPED_GROUPS. That is deliberate: this + function is called directly by tools/run_all_tests.py, which only sees + (passed, failed), so a group vanishing from the denominator would otherwise + be indistinguishable from a clean run there too. """ total_passed = 0 total_failed = 0 + SKIPPED_GROUPS.clear() # --- DER parser tests (fast, ~seconds) --- test_groups = [ @@ -663,7 +726,7 @@ def run_tests(transport, labels): traceback.print_exc() # --- ECDSA verify tests (slow, minutes each) --- - ecdsa_ok = check_labels(labels, ECDSA_LABELS) + ecdsa_ok = check_labels(labels, ECDSA_LABELS, "Group 3: ECDSA P-256 Verify") if ecdsa_ok: # One-time sqtab_init before any ECDSA tests print(f"\n{'='*60}") @@ -696,6 +759,19 @@ def run_tests(transport, labels): # CertificateVerify tests skipped for now # (re-enable after core ECDSA verify is proven) + # A group that could not run is a failure, not a hole in the denominator. + if SKIPPED_GROUPS: + print(f"\n{'='*60}") + print(" GROUPS THAT COULD NOT RUN (counted as failures)") + print(f"{'='*60}") + for group, missing in SKIPPED_GROUPS: + print(f" [-] {group}: missing label(s) {', '.join(missing)}") + print(" A missing label means the build stopped exporting a symbol " + "this suite\n depends on (e.g. a libs/nistcurves bump renamed " + "an export). The group's\n assertions never executed, so this " + "run proves nothing about it.") + total_failed += len(SKIPPED_GROUPS) + return total_passed, total_failed @@ -745,17 +821,36 @@ def main(): labels = Labels.from_file(LABELS_PATH) # Check which test groups can run - der_ok = all(labels.address(n) is not None for n in DER_LABELS) - ecdsa_ok = all(labels.address(n) is not None for n in ECDSA_LABELS) - cv_ok = all(labels.address(n) is not None for n in CV_LABELS) + der_ok = not missing_labels(labels, DER_LABELS) + ecdsa_ok = not missing_labels(labels, ECDSA_LABELS) + cv_ok = not missing_labels(labels, CV_LABELS) print(f" Labels loaded from {LABELS_PATH}") print(f" DER parser labels: {'OK' if der_ok else 'MISSING'}") print(f" ECDSA verify labels: {'OK' if ecdsa_ok else 'MISSING'}") - print(f" CertificateVerify labels: {'OK' if cv_ok else 'MISSING'}") - - if not (der_ok or ecdsa_ok or cv_ok): - print("\nFATAL: No test group has all required labels. Nothing to test.") + print(f" CertificateVerify labels: " + f"{'OK' if cv_ok else 'MISSING'} (informational; no group uses these yet)") + + # Fail closed: every group in REQUIRED_LABEL_SETS is one this suite claims + # to run. If the build no longer exports the symbols it needs, the group's + # assertions cannot execute — and a suite that cannot execute its + # assertions has not passed. Abort here, before spending a VICE session + # producing a green result that covers less than it claims. + broken = preflight_required_labels(labels) + if broken: + names = "; ".join(f"{g} [missing: {', '.join(m)}]" for g, m in broken) + print(f"\n{'='*60}") + print("RESULTS") + print(f"{'='*60}") + print(f" Passed: 0/0") + print(f" Failed: 0/0") + print(f"\n [-] X.509/ECDSA: ABORTED -- declared test group(s) " + f"cannot run: {names}") + print(" A missing label means the build stopped exporting a symbol") + print(" this suite depends on (e.g. a libs/nistcurves bump renamed") + print(" an export). Skipping the group would report success while") + print(" testing nothing, so this is a failure.") + print(f"{'='*60}") sys.exit(1) # Estimate test duration @@ -793,16 +888,24 @@ def main(): mgr.release(inst) # Summary + no_tests_ran = (passed + failed) == 0 + if no_tests_ran: + # A suite that executed no assertions has not passed. + failed = 1 total = passed + failed print(f"\n{'='*60}") print("RESULTS") print(f"{'='*60}") print(f" Passed: {passed}/{total}") print(f" Failed: {failed}/{total}") - if total == 0: - print("\n [?] No tests ran (routines not yet implemented?)") + skipped = "; ".join(f"{g} [missing: {', '.join(m)}]" for g, m in SKIPPED_GROUPS) + if no_tests_ran: + print("\n [-] X.509/ECDSA: NO TESTS RAN -- nothing was verified") elif failed == 0: print(f"\n [+] X.509/ECDSA: ALL {total} TESTS PASSED") + elif skipped: + print(f"\n [-] X.509/ECDSA: {failed} TEST(S) FAILED " + f"-- group(s) could not run: {skipped}") else: print(f"\n [-] X.509/ECDSA: {failed} TEST(S) FAILED") print(f"{'='*60}")