From e9a3a8de426e6562c97d6e5145626bb1d2cabf72 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Mon, 23 Mar 2026 15:39:03 -0500 Subject: [PATCH 1/2] Switch all tests back to wait_for_text() now that harness PR #29 is merged The harness wait_for_text() now calls transport.resume() between polls internally, so the inline polling loops are no longer needed. This replaces 13 copies of the same ~10-line loop with single wait_for_text() calls, reducing total code by 120 lines. Co-Authored-By: Claude Opus 4.6 (1M context) --- tools/_diag_carry.py | 15 +++------------ tools/run_all_tests.py | 14 +++----------- tools/test_chained_hmac.py | 14 +++----------- tools/test_crypto.py | 16 +++------------- tools/test_entropy.py | 16 +++------------- tools/test_hkdf.py | 15 +++------------ tools/test_http.py | 18 ++++-------------- tools/test_keyschedule_steps.py | 16 +++------------- tools/test_net.py | 17 ++++------------- tools/test_sha256.py | 15 +++------------ tools/test_tls_handshake.py | 14 ++------------ tools/test_tls_record.py | 14 ++------------ tools/test_x509.py | 14 +++----------- 13 files changed, 39 insertions(+), 159 deletions(-) diff --git a/tools/_diag_carry.py b/tools/_diag_carry.py index a8a053f..81750b9 100644 --- a/tools/_diag_carry.py +++ b/tools/_diag_carry.py @@ -4,8 +4,8 @@ os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) from c64_test_harness import ( - Labels, ViceConfig, ViceInstanceManager, ScreenGrid, - read_bytes, write_bytes, goto, jsr, + Labels, ViceConfig, ViceInstanceManager, + read_bytes, write_bytes, goto, jsr, wait_for_text, ) import subprocess @@ -86,16 +86,7 @@ def jsr_with_carry_diag(transport, addr, timeout=60.0, poll_interval=0.5): t = inst.transport print(f"VICE PID={inst.pid}, port={inst.port}", flush=True) - # Binary monitor: resume CPU between screen polls - grid = None - deadline = time.monotonic() + 180.0 - while time.monotonic() < deadline: - g = ScreenGrid.from_transport(t) - if "Q=QUIT" in g.continuous_text().upper(): - grid = g - break - t.resume() - time.sleep(1.0) + grid = wait_for_text(t, "Q=QUIT", timeout=180.0, verbose=False) if grid is None: print("FATAL: menu not found") sys.exit(1) diff --git a/tools/run_all_tests.py b/tools/run_all_tests.py index a881549..6df9b48 100644 --- a/tools/run_all_tests.py +++ b/tools/run_all_tests.py @@ -13,8 +13,8 @@ os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) from c64_test_harness import ( - Labels, ViceConfig, ViceInstanceManager, ScreenGrid, - read_bytes, write_bytes, jsr, + Labels, ViceConfig, ViceInstanceManager, + read_bytes, write_bytes, jsr, wait_for_text, ) PRG_PATH = os.path.join("build", "c64-https.prg") @@ -119,15 +119,7 @@ def main(): # Wait for all instances to boot (binary monitor: resume CPU between polls) for i, inst in enumerate(instances): - grid = None - deadline = time.monotonic() + 120.0 - while time.monotonic() < deadline: - g = ScreenGrid.from_transport(inst.transport) - if "Q=QUIT" in g.continuous_text().upper(): - grid = g - break - inst.transport.resume() - time.sleep(1.0) + grid = wait_for_text(inst.transport, "Q=QUIT", timeout=120.0, verbose=False) if grid is None: print(f" Worker {i}: FATAL - menu did not appear") sys.exit(1) diff --git a/tools/test_chained_hmac.py b/tools/test_chained_hmac.py index c273d94..05394ac 100644 --- a/tools/test_chained_hmac.py +++ b/tools/test_chained_hmac.py @@ -19,13 +19,13 @@ Labels, ViceConfig, ViceInstanceManager, - ScreenGrid, read_bytes, write_bytes, set_breakpoint, delete_breakpoint, goto, wait_for_pc, + wait_for_text, ) # --------------------------------------------------------------------------- @@ -91,16 +91,8 @@ def main(): transport = inst.transport print(f" N={n}: VICE PID={inst.pid}, port={inst.port}") - # Wait for program menu (binary monitor: resume CPU between polls) - grid = None - deadline = time.time() + 60 - while time.time() < deadline: - g = ScreenGrid.from_transport(transport) - if "Q=QUIT" in g.continuous_text().upper(): - grid = g - break - transport.resume() - time.sleep(1.0) + # Wait for program menu + grid = wait_for_text(transport, "Q=QUIT", timeout=60, verbose=False) if grid is None: print(f" N={n}: FAIL - main menu did not appear") results.append((n, False, 0.0, True)) diff --git a/tools/test_crypto.py b/tools/test_crypto.py index 171eb6b..31cb65b 100644 --- a/tools/test_crypto.py +++ b/tools/test_crypto.py @@ -14,10 +14,9 @@ import struct import subprocess import sys -import time from c64_test_harness import ( - Labels, ViceConfig, ViceInstanceManager, ScreenGrid, - read_bytes, write_bytes, jsr, + Labels, ViceConfig, ViceInstanceManager, + read_bytes, write_bytes, jsr, wait_for_text, ) PROJECT_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") @@ -763,16 +762,7 @@ def main(): transport = inst.transport print(f"VICE PID={inst.pid}, port={inst.port}") - # Binary monitor: resume CPU between screen polls - grid = None - deadline = time.monotonic() + 60.0 - while time.monotonic() < deadline: - g = ScreenGrid.from_transport(transport) - if "Q=QUIT" in g.continuous_text().upper(): - grid = g - break - transport.resume() - time.sleep(1.0) + grid = wait_for_text(transport, "Q=QUIT", timeout=60.0, verbose=False) if grid is None: print("FATAL: Program menu did not appear") sys.exit(1) diff --git a/tools/test_entropy.py b/tools/test_entropy.py index 4c5c3be..ce063d4 100644 --- a/tools/test_entropy.py +++ b/tools/test_entropy.py @@ -14,13 +14,10 @@ import os import subprocess import sys -import time - from c64_test_harness import ( Labels, ViceConfig, ViceInstanceManager, - ScreenGrid, read_bytes, write_bytes, jsr, @@ -28,6 +25,7 @@ delete_breakpoint, goto, wait_for_pc, + wait_for_text, ) # --------------------------------------------------------------------------- @@ -388,17 +386,9 @@ def main(): transport = inst.transport print(f" VICE PID={inst.pid}, port={inst.port}") - # Wait for main menu (binary monitor: resume CPU between screen polls) + # Wait for main menu print(" Waiting for main menu...") - grid = None - deadline = time.monotonic() + 60.0 - while time.monotonic() < deadline: - g = ScreenGrid.from_transport(transport) - if "Q=QUIT" in g.continuous_text().upper(): - grid = g - break - transport.resume() - time.sleep(1.0) + grid = wait_for_text(transport, "Q=QUIT", timeout=60.0, verbose=False) if grid is None: print("FATAL: Main menu did not appear") sys.exit(1) diff --git a/tools/test_hkdf.py b/tools/test_hkdf.py index eb5d934..d363d8d 100644 --- a/tools/test_hkdf.py +++ b/tools/test_hkdf.py @@ -26,12 +26,11 @@ Labels, ViceConfig, ViceInstanceManager, - ScreenGrid, read_bytes, write_bytes, jsr, + wait_for_text, ) -import time # --------------------------------------------------------------------------- # Constants @@ -473,17 +472,9 @@ def main(): transport = inst.transport print(f" VICE PID={inst.pid}, port={inst.port}") - # Wait for main menu (binary monitor: resume CPU between polls) + # Wait for main menu print(" Waiting for main menu...") - grid = None - deadline = time.monotonic() + 60.0 - while time.monotonic() < deadline: - g = ScreenGrid.from_transport(transport) - if "Q=QUIT" in g.continuous_text().upper(): - grid = g - break - transport.resume() - time.sleep(1.0) + grid = wait_for_text(transport, "Q=QUIT", timeout=60.0, verbose=False) if grid is None: print("FATAL: Main menu did not appear") sys.exit(1) diff --git a/tools/test_http.py b/tools/test_http.py index 273a2c9..f81206b 100755 --- a/tools/test_http.py +++ b/tools/test_http.py @@ -13,11 +13,9 @@ import struct import subprocess import sys -import time - from c64_test_harness import ( - Labels, ViceConfig, ViceInstanceManager, ScreenGrid, - read_bytes, write_bytes, jsr, + Labels, ViceConfig, ViceInstanceManager, + read_bytes, write_bytes, jsr, wait_for_text, ) PROJECT_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") @@ -454,16 +452,8 @@ def main(): transport = inst.transport print(f" VICE PID={inst.pid}, port={inst.port}") - # Wait for menu to appear (binary monitor: resume CPU between polls) - grid = None - deadline = time.monotonic() + 60.0 - while time.monotonic() < deadline: - g = ScreenGrid.from_transport(transport) - if "Q=QUIT" in g.continuous_text().upper(): - grid = g - break - transport.resume() - time.sleep(1.0) + # Wait for menu to appear + grid = wait_for_text(transport, "Q=QUIT", timeout=60.0, verbose=False) if grid is None: print(" FATAL: Program menu did not appear") sys.exit(1) diff --git a/tools/test_keyschedule_steps.py b/tools/test_keyschedule_steps.py index 0fc1ad6..905d88b 100644 --- a/tools/test_keyschedule_steps.py +++ b/tools/test_keyschedule_steps.py @@ -18,16 +18,14 @@ import subprocess import sys -import time - from c64_test_harness import ( Labels, ViceConfig, ViceInstanceManager, - ScreenGrid, read_bytes, write_bytes, jsr, + wait_for_text, ) # --------------------------------------------------------------------------- @@ -395,17 +393,9 @@ def main(): transport = inst.transport print(f" VICE PID={inst.pid}, port={inst.port}") - # Wait for main menu (binary monitor: resume CPU between polls) + # Wait for main menu print(" Waiting for main menu...") - grid = None - deadline = time.monotonic() + 60.0 - while time.monotonic() < deadline: - g = ScreenGrid.from_transport(transport) - if "Q=QUIT" in g.continuous_text().upper(): - grid = g - break - transport.resume() - time.sleep(1.0) + grid = wait_for_text(transport, "Q=QUIT", timeout=60.0, verbose=False) if grid is None: print("FATAL: Main menu did not appear") sys.exit(1) diff --git a/tools/test_net.py b/tools/test_net.py index 458ff4e..00fcbec 100644 --- a/tools/test_net.py +++ b/tools/test_net.py @@ -13,10 +13,9 @@ import struct import subprocess import sys -import time from c64_test_harness import ( - Labels, ViceConfig, ViceInstanceManager, ScreenGrid, - read_bytes, write_bytes, jsr, + Labels, ViceConfig, ViceInstanceManager, + read_bytes, write_bytes, jsr, wait_for_text, ) PROJECT_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") @@ -481,16 +480,8 @@ def main(): transport = inst.transport print(f" VICE PID={inst.pid}, port={inst.port}") - # Wait for menu to appear (binary monitor: resume CPU between polls) - grid = None - deadline = time.monotonic() + 60.0 - while time.monotonic() < deadline: - g = ScreenGrid.from_transport(transport) - if "Q=QUIT" in g.continuous_text().upper(): - grid = g - break - transport.resume() - time.sleep(1.0) + # Wait for menu to appear + grid = wait_for_text(transport, "Q=QUIT", timeout=60.0, verbose=False) if grid is None: print(" FATAL: Program menu did not appear") sys.exit(1) diff --git a/tools/test_sha256.py b/tools/test_sha256.py index 3866bf3..836dfe6 100644 --- a/tools/test_sha256.py +++ b/tools/test_sha256.py @@ -17,15 +17,14 @@ import struct import subprocess import sys -import time from c64_test_harness import ( Labels, ViceConfig, ViceInstanceManager, - ScreenGrid, read_bytes, write_bytes, jsr, + wait_for_text, ) # --------------------------------------------------------------------------- @@ -325,17 +324,9 @@ def main(): transport = inst.transport print(f" VICE PID={inst.pid}, port={inst.port}") - # Wait for main menu (binary monitor: resume CPU between screen polls) + # Wait for main menu print(" Waiting for main menu...") - grid = None - deadline = time.monotonic() + 60.0 - while time.monotonic() < deadline: - g = ScreenGrid.from_transport(transport) - if "Q=QUIT" in g.continuous_text().upper(): - grid = g - break - transport.resume() - time.sleep(1.0) + grid = wait_for_text(transport, "Q=QUIT", timeout=60.0, verbose=False) if grid is None: print("FATAL: Main menu did not appear") sys.exit(1) diff --git a/tools/test_tls_handshake.py b/tools/test_tls_handshake.py index cd5df74..5c1b3ba 100644 --- a/tools/test_tls_handshake.py +++ b/tools/test_tls_handshake.py @@ -20,13 +20,10 @@ import subprocess import sys -import time as _time - from c64_test_harness import ( Labels, ViceConfig, ViceInstanceManager, - ScreenGrid, read_bytes, write_bytes, jsr, @@ -34,6 +31,7 @@ delete_breakpoint, goto, wait_for_pc, + wait_for_text, ) # --------------------------------------------------------------------------- @@ -1272,15 +1270,7 @@ def main(): # Wait for main menu (binary monitor: resume CPU between polls) print(" Waiting for main menu...") - grid = None - deadline = _time.monotonic() + 60.0 - while _time.monotonic() < deadline: - g = ScreenGrid.from_transport(transport) - if "Q=QUIT" in g.continuous_text().upper(): - grid = g - break - transport.resume() - _time.sleep(1.0) + grid = wait_for_text(transport, "Q=QUIT", timeout=60.0, verbose=False) if grid is None: print("FATAL: Main menu did not appear") sys.exit(1) diff --git a/tools/test_tls_record.py b/tools/test_tls_record.py index a87e0ef..df8b527 100644 --- a/tools/test_tls_record.py +++ b/tools/test_tls_record.py @@ -19,13 +19,10 @@ from cryptography.hazmat.primitives.ciphers.aead import ChaCha20Poly1305 -import time - from c64_test_harness import ( Labels, ViceConfig, ViceInstanceManager, - ScreenGrid, read_bytes, write_bytes, jsr, @@ -33,6 +30,7 @@ delete_breakpoint, goto, wait_for_pc, + wait_for_text, ) # --------------------------------------------------------------------------- @@ -777,15 +775,7 @@ def main(): # Wait for main menu (binary monitor: resume CPU between polls) print(" Waiting for main menu...") - grid = None - deadline = time.monotonic() + 60.0 - while time.monotonic() < deadline: - g = ScreenGrid.from_transport(transport) - if "Q=QUIT" in g.continuous_text().upper(): - grid = g - break - transport.resume() - time.sleep(1.0) + grid = wait_for_text(transport, "Q=QUIT", timeout=60.0, verbose=False) if grid is None: print("FATAL: Main menu did not appear") sys.exit(1) diff --git a/tools/test_x509.py b/tools/test_x509.py index 370bfeb..41a5518 100644 --- a/tools/test_x509.py +++ b/tools/test_x509.py @@ -30,11 +30,11 @@ Labels, ViceConfig, ViceInstanceManager, - ScreenGrid, read_bytes, write_bytes, jsr, goto, + wait_for_text, ) # --------------------------------------------------------------------------- @@ -771,17 +771,9 @@ def main(): print(f"\n=== Starting VICE ===") print(f" VICE PID={inst.pid}, port={inst.port}") - # Wait for main menu (binary monitor: resume CPU between polls) + # Wait for main menu print(" Waiting for main menu...") - grid = None - deadline = time.monotonic() + 60.0 - while time.monotonic() < deadline: - g = ScreenGrid.from_transport(transport) - if "Q=QUIT" in g.continuous_text().upper(): - grid = g - break - transport.resume() - time.sleep(1.0) + grid = wait_for_text(transport, "Q=QUIT", timeout=60.0, verbose=False) if grid is None: print("FATAL: Main menu did not appear") sys.exit(1) From 6cb9f54e04c1a349c1f2af1c2db081d2152ac5c2 Mon Sep 17 00:00:00 2001 From: JC-000 <3798556+JC-000@users.noreply.github.com> Date: Sat, 28 Mar 2026 20:11:35 -0500 Subject: [PATCH 2/2] Fix parallel test runner: all 10 suites, instance-per-suite, no worker reuse The parallel runner had two bugs: (1) as_completed() didn't see futures added mid-iteration, so only the first N suites were collected, and (2) reusing VICE instances across suites caused state contamination (HKDF 0/12 on reused workers). Fix: allocate a fresh VICE instance per suite via run_suite_in_own_instance(). Add all 10 suites (was 5). Add --skip-slow and --seed flags. 193/193 pass. Co-Authored-By: Claude Opus 4.6 (1M context) --- README.md | 11 ++- tools/run_all_tests.py | 178 ++++++++++++++++++++--------------------- 2 files changed, 92 insertions(+), 97 deletions(-) diff --git a/README.md b/README.md index 25bc431..58674f1 100644 --- a/README.md +++ b/README.md @@ -121,13 +121,15 @@ Current status (24.8 KB binary, 487 labels): ## Test Automation -193 tests across 9 suites + 2 diagnostic suites, using the [`c64-test-harness`](../c64-test-harness) package to drive VICE via its binary monitor protocol. All tests log VICE PID and port for multi-agent safety. +193 tests across 10 suites (+ 1 standalone diagnostic), using the [`c64-test-harness`](../c64-test-harness) package to drive VICE via its binary monitor protocol. The parallel runner allocates a fresh VICE instance per suite to avoid state contamination. All tests log VICE PID and port for multi-agent safety. ```bash pip install -e ../c64-test-harness -# Run all suites in parallel (5 VICE instances, ~2.5 min wall time) -python3 tools/run_all_tests.py --workers 5 +# Run all 10 suites in parallel (one VICE instance per suite, ~5 min with ECDSA) +python3 tools/run_all_tests.py +python3 tools/run_all_tests.py --skip-slow # Skip x509/ECDSA (~5s wall time) +python3 tools/run_all_tests.py --workers 6 # Limit concurrent VICE instances # Individual suites python3 tools/test_net.py # 60 tests: ip65 integration, ZP save/restore, ring buffer, TCP recv callback @@ -139,7 +141,8 @@ python3 tools/test_x509.py # 11 tests: DER parse P-256/P-384, ECDSA ver python3 tools/test_tls_handshake.py # 21 tests: transcript hash, ClientHello, ServerHello, key schedule (RFC 8448), Finished MAC python3 tools/test_keyschedule_steps.py # 9 tests: key schedule step-by-step (RFC 8448 vectors) python3 tools/test_entropy.py # 7 tests: SID/CIA hardware init, DRBG seeding, output quality -python3 tools/test_chained_hmac.py # 10 tests: chained HMAC-SHA256 stability (N=1..10) +python3 tools/test_http.py # 27 tests: HTTP/1.1 GET builder, response parser, status codes +python3 tools/test_chained_hmac.py # 10 tests: chained HMAC-SHA256 stability (N=1..10, standalone) ``` ## Related Projects diff --git a/tools/run_all_tests.py b/tools/run_all_tests.py index 6df9b48..783733c 100644 --- a/tools/run_all_tests.py +++ b/tools/run_all_tests.py @@ -2,13 +2,15 @@ """Run all c64-https test suites in parallel using ViceInstanceManager. Usage: - python3 tools/run_all_tests.py [--workers N] + python3 tools/run_all_tests.py [--workers N] [--seed S] [--skip-slow] """ import os +import random import subprocess import sys import time +from concurrent.futures import ThreadPoolExecutor, as_completed os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) @@ -35,48 +37,26 @@ def build(): return Labels.from_file(LABELS_PATH) -def run_test_suite(name, transport, labels, port, pid): +def run_test_suite(name, transport, labels, seed): """Run a single test suite, return (name, passed, failed, duration).""" + # Ensure CPU is running before each suite (previous suite leaves it paused + # after jsr() returns at a breakpoint) + transport.resume() start = time.time() passed = failed = 0 try: if name == "net": - from test_net import test_build_integrity, test_ip65_jump_table - from test_net import test_zp_save_restore, test_recv_ring_buffer - from test_net import test_ip65_init_without_hardware - - p, f = test_build_integrity(labels) - passed += p; failed += f - p, f = test_ip65_jump_table(transport) - passed += p; failed += f - p, f = test_zp_save_restore(transport, labels) - passed += p; failed += f - p, f = test_recv_ring_buffer(transport, labels) - passed += p; failed += f - p, f = test_ip65_init_without_hardware(transport, labels) - passed += p; failed += f + from test_net import run_tests as net_run + passed, failed = net_run(transport, labels) elif name == "sha256": from test_sha256 import run_tests as sha256_run passed, failed = sha256_run(transport, labels, iterations=5) elif name == "crypto": - from test_crypto import (test_sqtab_init, test_chacha20_block_rfc, - test_chacha20_encrypt_rfc, test_poly1305_mac_rfc, - test_aead_encrypt_rfc, test_aead_decrypt_roundtrip, - test_aead_random) - import random - rng = random.Random(42) - for fn in [test_sqtab_init, test_chacha20_block_rfc, - test_chacha20_encrypt_rfc, test_poly1305_mac_rfc, - test_aead_encrypt_rfc]: - p, f = fn(transport, labels) - passed += p; failed += f - p, f = test_aead_decrypt_roundtrip(transport, labels, rng) - passed += p; failed += f - p, f = test_aead_random(transport, labels, rng) - passed += p; failed += f + from test_crypto import run_tests as crypto_run + passed, failed = crypto_run(transport, labels, seed=seed) elif name == "hkdf": from test_hkdf import run_tests as hkdf_run @@ -84,10 +64,32 @@ def run_test_suite(name, transport, labels, port, pid): elif name == "tls_record": from test_tls_record import run_tests as record_run - passed, failed = record_run(transport, labels, seed=42) + passed, failed = record_run(transport, labels, seed=seed) + + elif name == "tls_handshake": + from test_tls_handshake import run_tests as handshake_run + passed, failed = handshake_run(transport, labels, seed=seed) + + elif name == "keyschedule": + from test_keyschedule_steps import run_tests as ks_run + passed, failed = ks_run(transport, labels) + + elif name == "entropy": + from test_entropy import run_tests as entropy_run + passed, failed = entropy_run(transport, labels) + + elif name == "http": + from test_http import run_tests as http_run + passed, failed = http_run(transport, labels) + + elif name == "x509": + from test_x509 import run_tests as x509_run + passed, failed = x509_run(transport, labels) except Exception as e: + import traceback print(f" [{name}] EXCEPTION: {e}") + traceback.print_exc() failed += 1 duration = time.time() - start @@ -95,64 +97,66 @@ def run_test_suite(name, transport, labels, port, pid): def main(): - workers = 3 - for i, arg in enumerate(sys.argv[1:]): - if arg == "--workers": - workers = int(sys.argv[i + 2]) + workers = 4 + seed = random.randint(0, 2**32 - 1) + skip_slow = False + + args = sys.argv[1:] + i = 0 + while i < len(args): + if args[i] == "--workers": + workers = int(args[i + 1]) + i += 2 + elif args[i] == "--seed": + seed = int(args[i + 1]) + i += 2 + elif args[i] == "--skip-slow": + skip_slow = True + i += 1 + else: + i += 1 + + print(f"Random seed: {seed} (reproduce with --seed {seed})") labels = build() - suites = ["net", "sha256", "crypto", "hkdf", "tls_record"] + # x509 is by far the slowest (~5 min for ECDSA verify), so start it first. + # Entropy uses manual breakpoints sensitive to CPU state, so start it early + # on a fresh worker. Remaining fast suites fill in around them. + suites = ["entropy", "net", "sha256", "crypto", "hkdf", + "keyschedule", "http", "tls_record", "tls_handshake"] + if not skip_slow: + suites.insert(0, "x509") config = ViceConfig(prg_path=PRG_PATH, warp=True, ntsc=True, sound=False) + num_instances = min(workers, len(suites)) - print(f"\n=== Starting {workers} VICE instances (staggered 100ms) ===") + print(f"\n=== Launching {len(suites)} suites across " + f"{num_instances} concurrent VICE instances ===") - with ViceInstanceManager(config=config) as mgr: - instances = [] - for i in range(min(workers, len(suites))): - inst = mgr.acquire() - print(f" Worker {i}: VICE PID={inst.pid}, port={inst.port}") - instances.append(inst) - if i < workers - 1: - time.sleep(0.1) # 100ms stagger per PATTERNS.md - - # Wait for all instances to boot (binary monitor: resume CPU between polls) - for i, inst in enumerate(instances): - grid = wait_for_text(inst.transport, "Q=QUIT", timeout=120.0, verbose=False) + def run_suite_in_own_instance(mgr, suite_name): + """Acquire a fresh VICE instance, run one suite, release.""" + inst = mgr.acquire() + try: + grid = wait_for_text(inst.transport, "Q=QUIT", timeout=120.0, + verbose=False) if grid is None: - print(f" Worker {i}: FATAL - menu did not appear") - sys.exit(1) + return suite_name, 0, 1, 0.0 # Safety loop: JMP $0339 prevents crash when BASIC ROM banked out write_bytes(inst.transport, 0x0339, bytes([0x4C, 0x39, 0x03])) - print(f" Worker {i}: ready") - - # Each suite gets its own worker — suites run in parallel - # If more suites than workers, extra suites wait for a free worker - from concurrent.futures import ThreadPoolExecutor, as_completed - - def worker_fn(suite_name, inst): - return run_test_suite(suite_name, inst.transport, labels, - inst.port, inst.pid) - - results = [] - print(f"\n=== Running {len(suites)} test suites across " - f"{len(instances)} workers ===\n") - - # Map suites to workers 1:1 (first batch), then reuse freed workers - with ThreadPoolExecutor(max_workers=len(instances)) as pool: - futures = {} - inst_queue = list(instances) - pending_suites = list(suites) - active = {} - - # Submit up to N suites (one per worker) - while pending_suites and inst_queue: - suite = pending_suites.pop(0) - inst = inst_queue.pop(0) - fut = pool.submit(worker_fn, suite, inst) - futures[fut] = suite - active[fut] = inst + + return run_test_suite(suite_name, inst.transport, labels, seed) + finally: + mgr.release(inst) + + results = [] + + with ViceInstanceManager(config=config) as mgr: + with ThreadPoolExecutor(max_workers=num_instances) as pool: + futures = { + pool.submit(run_suite_in_own_instance, mgr, suite): suite + for suite in suites + } for fut in as_completed(futures): name, passed, failed, duration = fut.result() @@ -161,18 +165,6 @@ def worker_fn(suite_name, inst): print(f" [{status}] {name}: {passed}/{passed+failed} " f"({duration:.1f}s)") - # Return this worker's instance and submit next suite - freed_inst = active.pop(fut) - if pending_suites: - suite = pending_suites.pop(0) - new_fut = pool.submit(worker_fn, suite, freed_inst) - futures[new_fut] = suite - active[new_fut] = freed_inst - - # Release instances - for inst in instances: - mgr.release(inst) - # Summary total_passed = sum(r[1] for r in results) total_failed = sum(r[2] for r in results) @@ -183,7 +175,7 @@ def worker_fn(suite_name, inst): f"{total_failed} failed") for name, passed, failed, duration in sorted(results): status = "OK" if failed == 0 else "FAIL" - print(f" {status:4s} {name:15s} {passed:3d}/{passed+failed:3d} " + print(f" {status:4s} {name:20s} {passed:3d}/{passed+failed:3d} " f"({duration:.1f}s)") print(f"{'='*60}")