diff --git a/.gitignore b/.gitignore index 873a02e..a4b6730 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ __pycache__/ *.pyc +.pytest_cache/ build/ dist/ ip65-build/*.o diff --git a/CLAUDE.md b/CLAUDE.md index 2c10445..1386c7b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -156,9 +156,9 @@ Test harness expectations: - Most `tools/test_*.py` scripts run `make clean && make` themselves before launching VICE. Set `C64_SKIP_BUILD=1` in the environment to reuse the already-built PRG. 14 scripts honor it as of 2026-08-13 - (13 under `tools/`, plus `tests/test_vice_https_macos.py`); the + (13 under `tools/`, plus `tests/rig_vice_https_macos.py`); the current list is `grep -ln 'environ.*C64_SKIP_BUILD' tools/test_*.py - tests/test_*.py` rather than a number that goes stale here. + tests/rig_*.py` rather than a number that goes stale here. - Use the `c64-test-harness` Python package to launch VICE; never run `x64sc` directly from tests. @@ -588,7 +588,7 @@ The TLS 1.3 handshake now completes end-to-end against the local test listener (ECDSA-P256 cert, `tools/https_e2e/certs/`) on **both** backends: UCI/U64E at 48 MHz turbo and stock 1 MHz, and ip65/VICE at stock 1 MHz no-WARP (after the 255-byte TCP RX clamp fix in `src/net/ip65/net.s`; -see `tests/test_phase3_https_1mhz.py`). The flow, identical across both +see `tests/rig_phase3_https_1mhz.py`). The flow, identical across both backends: **U64E wedge episode (2026-07-27/28, resolved — know the signature):** @@ -1652,6 +1652,35 @@ the TLS state machine. For a quick sanity check after a build: All 7 pass as of the ca65-conversion branch (97/97 assertions). +### `pytest` is not the runner — the collection boundary + +Do not reach for `pytest` to check this repo. Its suites are dispatched by +`tools/run_all_tests.py` as `run_tests(transport, labels, seed)`, so the +`test_*` functions take positional arguments, not fixtures; under pytest +they can only produce `fixture 'transport' not found`. Measured on master +before the fix, a bare `pytest` at the repo root gave **25 passed, 75 +errors**, and `tests/` contributed a silent zero because its five live-rig +`main()` scripts had no `def test_` at all (issue #109). + +The boundary is now pinned rather than accidental: + + - the rig scripts are `tests/rig_*.py`, outside pytest's discovery + namespace whatever the working directory (`tests/README.md` says how + to run them; they need `sudo` and a live rig) + - `pytest.ini` pins `testpaths` to the three genuinely pure-logic + modules and keeps collection out of `libs/`, `ip65/`, `tests/` and + `tools/uci/` (all of which are `test_*.py`-named and collect zero) + - root `conftest.py` prints what the run does and does not cover, in + both the header and the summary — no skips, because a vague skip + reads like coverage + - `tools/test_pytest_boundary.py` fails if either direction drifts + +Bare `pytest` at the repo root is now **30 passed** (exit 0), and +`pytest tests/` still exits 5, now with an explanation. `testpaths` only +applies at the rootdir, so `pytest` from a subdirectory collects that +subdirectory: from `tools/` it is 30 passed + 74 fixture errors, exit 1 — +loud and correct, since those modules cannot run under pytest at all. + ### Negative-path coverage — the server Finished `tools/test_finished_verify.py` and `tools/uci/test_https_bad_finished.py` @@ -1716,7 +1745,7 @@ correct for a submodule that was never `--init`'d. ### VICE ip65 rig (hardware-free e2e) -`tests/test_vice_https_macos.py` runs the **full HTTPS handshake + GET +`tests/rig_vice_https_macos.py` runs the **full HTTPS handshake + GET over the ip65 backend with no hardware at all** — emulated RR-Net (cs8900a) in VICE talking to a host-side TLS 1.3 listener. This is how the REU-less stock-C64 numbers above were measured. Knobs: diff --git a/README.md b/README.md index 3db2d76..23f27ed 100644 --- a/README.md +++ b/README.md @@ -270,7 +270,7 @@ Progress: - [x] HTTP/1.1 GET request — build GET, parse response (status + headers + body), plain HTTP end-to-end - [x] **End-to-end HTTPS GET demo (both backends)** — TLS 1.3 handshake + HTTP GET completes against a local Python TLS listener (ECDSA-P256 cert). Returns `http_status=200`, body `"HELLO FROM TLS SERVER"`. - UCI: real Ultimate 64 Elite hardware at both 48 MHz turbo and stock 1 MHz. See `tools/uci/test_https_local.py` (supports `TURBO_MHZ` env var). - - ip65: VICE + RR-Net at stock 1 MHz, no warp. The bridge-rig script is `tests/test_phase3_https_1mhz.py`; the hardware-free macOS feth/pcap rig is `tests/test_vice_https_macos.py`, and that is where the wall-clock below was taken. + - ip65: VICE + RR-Net at stock 1 MHz, no warp. The bridge-rig script is `tests/rig_phase3_https_1mhz.py`; the hardware-free macOS feth/pcap rig is `tests/rig_vice_https_macos.py`, and that is where the wall-clock below was taken. ### Known Issues @@ -321,6 +321,7 @@ python3 tools/test_chained_hmac.py # 10 cases: chained HMAC-SHA256 stability python3 tools/test_finished_verify.py # 18 cases: the server-Finished REJECTION path, driven over DMA python3 tools/test_ecdsa_kat_oracle.py # 6 vectors: ECDSA P-256 KAT, 3 valid + 3 negative CAVP python3 tools/test_package_verify.py # 31 cases: pure-logic tests for the release gate (no VICE, no build) +python3 tools/test_pytest_boundary.py # 4 checks: the pytest collection boundary below is intact # Benchmark python3 tools/bench_x25519.py # X25519 basepoint multiply: 12,635 jiffies / 211 s C64 time, ~14 s wall under warp @@ -330,17 +331,44 @@ python3 tools/test_dns.py # 4 tests: DNS resolution via ip65 over TA python3 tools/test_http_integration.py # 5 tests: end-to-end plain HTTP GET over TAP (DNS + TCP + request/response) # End-to-end bridge tests (require br-c64 bridge, RR-Net; see below) -sudo PYTHONPATH=tools python3 tests/test_phase1_dhcp.py # DHCP over RR-Net bridge -sudo PYTHONPATH=tools python3 tests/test_phase2_http.py # Plain HTTP GET over bridge +sudo PYTHONPATH=tools python3 tests/rig_phase1_dhcp.py # DHCP over RR-Net bridge +sudo PYTHONPATH=tools python3 tests/rig_phase2_http.py # Plain HTTP GET over bridge ``` +### `pytest` is not the runner here + +Almost nothing in this repo is a pytest test, and the file names hide +that. The suites above are dispatched by `tools/run_all_tests.py`, which +allocates a VICE instance per suite and calls +`run_tests(transport, labels, seed)`; their `test_*` functions take +positional arguments rather than fixtures, so pytest can only ever report +`fixture 'transport' not found`. The scripts in `tests/`, `tools/uci/` and +several under `tools/` are `main()` programs with no `def test_` at all, +so pytest collects zero from them and says nothing about it. + +`pytest.ini` therefore pins `testpaths` to the three modules that really +are pure-logic and pytest-runnable, and `conftest.py` prints the scope of +the run in both the header and the summary. A bare `pytest` at the repo +root reports **30 passed**, and says in the same breath that this is not a +statement about the C64 suites or the rig scripts. + +`testpaths` applies only when pytest is invoked from the rootdir, so from +a subdirectory you get that subdirectory instead — measured from `tools/`: +30 passed, 74 `fixture 'transport' not found` errors, exit 1. That is the +honest signal (pytest genuinely cannot run those modules) and it is loud, +which is the opposite of the problem being fixed here. + +`tools/test_pytest_boundary.py` fails if the boundary drifts in either +direction — a pure-logic module missing from `testpaths`, or a `test_*.py` +reappearing in `tests/`. See issue #109. + ### End-to-End Bridge Tests (ip65 backend) -Full end-to-end tests that drive the real c64-https binary in VICE over a Linux bridge with RR-Net ethernet (the same pattern used by [`c64-test-harness` bridge networking](https://github.com/JC-000/c64-test-harness/blob/master/docs/bridge_networking.md)). These exercise the **ip65/RR-Net path only**: DHCP (phase1), plain HTTP (phase2), and HTTPS (phase3 via `tests/test_phase3_https_1mhz.py`). VICE runs at **normal speed** (warp breaks RR-Net DHCP), so these tests need generous timeouts (~90-120s per phase). +Full end-to-end tests that drive the real c64-https binary in VICE over a Linux bridge with RR-Net ethernet (the same pattern used by [`c64-test-harness` bridge networking](https://github.com/JC-000/c64-test-harness/blob/master/docs/bridge_networking.md)). These exercise the **ip65/RR-Net path only**: DHCP (phase1), plain HTTP (phase2), and HTTPS (phase3 via `tests/rig_phase3_https_1mhz.py`). VICE runs at **normal speed** (warp breaks RR-Net DHCP), so these tests need generous timeouts (~90-120s per phase). The HTTPS phase is long. The nearest measured figure is from the hardware-free macOS rig rather than this Linux bridge: -`tests/test_vice_https_macos.py`, ip65 + onchip profile with no REU, +`tests/rig_vice_https_macos.py`, ip65 + onchip profile with no REU, honest 1 MHz, **2,159.7 s = 36.0 min** from `G` to `CONNECTION CLOSED`. Budget accordingly; do not assume the bridge rig matches it exactly. diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..98b0a83 --- /dev/null +++ b/conftest.py @@ -0,0 +1,53 @@ +"""Make the boundary of a pytest run impossible to misread. + +c64-https is an assembly project. Its real test suites drive VICE or real +Ultimate 64 hardware and are launched by `python3 tools/run_all_tests.py` +and by the manual rig scripts in `tests/` — not by pytest. Only a few +pure-logic host-side modules are pytest-runnable, and `pytest.ini` pins +`testpaths` to exactly those. + +Without this file, `pytest` at the repo root prints a bare pass count that +reads like whole-project coverage. It is not. See issue #109. + +This module deliberately contains no fixtures, no skips and no imports of +pytest: it must stay inert for anyone who does not have pytest installed, +and the repo declares no pytest dependency. +""" + +_BOUNDARY = [ + "c64-https: pytest runs ONLY the pure-logic host-side modules pinned in", + "pytest.ini `testpaths`. It does NOT run the C64 suites (those need VICE:", + "`python3 tools/run_all_tests.py`) and it does NOT run the live-rig", + "scripts in tests/ (manual, sudo + network rig: see tests/README.md).", + "A green run here says nothing about either.", +] + +_EMPTY_RUN = [ + "pytest collected nothing from the paths you gave it.", + "If that was `pytest tests/`: tests/ holds manual live-rig scripts", + "(tests/rig_*.py, main() programs needing sudo and a network rig), not", + "pytest tests. See tests/README.md for how to run them.", +] + + +def pytest_report_header(config): + """Printed in the header of every pytest invocation.""" + return _BOUNDARY + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + """Repeat the boundary immediately above the final pass/fail line. + + The header scrolls away on a long run; the summary line is the thing + people actually read, so the caveat has to sit next to it. + """ + write = terminalreporter.write_line + terminalreporter.write_sep("=", "scope of this run") + for line in _BOUNDARY: + write(line) + # pytest.ExitCode.NO_TESTS_COLLECTED == 5, spelled numerically so this + # file never has to import pytest. + if exitstatus == 5: + write("") + for line in _EMPTY_RUN: + write(line) diff --git a/docs/library-ingestion-architecture.md b/docs/library-ingestion-architecture.md index e146a7d..2405544 100644 --- a/docs/library-ingestion-architecture.md +++ b/docs/library-ingestion-architecture.md @@ -359,7 +359,7 @@ Suitable to file as GitHub issues against `JC-000/c64-https`. Each is sized for **Scope:** Same as W1 but for `cfg/c64-https-ip65.cfg`. Reclaim `NET_BSS_TAIL` ($4F8C-$5FFF, ~4.1 KB) as `CRYPTO_OVERLAY_IP65`. Confirm that with TLS_CODE + sha256 *back* in CRYPTO_HOT (instead of NET_BSS_TAIL), CRYPTO_HOT still fits in 16 KB. -**Expected output:** `make BACKEND=ip65` PRG builds clean, `make USE_X25519_SIBLING=1 BACKEND=ip65` now builds clean too (closes the long-standing ip65 sibling-fit issue), `test_phase3_https_1mhz.py` PASSes. +**Expected output:** `make BACKEND=ip65` PRG builds clean, `make USE_X25519_SIBLING=1 BACKEND=ip65` now builds clean too (closes the long-standing ip65 sibling-fit issue), `tests/rig_phase3_https_1mhz.py` PASSes. **Dependencies:** W1 (proves the hot/cold split works) and Issue A/B (smaller P-256 verify archive that fits in 4 KB overlay slot). diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..ea2f6c9 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,38 @@ +; pytest scope for c64-https. +; +; This repo's test suites are NOT pytest suites. The C64 suites under +; tools/ are driven by tools/run_all_tests.py, which allocates a VICE +; instance per suite and calls run_tests(transport, labels, seed); their +; `test_*` functions take positional arguments, not fixtures, so pytest +; can only ever report them as "fixture 'transport' not found". The live +; rig scripts in tests/ are main() programs needing sudo and a network +; rig (see tests/README.md). +; +; Only a handful of modules are pure-logic and genuinely runnable by +; pytest. testpaths names them explicitly, so that `pytest` with no +; arguments from the repo root is a fixed, honest set rather than +; whatever the tree happens to look like, and so that it never wanders +; into directories where every file named test_*.py contributes zero +; tests (tests/, tools/uci/, libs/). +; +; testpaths only applies when pytest runs from the rootdir. That is why +; the rig scripts had to be RENAMED to tests/rig_*.py as well: a rename +; holds from any working directory, config does not. +; +; The list is enforced, in both directions, by +; tools/test_pytest_boundary.py: add a pure-logic tools/test_*.py module +; and that guard fails until you list it here. +; +; A green run here does NOT mean the C64 test suites passed. conftest.py +; prints that on every invocation; do not remove it. + +[pytest] +testpaths = + tools/test_net_test_env.py + tools/test_package_verify.py + tools/test_pytest_boundary.py + +norecursedirs = .git libs ip65 ip65-build build dist tests tools/uci + +; A collection error must never be mistaken for a passing run. +addopts = -ra diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..791473b --- /dev/null +++ b/tests/README.md @@ -0,0 +1,59 @@ +# tests/ — manual live-rig scripts + +Everything in this directory is a **manual** script. Each is a `main()` +program behind `if __name__ == "__main__": sys.exit(main())`, each needs a +live network rig (and usually `sudo`), and each takes minutes to tens of +minutes to run. They are not part of any automated suite and nothing in +`make` or `tools/run_all_tests.py` invokes them. + +They are named `rig_*.py`, **not** `test_*.py`, and that is deliberate — +see "Why not pytest" below. + +| Script | Rig | What it proves | +|---|---|---| +| `rig_phase1_dhcp.py` | Linux `br-c64` bridge + RR-Net | ip65 acquires a DHCP lease | +| `rig_phase2_http.py` | Linux `br-c64` bridge + RR-Net | plain HTTP GET end to end | +| `rig_phase3_https.py` | Linux `br-c64` bridge + RR-Net | HTTPS handshake + GET | +| `rig_phase3_https_1mhz.py` | Linux `br-c64` bridge + RR-Net | the same at honest 1 MHz, no warp | +| `rig_vice_https_macos.py` | macOS feth pair + pcap, no hardware | HTTPS handshake + GET, hardware-free | + +Setup lives in `scripts/setup-bridge-tap.sh` (Linux) and +`tools/rig-up-macos.sh` (macOS); the macOS rig additionally needs a VICE +built with the pcap driver's `geteuid()==0` gate patched out. See the +"End-to-End Bridge Tests" and "VICE ip65 rig" sections of `README.md` and +`CLAUDE.md` for the full prerequisites. + +Run them directly: + +```sh +sudo PYTHONPATH=tools python3 tests/rig_phase1_dhcp.py +sudo PYTHONPATH=tools python3 tests/rig_phase2_http.py +sudo env VICE_HTTPS_OK_TO_RUN=1 PYTHONPATH=tools python3 tests/rig_phase3_https_1mhz.py +python3 tests/rig_vice_https_macos.py +``` + +Each prints `PASS` or `FAIL` and exits non-zero on failure. They verify +properly — `rig_vice_https_macos.py`, for instance, hard-fails on +`FAIL: DHCP not acquired after 3 attempts` and only prints `PASS` after a +completed TLS handshake and HTTP response. + +## Why not pytest + +These scripts cannot be pytest tests without inventing a rig fixture, and +a rig fixture that quietly skips is worse than no fixture at all: it turns +"nobody ran the network tests" into a green-looking run with a skip nobody +reads. + +Until they were renamed they were called `tests/test_*.py`, which is +exactly pytest's discovery convention — so `pytest` at the repo root +walked this directory, found no `def test_` functions, collected zero, and +said nothing about it. The pass count it printed came entirely from +`tools/`, and read like whole-project coverage. That was issue #109. + +The `rig_` prefix keeps them out of pytest's namespace no matter which +directory you invoke from — which is the part that had to be a rename +rather than config, since `testpaths` in `pytest.ini` only takes effect +when pytest is run from the repo root. `conftest.py` then prints what a +pytest run does and does not cover, in the header and again in the +summary. `tools/test_pytest_boundary.py` fails if a `test_*.py` file +reappears here. diff --git a/tests/test_phase1_dhcp.py b/tests/rig_phase1_dhcp.py similarity index 98% rename from tests/test_phase1_dhcp.py rename to tests/rig_phase1_dhcp.py index ad12dc3..8992b9f 100644 --- a/tests/test_phase1_dhcp.py +++ b/tests/rig_phase1_dhcp.py @@ -7,7 +7,7 @@ boot menu appears and that pressing 'I' produces the 'DHCP OK' banner. Run: - PYTHONPATH=tools python3 tests/test_phase1_dhcp.py + PYTHONPATH=tools python3 tests/rig_phase1_dhcp.py Exit codes: 0 -- PASS diff --git a/tests/test_phase2_http.py b/tests/rig_phase2_http.py similarity index 99% rename from tests/test_phase2_http.py rename to tests/rig_phase2_http.py index 6375b0b..765aae5 100644 --- a/tests/test_phase2_http.py +++ b/tests/rig_phase2_http.py @@ -7,7 +7,7 @@ a known response body. Run: - sudo PYTHONPATH=tools python3 tests/test_phase2_http.py + sudo PYTHONPATH=tools python3 tests/rig_phase2_http.py Exit codes: 0 -- PASS diff --git a/tests/test_phase3_https.py b/tests/rig_phase3_https.py similarity index 99% rename from tests/test_phase3_https.py rename to tests/rig_phase3_https.py index 7140d6e..e9097d2 100644 --- a/tests/test_phase3_https.py +++ b/tests/rig_phase3_https.py @@ -12,7 +12,7 @@ 6-8 minutes. Run: - sudo PYTHONPATH=tools python3 tests/test_phase3_https.py + sudo PYTHONPATH=tools python3 tests/rig_phase3_https.py Exit codes: 0 -- PASS diff --git a/tests/test_phase3_https_1mhz.py b/tests/rig_phase3_https_1mhz.py similarity index 99% rename from tests/test_phase3_https_1mhz.py rename to tests/rig_phase3_https_1mhz.py index a9e9b65..86beabf 100644 --- a/tests/test_phase3_https_1mhz.py +++ b/tests/rig_phase3_https_1mhz.py @@ -18,7 +18,7 @@ Gated by the ``VICE_HTTPS_OK_TO_RUN=1`` environment variable. Run (after the UCI test has fully stopped): - sudo env VICE_HTTPS_OK_TO_RUN=1 PYTHONPATH=tools python3 tests/test_phase3_https_1mhz.py + sudo env VICE_HTTPS_OK_TO_RUN=1 PYTHONPATH=tools python3 tests/rig_phase3_https_1mhz.py Exit codes: 0 -- PASS @@ -550,7 +550,7 @@ def main() -> int: " U64E test is likely still running. Wait for it to finish,\n" " then re-run with:\n" " sudo env VICE_HTTPS_OK_TO_RUN=1 PYTHONPATH=tools \\\n" - " python3 tests/test_phase3_https_1mhz.py" + " python3 tests/rig_phase3_https_1mhz.py" ) return 2 for port in (443,): diff --git a/tests/test_vice_https_macos.py b/tests/rig_vice_https_macos.py similarity index 98% rename from tests/test_vice_https_macos.py rename to tests/rig_vice_https_macos.py index 349686c..52e547f 100644 --- a/tests/test_vice_https_macos.py +++ b/tests/rig_vice_https_macos.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """macOS hardware-free HTTPS e2e: ip65 PRG in ethernet-VICE vs local TLS 1.3 listener. -The macOS counterpart to tests/test_phase3_https.py (which is Linux-only: +The macOS counterpart to tests/rig_phase3_https.py (which is Linux-only: sysfs TAP checks + sudo dnsmasq). This variant expects the feth/pcap rig from tools/rig-up-macos.sh to be up already (one sudo command per boot) and runs everything else unprivileged: @@ -92,7 +92,7 @@ def _rig_check() -> list[str]: """Return a list of missing-prerequisite messages (empty = rig OK).""" problems = [] if sys.platform != "darwin": - problems.append("not macOS (use tests/test_phase3_https.py on Linux)") + problems.append("not macOS (use tests/rig_phase3_https.py on Linux)") return problems if not os.path.exists(VICE_BIN): problems.append( @@ -340,7 +340,7 @@ def _sigterm(_sig, _frame): # Binary-monitor reads leave the CPU PAUSED — resume every # iteration or the emulation only runs between polls (seen # as 14 s of CPU in 38 min, "CH" forever). Mirrors - # wait_for_screen_text / tests/test_phase3_https.py. + # wait_for_screen_text / tests/rig_phase3_https.py. try: transport.resume() except Exception: # noqa: BLE001 diff --git a/tools/https_e2e/__init__.py b/tools/https_e2e/__init__.py index c8e3fbd..59f9a8c 100644 --- a/tools/https_e2e/__init__.py +++ b/tools/https_e2e/__init__.py @@ -1,6 +1,6 @@ """https_e2e -- End-to-end test helpers for the c64-https program. -Public API used by tests/test_phase1_dhcp.py and (later) higher phases: +Public API used by tests/rig_phase1_dhcp.py and (later) higher phases: from https_e2e import ( BridgeEnv, diff --git a/tools/https_e2e/certs/README b/tools/https_e2e/certs/README index c7a0dd8..1c0aadb 100644 --- a/tools/https_e2e/certs/README +++ b/tools/https_e2e/certs/README @@ -31,7 +31,7 @@ thereafter. Every in-tree consumer goes through packaged listener mint identical material: - `https_listener.py` (via `_ensure_certs(cert_profile)`), and so - everything built on it -- `tests/test_vice_https_macos.py`, + everything built on it -- `tests/rig_vice_https_macos.py`, `evil_listener.py`, `tools/uci/test_https_bad_finished.py` - `tools/uci/test_https_local.py`, which inlines its own listener (this one used to fail with `ERROR: cert/key not found` instead -- diff --git a/tools/test_package_verify.py b/tools/test_package_verify.py index d6becd4..6531552 100644 --- a/tools/test_package_verify.py +++ b/tools/test_package_verify.py @@ -187,7 +187,20 @@ def test_expected_images_tolerate_old_build_info() -> None: # build-info parsing # --------------------------------------------------------------------------- -def test_parse_build_info_records(tmp_lines: list[str]) -> None: +# Sample build-info lines. Defaulted rather than passed in, so that pytest +# can run this module too: a parameter without a default is a fixture +# request, and there is no `tmp_lines` fixture (issue #109). +SAMPLE_BUILD_INFO = [ + "variant=uci-onchip prg=x.prg args=BACKEND=uci USE_NISTCURVES_ONCHIP=1" + " result=OK bytes=62977 sha256=abc123 backend=uci", + "variant=ip65-onchip prg=y.prg args=BACKEND=ip65 result=FAILED" + " log=build-ip65-onchip.log", +] + + +def test_parse_build_info_records(tmp_lines: list[str] = None) -> None: + if tmp_lines is None: + tmp_lines = SAMPLE_BUILD_INFO print("\n-- build-info records parse, including args with spaces --") import tempfile with tempfile.TemporaryDirectory() as tmp: @@ -221,12 +234,7 @@ def main() -> int: test_expected_images_skip_failed_variants() test_expected_images_empty_when_nothing_built() test_expected_images_tolerate_old_build_info() - test_parse_build_info_records([ - "variant=uci-onchip prg=x.prg args=BACKEND=uci USE_NISTCURVES_ONCHIP=1" - " result=OK bytes=62977 sha256=abc123 backend=uci", - "variant=ip65-onchip prg=y.prg args=BACKEND=ip65 result=FAILED" - " log=build-ip65-onchip.log", - ]) + test_parse_build_info_records(SAMPLE_BUILD_INFO) print(f"\n{'=' * 60}") print(f"{PASSED} passed, {FAILED} failed") return 1 if FAILED else 0 diff --git a/tools/test_pytest_boundary.py b/tools/test_pytest_boundary.py new file mode 100644 index 0000000..51a72bd --- /dev/null +++ b/tools/test_pytest_boundary.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Guard the pytest collection boundary (issue #109). + +Nothing here touches VICE, hardware or a build; it is pure AST inspection +and runs in milliseconds. It exists because the failure it prevents is +silent by construction: a file named ``test_*.py`` that pytest collects +zero tests from disappears into a green pass count, and a pure-logic +module that nobody adds to ``testpaths`` never runs at all. + +Three invariants, checked in both directions: + +1. ``tests/`` contains no ``test_*.py``. Those are manual live-rig + scripts (``tests/rig_*.py``); named the pytest way they would be + walked, collected as zero, and reported as nothing. + +2. Every path in ``pytest.ini``'s ``testpaths`` exists. + +3. ``testpaths`` is exactly the set of ``tools/test_*.py`` modules pytest + can actually run — that is, modules with at least one module-level + ``test_*`` function where every such function's parameters all have + defaults. A parameter without a default is a fixture request, and this + repo defines no fixtures, so such a module can only ever error. + +Runs under pytest, and standalone for anyone without pytest installed +(the repo declares no pytest dependency):: + + python3 tools/test_pytest_boundary.py +""" + +import ast +import configparser +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +PYTEST_INI = REPO / "pytest.ini" + + +def _testpaths(): + """The `testpaths` entries from pytest.ini, as repo-relative strings.""" + parser = configparser.ConfigParser() + parser.read(PYTEST_INI) + raw = parser.get("pytest", "testpaths") + return [line.strip() for line in raw.splitlines() if line.strip()] + + +def _is_function(node): + return isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + + +def _module_level_test_functions(path_or_tree): + """Module-level `def test_*` nodes.""" + tree = path_or_tree + return [n for n in tree.body if _is_function(n) and n.name.startswith("test_")] + + +def _unittest_test_methods(tree): + """`test_*` methods of unittest.TestCase subclasses. + + pytest collects these natively and never fixture-injects their + arguments, so extra parameters (typically from `@mock.patch`) are not + a fixture request. + """ + found = [] + for node in tree.body: + if not isinstance(node, ast.ClassDef): + continue + bases = [] + for b in node.bases: + bases.append(b.attr if isinstance(b, ast.Attribute) else + getattr(b, "id", "")) + if not any(b.endswith("TestCase") for b in bases): + continue + found += [n for n in node.body + if _is_function(n) and n.name.startswith("test_")] + return found + + +def _requests_fixtures(fn): + """True if `fn` has any parameter pytest would try to fill as a fixture. + + pytest ignores parameters that carry defaults, so only the + non-defaulted positional/keyword-only ones count. + """ + args = fn.args + positional = args.posonlyargs + args.args + n_defaulted = len(args.defaults) + undefaulted = positional[:len(positional) - n_defaulted] if n_defaulted \ + else positional + kwonly = [a for a, d in zip(args.kwonlyargs, args.kw_defaults) if d is None] + return bool(undefaulted or kwonly) + + +def _pytest_runnable_tools_modules(): + """tools/test_*.py modules pytest could run cleanly, repo-relative.""" + runnable = [] + for path in sorted((REPO / "tools").glob("test_*.py")): + tree = ast.parse(path.read_text(), filename=str(path)) + fns = _module_level_test_functions(tree) + methods = _unittest_test_methods(tree) + if not fns and not methods: + continue # script-style; pytest sees nothing + # A bare parameter on a plain module-level function is a fixture + # request. Decorated ones (@mock.patch and friends) inject their + # own arguments, so they are not decidable from the AST and are + # left alone. + if any(not fn.decorator_list and _requests_fixtures(fn) for fn in fns): + continue # harness-driven; pytest can only error + runnable.append(str(path.relative_to(REPO))) + return runnable + + +def test_tests_dir_holds_no_pytest_named_files() -> None: + """tests/ must not look collectable, because it is not.""" + stray = sorted(p.name for p in (REPO / "tests").glob("test_*.py")) + assert stray == [], ( + f"tests/ contains pytest-named files {stray}, but everything in " + "tests/ is a manual live-rig script. pytest would walk them, collect " + "zero tests, and report nothing. Rename to rig_*.py — see " + "tests/README.md and issue #109." + ) + + +def test_every_testpath_exists() -> None: + """A stale testpaths entry silently shrinks the default run.""" + missing = [p for p in _testpaths() if not (REPO / p).exists()] + assert missing == [], ( + f"pytest.ini testpaths names paths that do not exist: {missing}. " + "pytest would skip them without comment, so the default `pytest` run " + "would quietly cover less than it claims." + ) + + +def test_testpaths_lists_every_runnable_tools_module() -> None: + """A new pure-logic suite must not be invisible to a bare `pytest`.""" + listed = {p for p in _testpaths() if p.startswith("tools/")} + runnable = set(_pytest_runnable_tools_modules()) + unlisted = sorted(runnable - listed) + assert unlisted == [], ( + f"these tools/ modules are pytest-runnable but absent from " + f"pytest.ini testpaths: {unlisted}. A bare `pytest` would never run " + "them. Add them to testpaths." + ) + + +def test_testpaths_lists_nothing_pytest_cannot_run() -> None: + """The inverse: a listed module must not error on missing fixtures.""" + listed = {p for p in _testpaths() if p.startswith("tools/")} + runnable = set(_pytest_runnable_tools_modules()) + broken = sorted(listed - runnable) + assert broken == [], ( + f"pytest.ini testpaths lists modules pytest cannot run cleanly: " + f"{broken}. Their test functions take positional arguments supplied " + "by tools/run_all_tests.py, not fixtures, so pytest reports " + "'fixture not found' errors. Remove them from testpaths." + ) + + +def main() -> int: + print("=== pytest collection boundary ===") + failed = 0 + for name, fn in sorted(globals().items()): + if not name.startswith("test_") or not callable(fn): + continue + try: + fn() + except AssertionError as exc: + failed += 1 + print(f" FAIL {name}\n {exc}") + else: + print(f" ok {name}") + print(f"\n{'FAILED' if failed else 'PASSED'}: {failed} failure(s)") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main())