diff --git a/CLAUDE.md b/CLAUDE.md index 4e7aa91..0516591 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -132,11 +132,23 @@ into `uci_host_buf` (256 bytes in UCI_BSS); `net_tcp_connect` passes it to firmware. Dotted-quad IP literals work because firmware passes them through. -### Firmware quirk — per-byte NEXT_DATA ACK +### Firmware quirk — FPGA register timing (delay-loop fence) -Per-byte `NEXT_DATA` ACK truncates multi-byte responses on the current -U64E firmware revision. The read path uses a tight-poll pattern instead -(read `$DF1E` until `DATA_AV` clears). Documented in `uci_cmd.s`. +The U64E's UCI FPGA needs **~38 us** between consecutive register +accesses regardless of CPU clock speed. At stock 1 MHz the bus cycle +time naturally satisfies this. At turbo speeds (4-48 MHz) the CPU +outruns the FPGA, causing double-latched writes and stale reads that +corrupt the UCI command protocol. + +**Fix:** A nested delay-loop macro `uci_fence` (defined in +`src/net/uci/uci_regs.inc`) is inserted after every read/write to UCI +registers `$DF1C-$DF1F`. Parameters: `UCI_FENCE_OUTER = 5`, +`UCI_FENCE_INNER = 100`, yielding ~2525 cycles (~52 us at 48 MHz, +35% safety margin). 14 bytes per fence site, 24 fence sites total +(11 write + 13 read). At 1 MHz the same loop costs ~2.5 ms per +access — negligible for networking. + +48 MHz turbo is fully supported and verified on real U64E hardware. ### Memory layout under UCI @@ -160,11 +172,30 @@ Scripts under `tools/uci/` require a U64E at 192.168.1.81 and use - `test_http_local.py` — HTTP GET against a local test server - `test_http_live.py` — HTTP GET against a real internet host (requires internet access from the U64E) + - `test_https_local.py` — HTTPS e2e scaffolding against a local TLS 1.3 + listener (ECDSA-P256 cert from + `tools/https_e2e/certs/`). DMAs a 6502 stub + that calls `http_get`, flips the U64E to 48 + MHz turbo, and captures full diagnostics on + pass or timeout. `DEBUG_CAPTURE=1` enables a + bounded 6510 bus stream for post-mortem. + Each run writes a timestamped artifact dir + under `$UCI_DEBUG_DIR` (default + `/tmp/uci_https_debug//`) containing: + packed raw trace (`trace.bin` + meta sidecar), + derived `summary.txt` / `tail.txt` / + `uci_accesses.txt`, the full 4 KB ring + (`ring.bin` + `ring_meta.json`), a DMA-read + TLS state snapshot (`tls_state_dump.json`), + the listener's `server_result.json`, and + `run_info.txt`. Rotation keeps the last 5 + dirs; `UCI_DEBUG_KEEP_ON_PASS=1` preserves + PASS runs. Currently still fails inside + `tls_handle_certificate` (X.509 parsing) — + see Known issues below. ### Known issues - - Ring buffer needs explicit zeroing before `http_get_plain` calls - (stale data from auto-init polling). - `http_status` parsing is garbled on large responses because the poll-timeout counter in `http.s` expires before all headers are consumed under UCI's slower `net_poll` round-trip. Body arrives @@ -174,6 +205,35 @@ Scripts under `tools/uci/` require a U64E at 192.168.1.81 and use - Boot banner line 03 still says "rr-net" under ip65 build even though Phase 2 made it backend-aware — this is correct/expected behavior. Under UCI it says "ULTIMATE 64 ELITE (UCI)". + - The delay-loop fence adds ~2.5 ms overhead per UCI register access + at 1 MHz (negligible for networking, but visible in tight loops). + - TLS 1.3 handshake currently stalls inside `tls_handle_certificate` + (`src/tls_cert.s`) during X.509 parsing. Two upstream bugs that + used to mask this were fixed in the current branch: (a) a `net_poll` + entry gate that spun forever on post-drain residual STATE bits + (fixed by swapping `uci_wait_idle` → `uci_wait_not_busy` at the + `net_poll` preamble only — other call sites remain on wait_idle), + and (b) `tls_transcript_hash` was defined in `src/tls_transcript.s` + but never called, so handshake + application key derivation fed + 32 zero bytes into HKDF-Expand-Label as the transcript context. + After the fix, handshake AEAD decryption succeeds, EncryptedExt + processes, and the 352 B Certificate record decrypts into + `tls_hs_buf` — stall moved forward from ENCRYPTED_EXT (0x03) to + CERTIFICATE (0x04). DHCP and plain HTTP are unaffected at all + speeds. + +### Design note — bounded timeouts must use wall-clock time + +Any future robustness work on the UCI adapter's spin-wait helpers +(`uci_wait_idle`, `uci_push_wait`, etc.) MUST use a wall-clock time +source — CIA timer on stock C64, TOD clock on U64E — rather than a +cycle-counted iteration budget. The fences around every UCI register +access make per-iteration cost scale with CPU speed: a budget that is +ample at 1 MHz collapses to far too short at 48 MHz because turbo +scales CPU cycles but not the FPGA's wire-level operation durations. +A prior attempt on branch `feat/net-drain-abi` split waits into +fast/long tiers with cycle-count budgets and broke DHCP at turbo for +exactly this reason; the branch was abandoned. ## Memory layout diff --git a/src/net/uci/net.s b/src/net/uci/net.s index 360d82a..e7cb1ec 100644 --- a/src/net/uci/net.s +++ b/src/net/uci/net.s @@ -54,6 +54,7 @@ ; --- primitives from uci_cmd.s --- .import uci_abort .import uci_wait_idle +.import uci_wait_not_busy .import uci_begin_cmd .import uci_put_byte .import uci_push_wait @@ -91,6 +92,7 @@ net_init: jsr uci_abort lda UCI_ID + uci_fence ; settle before comparing ID cmp #UCI_ID_VALUE beq @present @@ -136,7 +138,7 @@ net_poll: beq @do_poll rts @do_poll: - jsr uci_wait_idle + jsr uci_wait_not_busy lda #UCI_TARGET_NETWORK jsr uci_begin_cmd @@ -175,16 +177,23 @@ net_poll: ; SMC dst. Loop style matches uci_read_resp_bytes — tight-poll ; DATA_AV and read UCI_RESP_DATA; the firmware FIFO auto-advances ; on read (Phase 2 finding), so NO per-byte NEXT_DATA. + uci_fence ; give firmware time to stage response ldy #$00 @hdr_loop: lda UCI_STATUS + uci_fence ; settle before testing DATA_AV and #UCI_STAT_DATA_AV - beq @hdr_done_short + bne @hdr_got ; branch past trampoline + jmp @hdr_done_short ; long branch: fence too wide for BEQ +@hdr_got: lda UCI_RESP_DATA + uci_fence ; settle before storing header byte sta uci_read_hdr,y iny cpy #2 - bcc @hdr_loop + bcs @hdr_got2 ; branch past trampoline (inverted BCC) + jmp @hdr_loop ; long branch back: fence too wide for BCC +@hdr_got2: jmp @hdr_done @hdr_done_short: @@ -234,13 +243,15 @@ net_poll: bne @not_full lda uci_next_hi cmp tcp_recv_head+1 - beq @done_data ; ring full — drop the rest + bne @not_full + jmp @done_data ; ring full — drop the rest @not_full: ; Wait for DATA_AV — the firmware streams data in bursts; if the ; FIFO drained mid-record we bail (shouldn't happen if firmware ; honored actual_len but we defend anyway). lda UCI_STATUS + uci_fence ; settle before testing DATA_AV and #UCI_STAT_DATA_AV bne @have_byte jmp @done_data @@ -256,6 +267,7 @@ net_poll: sta @rb_store+2 lda UCI_RESP_DATA + uci_fence ; settle before storing data byte @rb_store: sta $ffff ; SMC: patched each byte @@ -406,13 +418,18 @@ net_tcp_connect: ldy #$00 @host_loop: lda uci_host_buf,y - beq @host_done + bne @host_push ; branch past trampoline + jmp @host_done ; long branch: fence too wide for BEQ +@host_push: sta UCI_CMD_DATA + uci_fence ; heavy fence: hostname bytes at 48 MHz iny - bne @host_loop ; bounded by 256 B (and by null before that) + beq @host_done ; Y wrapped to 0 — stop (bounded by 256 B) + jmp @host_loop ; long branch back: fence too wide for BNE @host_done: lda #$00 sta UCI_CMD_DATA ; explicit null terminator + uci_fence jsr uci_push_wait @@ -531,6 +548,7 @@ net_tcp_send: @sb_load: lda $ffff,y ; SMC: source base patched above sta UCI_CMD_DATA + uci_fence ; heavy fence: FIFO overruns at 48 MHz with standard fence iny bne @sb_nohi inc @sb_load+2 ; advance base high byte @@ -820,7 +838,7 @@ net_recv_byte: .segment "RODATA" net_banner_str: - .byte "ULTIMATE 64 ELITE (UCI)" + .byte "UCI NETWORKING" .byte $0d, 0 ; ============================================================================= diff --git a/src/net/uci/uci_cmd.s b/src/net/uci/uci_cmd.s index eb337b3..6ff81eb 100644 --- a/src/net/uci/uci_cmd.s +++ b/src/net/uci/uci_cmd.s @@ -53,6 +53,7 @@ uci_abort: lda #UCI_CTRL_ABORT sta UCI_CONTROL + uci_fence ldx #$20 @spin: dex @@ -67,8 +68,11 @@ uci_abort: ; ============================================================================= uci_wait_idle: lda UCI_STATUS + uci_fence ; settle read before testing bits and #(UCI_STAT_STATE | UCI_STAT_CMD_BUSY) ; $31 - bne uci_wait_idle + beq @idle_done + jmp uci_wait_idle ; long branch: fence too wide for BNE +@idle_done: rts ; ============================================================================= @@ -79,8 +83,11 @@ uci_wait_idle: ; ============================================================================= uci_wait_not_busy: lda UCI_STATUS + uci_fence ; settle read before testing bits and #UCI_STAT_CMD_BUSY - bne uci_wait_not_busy + beq @busy_done + jmp uci_wait_not_busy ; long branch: fence too wide for BNE +@busy_done: rts ; ============================================================================= @@ -91,6 +98,7 @@ uci_wait_not_busy: ; ============================================================================= uci_begin_cmd: sta UCI_CMD_DATA + uci_fence rts ; ============================================================================= @@ -100,15 +108,33 @@ uci_begin_cmd: ; ============================================================================= uci_put_byte: sta UCI_CMD_DATA + uci_fence rts ; ============================================================================= ; uci_push_wait — commit pushed bytes as a command, then wait for CMD_BUSY=0 -; Clobbers: A +; +; At turbo speeds the FPGA may not have latched PUSH_CMD by the time the +; CPU starts polling CMD_BUSY. A plain uci_fence after the write gives only +; ≈ 2 µs at 48 MHz — insufficient for the FPGA to assert CMD_BUSY. We add +; a short delay loop ($40 iterations ≈ 6 µs at 48 MHz, ≈ 300 µs at 1 MHz) +; before polling, ensuring CMD_BUSY has been asserted by the time we check. +; +; Clobbers: A, X ; ============================================================================= uci_push_wait: lda #UCI_CTRL_PUSH_CMD sta UCI_CONTROL + uci_fence + ; Fixed settle delay — at turbo speeds the FPGA may not have + ; latched PUSH_CMD and asserted CMD_BUSY by the time the CPU + ; starts polling. $FF iterations × 5 cycles ≈ 27 µs at 48 MHz, + ; ≈ 1.3 ms at 1 MHz — sufficient for the FPGA to latch the + ; command without using inline NOP fences that bloat code size. + ldx #$FF +@pw_settle: + dex + bne @pw_settle jmp uci_wait_not_busy ; ============================================================================= @@ -118,16 +144,18 @@ uci_push_wait: ; ============================================================================= uci_check_err: lda UCI_STATUS + uci_fence ; settle before testing error bit and #UCI_STAT_ERROR - beq @no_err + bne @has_err + clc + rts +@has_err: ; clear the latched error lda #UCI_CTRL_CLR_ERR sta UCI_CONTROL + uci_fence sec rts -@no_err: - clc - rts ; ============================================================================= ; uci_ack — single NEXT_DATA pulse (advance response/status FIFO by one byte) @@ -136,6 +164,7 @@ uci_check_err: uci_ack: lda #UCI_CTRL_NEXT_DATA sta UCI_CONTROL + uci_fence rts ; ============================================================================= @@ -158,12 +187,10 @@ uci_ack: ; ============================================================================= uci_read_resp_bytes: ; Patch the dst pointer into the STA abs,Y instruction below. - ; The inner loop mirrors the SOCKET_READ read pattern in - ; c64-test-harness/scripts/test_uci_tcp_echo.py (lines ~350-362): - ; tight-poll DATA_AV and read $DF1E directly — the UCI response - ; FIFO auto-advances on read, so no per-byte NEXT_DATA is needed - ; inside the loop. NEXT_DATA acknowledgment happens once at the - ; end via uci_drain_resp / uci_ack. + ; At turbo speeds the firmware may not have staged response data + ; by the time the CPU reaches this point (e.g. TCP_CONNECT takes + ; a full network round-trip). Use a 16-bit spin-wait on DATA_AV + ; so we tolerate up to ~150 ms at 48 MHz without bailing early. lda uci_resp_dst sta @rd_store+1 lda uci_resp_dst+1 @@ -171,11 +198,36 @@ uci_read_resp_bytes: ldy #$00 @rd_loop: cpy uci_resp_max - bcs @rd_done + bcc @rd_not_max + jmp @rd_done +@rd_not_max: + ; 16-bit spin-wait for DATA_AV. ~65536 iterations; at 48 MHz + ; each iteration is ~110 cycles → total ≈ 150 ms, enough for + ; TCP handshakes over a LAN. X is preserved across the wait. + stx @rd_save_x + lda #$00 + sta @rd_ctr_hi + ldx #$00 +@rd_wait: lda UCI_STATUS + uci_fence ; settle before testing DATA_AV and #UCI_STAT_DATA_AV - beq @rd_done + bne @rd_have + dex + beq @rd_xzero + jmp @rd_wait ; long branch: fence too wide for BNE +@rd_xzero: + dec @rd_ctr_hi + beq @rd_timeout + jmp @rd_wait ; long branch: fence too wide for BNE +@rd_timeout: + ; Timeout: DATA_AV never appeared — bail with partial read. + ldx @rd_save_x + jmp @rd_done +@rd_have: + ldx @rd_save_x lda UCI_RESP_DATA + uci_fence ; settle before storing/looping @rd_store: sta $FFFF,y ; SMC: dst low/high patched above iny @@ -183,6 +235,8 @@ uci_read_resp_bytes: @rd_done: sty uci_resp_count rts +@rd_save_x: .byte 0 +@rd_ctr_hi: .byte 0 ; ============================================================================= ; uci_drain_resp — ACK remaining response bytes until DATA_AV is clear. @@ -193,14 +247,17 @@ uci_read_resp_bytes: ; ============================================================================= uci_drain_resp: lda UCI_STATUS + uci_fence ; settle before testing DATA_AV and #UCI_STAT_DATA_AV - beq @drn_done + bne @drn_have + rts +@drn_have: lda UCI_RESP_DATA + uci_fence ; settle before NEXT_DATA write lda #UCI_CTRL_NEXT_DATA sta UCI_CONTROL + uci_fence jmp uci_drain_resp -@drn_done: - rts ; ============================================================================= ; uci_drain_status — ACK remaining status string bytes until STAT_AV is clear. @@ -209,14 +266,17 @@ uci_drain_resp: ; ============================================================================= uci_drain_status: lda UCI_STATUS + uci_fence ; settle before testing STAT_AV and #UCI_STAT_STAT_AV - beq @dst_done + bne @dst_have + rts +@dst_have: lda UCI_STATUS_DATA + uci_fence ; settle before NEXT_DATA write lda #UCI_CTRL_NEXT_DATA sta UCI_CONTROL + uci_fence jmp uci_drain_status -@dst_done: - rts ; ============================================================================= ; Control block for uci_read_resp_bytes — lives in UCI_BSS so no ZP is needed diff --git a/src/net/uci/uci_regs.inc b/src/net/uci/uci_regs.inc index f813fda..b328ea7 100644 --- a/src/net/uci/uci_regs.inc +++ b/src/net/uci/uci_regs.inc @@ -46,6 +46,44 @@ UCI_CTRL_CLR_ERR = $08 ; clear error state ; ============================================================================= UCI_TARGET_NETWORK = $03 ; network stack +; ============================================================================= +; Delay-loop fence macro — insert after every STA or LDA that accesses a UCI +; register ($DF1C-$DF1F), giving the FPGA time to latch writes AND settle +; reads before the CPU acts on the value. +; +; A simple NOP sled can't provide enough wall-clock time at high CPU +; speeds without overflowing the code segment (256 NOPs is the max that +; fits, giving only ~10.7 µs at 48 MHz — insufficient for the ~38 µs +; the FPGA needs). Instead we use a nested delay loop: +; total cycles ≈ OUTER * (INNER * 5 + 5) +; +; Tuned empirically via binary search at 48 MHz: +; OUTER=3 INNER=121 (~1830 cycles, ~38 µs at 48 MHz) = FAIL +; OUTER=3 INNER=122 (~1845 cycles, ~38.4 µs at 48 MHz) = PASS (minimum) +; OUTER=5 INNER=100 (~2525 cycles, ~52 µs at 48 MHz) = chosen (35% margin) +; +; At 1 MHz the overhead is ~2.5 ms per access — acceptable for networking. +; The macro preserves A and X via the stack, costing ~14 bytes per call +; site (vs 256 for the NOP sled that still wasn't enough). +; ============================================================================= +UCI_FENCE_OUTER = 5 ; outer loop iterations +UCI_FENCE_INNER = 100 ; inner loop iterations + +.macro uci_fence + pha ; save A + txa + pha ; save X + ldx #UCI_FENCE_OUTER +: lda #UCI_FENCE_INNER +: sbc #1 + bne :- + dex + bne :-- + pla + tax ; restore X + pla ; restore A +.endmacro + ; ============================================================================= ; Command IDs (issued as the first command byte after selecting the target) ; Phase 2+ will use these; Phase 1b keeps them here purely as equates. diff --git a/src/tls13.s b/src/tls13.s index 838ce53..f1b312e 100644 --- a/src/tls13.s +++ b/src/tls13.s @@ -69,6 +69,7 @@ ; --- Transcript hash (tls_transcript.s) --- .import tls_transcript_init .import tls_transcript_update +.import tls_transcript_hash ; --- Key schedule (tls_keyschedule.s) --- .import tls_derive_handshake_keys @@ -158,6 +159,13 @@ tls_connect: ldy #>hk1_msg jsr print_string + ; finalize transcript hash = SHA-256(CH || SH) for "s hs traffic" + ; / "c hs traffic" context. tls_transcript_hash is non-destructive: + ; it snapshots the running state and restores it, so subsequent + ; tls_transcript_update calls (EE, Cert, CertVerify, ServerFinished, + ; client Finished) still feed the same streaming SHA-256. + jsr tls_transcript_hash + ; derive handshake keys from ECDHE shared secret jsr tls_derive_handshake_keys bcc @ok3 @@ -217,6 +225,12 @@ tls_connect: jmp @error @ok9: + ; finalize transcript hash = SHA-256(CH || .. || ServerFinished) + ; for application traffic key derivation. Non-destructive finalize + ; preserves the running state for the subsequent client Finished + ; update. + jsr tls_transcript_hash + ; derive application traffic keys jsr tls_derive_traffic_keys bcc @ok10 diff --git a/tools/uci/test_https_local.py b/tools/uci/test_https_local.py new file mode 100644 index 0000000..3d952c6 --- /dev/null +++ b/tools/uci/test_https_local.py @@ -0,0 +1,1256 @@ +#!/usr/bin/env python3 +""" +Phase 5 LOCAL HTTPS: exercise the real http_get (TLS 1.3) code path through +the UCI backend on a real Ultimate 64 Elite at 48 MHz turbo. + +Debug-stream capture: set DEBUG_CAPTURE=0 in env to disable. Default enabled. +Artifacts land in a per-run timestamped directory under $UCI_DEBUG_DIR +(default /tmp/uci_https_debug//). Each run dir holds: + summary.txt — stats + hot PCs + UCI-reg counts + tail.txt — last 2000 CPU cycles + uci_accesses.txt — every CPU cycle in $DF1B-$DF1F + trace.bin + .meta.json — packed raw BusCycle trace (4 bytes/cycle) + server_result.json — server-side listener state (request, error, ...) + run_info.txt — git HEAD, outcome, duration, exit code +The latest 5 run dirs are retained; older ones are pruned on startup. + +Flow: + 1. Boot the UCI-built PRG, wait for auto-init (net_init + DHCP). + 2. Start a Python HTTPS server on the dev host's LAN IP with the + self-signed ECDSA P-256 cert at tools/https_e2e/certs/server.pem. + 3. Quit to BASIC ('Q'), flip to 48 MHz turbo. + 4. DMA-inject a 6502 stub at $4200 that: + - Banks out BASIC ROM + - Sets http_host_ptr, http_host_len, http_path_ptr, http_path_len, http_port + - Calls http_get (the TLS+HTTP code from src/http.s) + - Writes a sentinel on completion + 5. Trigger with SYS 16896 via keyboard buffer. + 6. Poll sentinel for up to 120 s (handshake is ~13-15 s at 48 MHz). + 7. Assert response body contains "HELLO FROM TLS SERVER". +""" +from __future__ import annotations + +import base64 +import datetime +import json +import os +import shutil +import socket +import ssl +import struct +import subprocess +import sys +import threading +import time +import traceback +from pathlib import Path + +from c64_test_harness.backends.device_lock import DeviceLock +from c64_test_harness.backends.ultimate64 import Ultimate64Transport +from c64_test_harness.backends.ultimate64_client import Ultimate64Client +from c64_test_harness.backends.ultimate64_helpers import ( + set_turbo_mhz, + set_debug_stream_mode, + DEBUG_MODE_6510, +) +from c64_test_harness.backends.u64_debug_capture import ( + DebugCapture, + DEFAULT_DEBUG_PORT, +) +from c64_test_harness.uci_network import enable_uci, disable_uci +from c64_test_harness.keyboard import send_text + + +DEBUG_CAPTURE_ENABLED = os.environ.get("DEBUG_CAPTURE", "1") != "0" +UCI_DEBUG_BASE_DIR = Path( + os.environ.get("UCI_DEBUG_DIR", "/tmp/uci_https_debug") +) +UCI_DEBUG_KEEP = 5 +UCI_DEBUG_KEEP_ON_PASS = os.environ.get("KEEP_DEBUG_ON_PASS", "0") != "0" + + +def _keep_cycle(word: int) -> bool: + """DebugCapture filter: keep only CPU cycles in regions we care about. + + Keeps a cycle if PHI2=1 (CPU cycle, bit 31 set) AND the 16-bit address + falls in one of the three interesting ranges: + - $2000-$3FFF : UCI adapter code/data + - $6000-$9FFF : crypto + TLS code + - $DF1B-$DF1F : UCI I/O registers + """ + if not (word >> 31) & 1: + return False + addr = word & 0xFFFF + return ( + 0x2000 <= addr <= 0x3FFF + or 0x6000 <= addr <= 0x9FFF + or 0xDF1B <= addr <= 0xDF1F + ) + + +HOST = os.environ.get("U64_HOST", "192.168.1.81") +REPO_ROOT = Path(__file__).resolve().parents[2] +PRG_PATH = REPO_ROOT / "build" / "c64-https.prg" +LABELS_PATH = REPO_ROOT / "build" / "labels.txt" +CERT_PATH = REPO_ROOT / "tools" / "https_e2e" / "certs" / "server.pem" +KEY_PATH = REPO_ROOT / "tools" / "https_e2e" / "certs" / "server.key" + +ROUTINE_ADDR = 0x4200 +HOST_STR_ADDR = 0x4400 +PATH_STR_ADDR = 0x4440 +SENTINEL_ADDR = 0x4540 +PROGRESS_ADDR = 0x4541 +CARRY_FLAG_ADDR = 0x4542 + +SENTINEL_VALUE = 0xAA + +# Default HTTPS port; can override via HTTPS_PORT env, else fall back to 4433 +# if 443 bind fails (requires root on most systems). +DEFAULT_HTTPS_PORT = int(os.environ.get("HTTPS_PORT", "443")) +FALLBACK_HTTPS_PORT = 4433 + +SENTINEL_POLL_TIMEOUT = 120.0 # TLS handshake ~13-15s at 48 MHz; leave slack +ACCEPT_TIMEOUT = 180.0 # server-side accept + handshake slack +TURBO_MHZ = 48 + +EXPECTED_BODY = "HELLO FROM TLS SERVER" +HTTP_RESPONSE = ( + b"HTTP/1.0 200 OK\r\n" + b"Content-Length: 21\r\n" + b"\r\n" + b"HELLO FROM TLS SERVER" +) + + +def _detect_local_ip(target: str) -> str: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.connect((target, 80)) + return s.getsockname()[0] + finally: + s.close() + + +def _make_ssl_context() -> ssl.SSLContext: + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(certfile=str(CERT_PATH), keyfile=str(KEY_PATH)) + # Restrict to TLS 1.3 to match what the C64 client negotiates. + try: + ctx.minimum_version = ssl.TLSVersion.TLSv1_3 + ctx.maximum_version = ssl.TLSVersion.TLSv1_3 + except AttributeError: + pass + return ctx + + +def _try_bind(bind_ip: str, port: int) -> socket.socket | None: + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + srv.bind((bind_ip, port)) + except PermissionError: + srv.close() + return None + except OSError: + srv.close() + return None + return srv + + +def _run_https_server(srv: socket.socket, ctx: ssl.SSLContext, + result: dict) -> None: + srv.settimeout(ACCEPT_TIMEOUT) + try: + srv.listen(1) + result["listening"] = True + raw_conn, addr = srv.accept() + result["client_addr"] = addr + try: + tls_conn = ctx.wrap_socket(raw_conn, server_side=True) + except (ssl.SSLError, OSError) as exc: + result["error"] = f"TLS handshake failed: {type(exc).__name__}: {exc}" + try: + raw_conn.close() + except Exception: + pass + return + try: + tls_conn.settimeout(30.0) + try: + req = tls_conn.recv(1024) + result["request"] = req + except socket.timeout: + result["request"] = b"" + tls_conn.sendall(HTTP_RESPONSE) + # Keep alive briefly for the C64 to drain before FIN. + time.sleep(1.0) + finally: + try: + tls_conn.unwrap() + except Exception: + pass + try: + tls_conn.close() + except Exception: + pass + except Exception as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + finally: + try: + srv.close() + except Exception: + pass + + +def _load_labels() -> dict[str, int]: + labels: dict[str, int] = {} + for line in LABELS_PATH.read_text().splitlines(): + parts = line.split() + if len(parts) >= 3 and parts[0] == "al" and parts[2].startswith("."): + name = parts[2][1:] + _, hex_addr = parts[1].split(":", 1) + labels[name] = int(hex_addr, 16) + return labels + + +def _build_http_routine(labels: dict[str, int], port: int) -> tuple[bytes, int]: + """Emit a 6502 routine that calls http_get via the real TLS+HTTP layer.""" + code = bytearray() + + def emit(*bs: int) -> None: + code.extend(bs) + + def emit_lda_imm(v: int) -> None: + emit(0xA9, v & 0xFF) + + def emit_sta_abs(addr: int) -> None: + emit(0x8D, addr & 0xFF, (addr >> 8) & 0xFF) + + def emit_lda_abs(addr: int) -> None: + emit(0xAD, addr & 0xFF, (addr >> 8) & 0xFF) + + def emit_jsr(addr: int) -> None: + emit(0x20, addr & 0xFF, (addr >> 8) & 0xFF) + + def emit_jmp(addr: int) -> None: + emit(0x4C, addr & 0xFF, (addr >> 8) & 0xFF) + + def emit_progress(step: int) -> None: + emit_lda_imm(step) + emit_sta_abs(PROGRESS_ADDR) + + # ABI addresses + http_get = labels["http_get"] + http_host_ptr = labels["http_host_ptr"] + http_host_len = labels["http_host_len"] + http_path_ptr = labels["http_path_ptr"] + http_path_len = labels["http_path_len"] + http_port = labels["http_port"] + net_init = labels["net_init"] + tcp_recv_head = labels["tcp_recv_head"] + tcp_recv_tail = labels["tcp_recv_tail"] + + # 0) Bank BASIC ROM OUT so $A000-$BFFF is RAM + emit_lda_abs(0x0001) + emit(0x29, 0xFE) # AND #$FE + emit_sta_abs(0x0001) + + # Clear markers + emit_lda_imm(0x00) + emit_sta_abs(SENTINEL_ADDR) + emit_sta_abs(PROGRESS_ADDR) + emit_sta_abs(CARRY_FLAG_ADDR) + + emit_progress(0x01) + + # Re-init UCI (ensure idle state after auto-init) + emit_jsr(net_init) + + # Zero the TCP ring head/tail to flush any stale boot-poll data + emit_lda_imm(0x00) + emit_sta_abs(tcp_recv_head) + emit_sta_abs(tcp_recv_head + 1) + emit_sta_abs(tcp_recv_tail) + emit_sta_abs(tcp_recv_tail + 1) + + emit_progress(0x02) + + # Set http_host_ptr = HOST_STR_ADDR + emit_lda_imm(HOST_STR_ADDR & 0xFF) + emit_sta_abs(http_host_ptr) + emit_lda_imm((HOST_STR_ADDR >> 8) & 0xFF) + emit_sta_abs(http_host_ptr + 1) + + # Set http_host_len — patched by Python once the IP is known. + host_len_patch_offset = len(code) + 1 + emit_lda_imm(0x00) + emit_sta_abs(http_host_len) + + # Set http_path_ptr = PATH_STR_ADDR + emit_lda_imm(PATH_STR_ADDR & 0xFF) + emit_sta_abs(http_path_ptr) + emit_lda_imm((PATH_STR_ADDR >> 8) & 0xFF) + emit_sta_abs(http_path_ptr + 1) + + # Set http_path_len = 1 (just "/") + emit_lda_imm(1) + emit_sta_abs(http_path_len) + + # Set http_port = our test port (16-bit: low then high) + emit_lda_imm(port & 0xFF) + emit_sta_abs(http_port) + emit_lda_imm((port >> 8) & 0xFF) + emit_sta_abs(http_port + 1) + + emit_progress(0x03) + + # Call http_get — the REAL TLS 1.3 + HTTP code path. + emit_jsr(http_get) + + # Store carry (success/failure) via PHP/PLA + emit(0x08) # PHP + emit(0x68) # PLA + emit_sta_abs(CARRY_FLAG_ADDR) + + emit_progress(0x04) + + # Write sentinel + emit_lda_imm(SENTINEL_VALUE) + emit_sta_abs(SENTINEL_ADDR) + + emit_progress(0x05) + + # Park CPU + park = ROUTINE_ADDR + len(code) + emit_jmp(park) + + return bytes(code), host_len_patch_offset + + +def _decode_screen_ram(data: bytes) -> str: + lines = [] + for row in range(25): + line = data[row * 40:(row + 1) * 40] + chars = [] + for b in line: + if b == 0x20 or b == 0x00: + chars.append(' ') + elif 0x01 <= b <= 0x1A: + chars.append(chr(b + 0x40)) + elif 0x30 <= b <= 0x39: + chars.append(chr(b)) + elif 0x41 <= b <= 0x5A: + chars.append(chr(b)) + elif b in (0x2E, 0x2F, 0x3A, 0x2D, 0x28, 0x29): + chars.append(chr(b)) + else: + chars.append('.') + lines.append(''.join(chars).rstrip()) + return '\n'.join(lines) + + +def _dump_diag(transport: Ultimate64Transport, + labels: dict[str, int]) -> None: + def r8(name: str) -> int: + return transport.read_memory(labels[name], 1)[0] + + def r16(name: str) -> int: + b = transport.read_memory(labels[name], 2) + return b[0] | (b[1] << 8) + + print() + print("--- adapter / TLS state ---") + for name in ("net_last_error", "net_tcp_state", "uci_socket_id", + "net_initialized"): + if name in labels: + print(f" {name:22s} : 0x{r8(name):02X}") + for name in ("tls_state", "tls_last_state", + "tls_recv_progress", "tls_recv_sub_progress"): + if name in labels: + print(f" {name:22s} : 0x{r8(name):02X}") + for name in ("tcp_recv_head", "tcp_recv_tail", + "http_status", "http_resp_len"): + if name in labels: + print(f" {name:22s} : ${r16(name):04X}") + + +def _dump_tls_state_snapshot(transport: Ultimate64Transport, + labels: dict[str, int], + run_dir: Path) -> None: + """DMA-snapshot every TLS state-machine variable we know the label for. + + Written as ``tls_state_dump.json`` in ``run_dir`` to support post-mortem + decoding of stalls. Byte blobs come out as hex strings; word values are + unsigned little-endian ints; per-variable entries also carry the label + address for sanity-checking vs ``build/labels.txt``. + + Silently skips any label that isn't present in ``labels`` (older builds + may omit some TLS progress counters). + """ + # Layout: (label_name, byte_count). Anything not present is skipped. + byte_specs: list[tuple[str, int]] = [ + # --- top-level state / progress --- + ("tls_state", 1), + ("tls_last_state", 1), + ("tls_recv_progress", 1), + ("tls_recv_sub_progress", 1), + ("tls_recv_poll_count", 2), + # --- record-layer framer --- + ("tls_rec_header", 5), + ("tls_rec_type", 1), + ("tls_rec_len", 2), + ("tls_recv_state", 1), + ("tls_recv_count", 2), + # --- handshake msg buffer (256B total per data.s) --- + ("tls_hs_buf", 256), + ("tls_hs_len", 2), + # --- app-data plumbing --- + ("tls_app_ptr", 2), + ("tls_app_len", 2), + # --- seq counters / key schedule --- + ("tls_read_seq", 8), + ("tls_write_seq", 8), + ("tls_hs_read_key", 32), + ("tls_hs_read_iv", 12), + ("tls_hs_write_key", 32), + ("tls_hs_write_iv", 12), + ("tls_app_read_key", 32), + ("tls_app_read_iv", 12), + ("tls_app_write_key", 32), + ("tls_app_write_iv", 12), + # --- secrets (handshake/master) --- + ("tls_early_secret", 32), + ("tls_handshake_secret", 32), + ("tls_master_secret", 32), + # --- key-schedule intermediates (for post-mortem verification of + # x25519 / HKDF-Extract / HKDF-Expand-Label stages) --- + ("tls_ecdhe_privkey", 32), # our X25519 private scalar + ("tls_ecdhe_pubkey", 32), # our X25519 public key (= G * priv) + ("tls_server_pubkey", 32), # server's X25519 public key (from SH) + ("tls_shared_secret", 32), # X25519(priv, server_pub) + ("tls_client_random", 32), # CH.random + ("tls_server_random", 32), # SH.random + ("tls_transcript", 32), # SHA-256(CH||SH), context for derives + ("tls_c_hs_secret", 32), # client-handshake-traffic-secret + ("tls_s_hs_secret", 32), # server-handshake-traffic-secret + ("tls_derived_tmp", 32), # "derived" intermediate + ("tls_verify_data", 32), # computed Finished verify_data + ("tls_finished_key", 32), # HKDF-Expand-Label(..., "finished", ...) + # --- net state --- + ("net_last_error", 1), + ("net_tcp_state", 1), + ("net_initialized", 1), + ("net_poll_entry_count", 2), + ("net_poll_return_count", 2), + # --- http parser --- + ("http_parse_state", 1), + ("http_status", 2), + ("http_resp_len", 2), + ] + + dump: dict = {"labels_file": str(LABELS_PATH)} + for name, n in byte_specs: + if name not in labels: + continue + addr = labels[name] + try: + raw = bytes(transport.read_memory(addr, n)) + except Exception as exc: + dump[name] = {"addr": f"${addr:04X}", "error": str(exc)} + continue + entry: dict = {"addr": f"${addr:04X}", "hex": raw.hex()} + if n == 1: + entry["u8"] = raw[0] + elif n == 2: + entry["u16_le"] = raw[0] | (raw[1] << 8) + elif n == 8: + entry["u64_le"] = int.from_bytes(raw, "little") + dump[name] = entry + + # Ring indices, in one struct for easy cross-reference with ring.bin + ring_head_addr = labels.get("tcp_recv_head") + ring_tail_addr = labels.get("tcp_recv_tail") + ring_ovf_addr = labels.get("tcp_recv_overflow") + ring_entry: dict = {} + if ring_head_addr is not None: + try: + b = transport.read_memory(ring_head_addr, 2) + ring_entry["head"] = b[0] | (b[1] << 8) + ring_entry["head_addr"] = f"${ring_head_addr:04X}" + except Exception as exc: + ring_entry["head_error"] = str(exc) + if ring_tail_addr is not None: + try: + b = transport.read_memory(ring_tail_addr, 2) + ring_entry["tail"] = b[0] | (b[1] << 8) + ring_entry["tail_addr"] = f"${ring_tail_addr:04X}" + except Exception as exc: + ring_entry["tail_error"] = str(exc) + if ring_ovf_addr is not None: + try: + b = transport.read_memory(ring_ovf_addr, 1) + ring_entry["overflow"] = b[0] + except Exception as exc: + ring_entry["overflow_error"] = str(exc) + dump["ring"] = ring_entry + + (run_dir / "tls_state_dump.json").write_text(json.dumps(dump, indent=2)) + + +def _dump_ring(transport: Ultimate64Transport, + labels: dict[str, int], + run_dir: Path) -> None: + """DMA-snapshot the entire TCP receive ring + metadata to ``run_dir``. + + ``ring.bin`` is the raw 4 KB buffer starting at ``tcp_recv_buf`` (no + reordering — consumers use ``ring_meta.json`` to find head/tail/size). + ``ring_meta.json`` records base address, size, and current head/tail + so a post-mortem tool can slice out just the live window. + """ + base = labels.get("tcp_recv_buf", 0xC000) + # Match TCP_RECV_MASK = $0FFF from constants.inc — 4 KB ring. + size = 4096 + try: + raw = bytes(transport.read_memory(base, size)) + except Exception as exc: + (run_dir / "ring_meta.json").write_text(json.dumps({ + "error": f"read_memory failed: {exc}", + "base_addr": f"${base:04X}", + "size": size, + }, indent=2)) + return + (run_dir / "ring.bin").write_bytes(raw) + + meta: dict = { + "base_addr": f"${base:04X}", + "size": size, + "mask": "0x0FFF (TCP_RECV_MASK)", + "file": "ring.bin", + "note": "ring.bin is tcp_recv_buf[0..4095] verbatim; head/tail " + "are masked indices *into* this buffer (not byte offsets " + "relative to a live window). See net/uci/net.s " + "net_recv_byte for addressing.", + } + try: + b = transport.read_memory(labels["tcp_recv_head"], 2) + meta["head"] = b[0] | (b[1] << 8) + except Exception as exc: + meta["head_error"] = str(exc) + try: + b = transport.read_memory(labels["tcp_recv_tail"], 2) + meta["tail"] = b[0] | (b[1] << 8) + except Exception as exc: + meta["tail_error"] = str(exc) + try: + meta["overflow"] = transport.read_memory( + labels["tcp_recv_overflow"], 1)[0] + except Exception as exc: + meta["overflow_error"] = str(exc) + + (run_dir / "ring_meta.json").write_text(json.dumps(meta, indent=2)) + + +def _dump_full(transport: Ultimate64Transport, + labels: dict[str, int], + server_result: dict, + run_dir: Path | None = None) -> None: + """Dump the full diagnostic set (used on both success and TIMEOUT paths). + + When ``run_dir`` is provided, extra binary/JSON artifacts are written + alongside the trace files for post-mortem decoding: + - ``ring.bin`` / ``ring_meta.json`` : full 4 KB tcp_recv_buf + head/tail + - ``tls_state_dump.json`` : every TLS state-machine variable + """ + _dump_diag(transport, labels) + + try: + carry_byte = transport.read_memory(CARRY_FLAG_ADDR, 1)[0] + carry = carry_byte & 0x01 + print(f"http_get carry = {carry} (0=success, 1=failure)") + except Exception as exc: + print(f"http_get carry = ") + + try: + status_raw = transport.read_memory(labels["http_status"], 2) + http_status = status_raw[0] | (status_raw[1] << 8) + print(f"http_status = {http_status}") + except Exception as exc: + print(f"http_status = ") + + resp_len = 0 + try: + resp_len_raw = transport.read_memory(labels["http_resp_len"], 2) + resp_len = resp_len_raw[0] | (resp_len_raw[1] << 8) + print(f"http_resp_len = {resp_len}") + except Exception as exc: + print(f"http_resp_len = ") + + try: + read_len = min(resp_len, 200) if resp_len > 0 else 200 + resp_data = bytes(transport.read_memory(labels["http_resp_buf"], + read_len)) + print(f"http_resp_buf = {resp_data[:80]!r}") + if read_len > 80: + print(f"http_resp_buf+ = {resp_data[80:200]!r}") + except Exception as exc: + print(f"http_resp_buf = ") + + try: + screen = bytes(transport.read_memory(0x0400, 1000)) + screen_text = _decode_screen_ram(screen) + print("\n--- screen RAM ---") + for line in screen_text.split('\n'): + if line.strip(): + print(f" {line}") + except Exception as exc: + print(f"screen RAM read failed: {exc}") + + try: + ring_data = bytes(transport.read_memory(0xC000, 256)) + print(f"\ntcp_recv_buf[0:128] hex :") + for off in (0, 64, 128, 192): + print(f" +{off:03X} : {ring_data[off:off+64].hex()}") + except Exception as exc: + print(f"tcp_recv_buf read failed: {exc}") + + print("\n--- server-side ---") + print(f" listening : {server_result.get('listening', False)}") + print(f" client_addr : {server_result.get('client_addr')}") + print(f" request : {server_result.get('request', b'')!r}") + print(f" error : {server_result.get('error', '')}") + + # --- Persist ring + TLS-state snapshots alongside the trace --- + if run_dir is not None: + try: + _dump_ring(transport, labels, run_dir) + print(f" ring.bin -> {run_dir / 'ring.bin'}") + except Exception as exc: + print(f"WARNING: ring dump failed: {exc}") + try: + _dump_tls_state_snapshot(transport, labels, run_dir) + print(f" tls_state_dump.json -> " + f"{run_dir / 'tls_state_dump.json'}") + except Exception as exc: + print(f"WARNING: tls_state_dump failed: {exc}") + + +def _git_head_sha() -> str: + """Return the short git HEAD SHA, or '' if git is unavailable.""" + try: + out = subprocess.check_output( + ["git", "rev-parse", "--short", "HEAD"], + cwd=str(REPO_ROOT), + stderr=subprocess.DEVNULL, + timeout=3.0, + ) + return out.decode("ascii", errors="replace").strip() or "" + except Exception: + return "" + + +def _create_run_dir(base_dir: Path) -> Path: + """Create and return a timestamped run directory under ``base_dir``.""" + base_dir.mkdir(parents=True, exist_ok=True) + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + run_dir = base_dir / stamp + # Disambiguate if a same-second run dir already exists + suffix = 0 + candidate = run_dir + while candidate.exists(): + suffix += 1 + candidate = base_dir / f"{stamp}_{suffix}" + candidate.mkdir(parents=True, exist_ok=False) + return candidate + + +def _prune_old_run_dirs(base_dir: Path, keep: int) -> list[Path]: + """Keep the ``keep`` most-recent run directories; remove the rest. + + Orders by mtime (not filename) so DST/timezone oddities don't misorder. + Returns the list of directories that were removed (for logging). + """ + if not base_dir.is_dir(): + return [] + entries: list[tuple[float, Path]] = [] + for child in base_dir.iterdir(): + if child.is_dir(): + try: + entries.append((child.stat().st_mtime, child)) + except OSError: + pass + entries.sort(key=lambda t: t[0], reverse=True) + removed: list[Path] = [] + for _, d in entries[keep:]: + try: + shutil.rmtree(d) + removed.append(d) + except Exception as exc: + print(f"WARNING: failed to prune {d}: {exc}") + return removed + + +def _serialize_trace_packed(cap_result, trace_path: Path, + meta_path: Path) -> dict: + """Serialize the raw BusCycle trace to a packed 4-byte-per-cycle file. + + Each BusCycle carries only a 32-bit ``raw`` word, so the packed + little-endian u32 stream is lossless and dramatically more compact + than pickle. Writes ``trace.bin`` and a ``trace.bin.meta.json`` + sidecar that describes the layout for post-hoc readers. + + Returns a small dict with size/count for logging. + """ + trace = getattr(cap_result, "trace", None) or [] + count = len(trace) + # Pre-size buffer and pack in one shot. Fall back to per-cycle pack if + # the in-tree BusCycle layout ever grows richer. + try: + fmt = f"<{count}I" + buf = struct.pack(fmt, *(int(c.raw) & 0xFFFFFFFF for c in trace)) + except Exception: + parts = [struct.pack(" None: + """Dump the HTTPS listener's per-connection dict to JSON. + + Bytes fields are base64-encoded (round-trippable, unlike repr()). + Non-serializable exception values are captured as type/str/traceback. + """ + out: dict = {} + for key, val in server_result.items(): + if isinstance(val, (bytes, bytearray)): + out[key] = { + "__type__": "bytes-b64", + "b64": base64.b64encode(bytes(val)).decode("ascii"), + "len": len(val), + } + elif isinstance(val, BaseException): + out[key] = { + "__type__": "exception", + "class": type(val).__name__, + "str": str(val), + "traceback": traceback.format_exception( + type(val), val, val.__traceback__), + } + elif isinstance(val, tuple): + # client_addr is (host, port) — JSON has no tuple, keep as list + out[key] = list(val) + else: + try: + json.dumps(val) + out[key] = val + except TypeError: + out[key] = repr(val) + path.write_text(json.dumps(out, indent=2, default=str)) + + +def _write_run_info(path: Path, *, outcome: str, duration: float, + exit_code: int, extra: dict | None = None) -> None: + """Write one-line metadata for a run (git SHA, outcome, duration, rc).""" + sha = _git_head_sha() + ts = datetime.datetime.now().isoformat(timespec="seconds") + lines = [ + f"timestamp = {ts}", + f"git_head = {sha}", + f"outcome = {outcome}", + f"duration_s = {duration:.3f}", + f"exit_code = {exit_code}", + ] + if extra: + for k, v in extra.items(): + lines.append(f"{k:<13} = {v}") + path.write_text("\n".join(lines) + "\n") + + +def _process_debug_trace(cap_result, + summary_path: str, + tail_path: str, + uci_path: str) -> dict: + """Post-process the debug-stream BusCycle list into three artifacts. + + Returns a small dict with high-level stats for console logging. + """ + stats: dict = {} + trace = getattr(cap_result, "trace", None) or [] + total = len(trace) + packets = getattr(cap_result, "packets_received", 0) + dropped = getattr(cap_result, "packets_dropped", 0) + duration = getattr(cap_result, "duration_seconds", 0.0) or 0.0 + bytes_recv = getattr(cap_result, "bytes_received", 0) or 0 + stats["total"] = total + stats["packets"] = packets + stats["dropped"] = dropped + stats["duration"] = duration + + cpu_count = 0 + vic_count = 0 + cpu_read_hist: dict = {} + uci_hist: dict = {} + uci_tail: list = [] + cpu_tail: list = [] # ring of last CPU cycles + + tail_cap = 2000 + huge = total > 2_000_000 + # If huge, still compute PC hotspots but only over the LAST 500k CPU + # cycles, to bound cost. For small traces use everything. + scan_window = total + scan_start = 0 + if huge: + scan_start = max(0, total - 2_000_000) + scan_window = total - scan_start + + cpu_idx_for_tail_start = max(0, total - tail_cap * 4) # oversample — trim later + + for i, c in enumerate(trace): + is_cpu = bool(getattr(c, "is_cpu", False)) + if is_cpu: + cpu_count += 1 + if i >= scan_start and bool(getattr(c, "is_read", False)): + addr = int(getattr(c, "address", 0)) & 0xFFFF + cpu_read_hist[addr] = cpu_read_hist.get(addr, 0) + 1 + if i >= cpu_idx_for_tail_start: + cpu_tail.append((i, c)) + if len(cpu_tail) > tail_cap * 2: + # keep it bounded while iterating + cpu_tail = cpu_tail[-tail_cap:] + addr = int(getattr(c, "address", 0)) & 0xFFFF + if 0xDF1B <= addr <= 0xDF1F: + uci_hist[addr] = uci_hist.get(addr, 0) + 1 + uci_tail.append((i, c)) + else: + vic_count += 1 + + # Trim tail to exactly last tail_cap CPU cycles + if len(cpu_tail) > tail_cap: + cpu_tail = cpu_tail[-tail_cap:] + + top_pcs = sorted(cpu_read_hist.items(), key=lambda kv: kv[1], + reverse=True)[:20] + top_uci = sorted(uci_hist.items(), key=lambda kv: kv[1], + reverse=True)[:10] + + stats["cpu"] = cpu_count + stats["vic"] = vic_count + stats["top_pcs"] = top_pcs + stats["top_uci"] = top_uci + stats["uci_tail"] = uci_tail[-10:] + stats["cpu_tail_sample"] = cpu_tail[-20:] + + # --- Write summary --- + with open(summary_path, "w") as f: + f.write(f"DebugCapture summary\n") + f.write(f" packets_received = {packets}\n") + f.write(f" packets_dropped = {dropped}\n") + f.write(f" duration_seconds = {duration:.3f}\n") + f.write(f" bytes_received = {bytes_recv}\n") + bps = (bytes_recv / duration) if duration > 0 else 0 + f.write(f" bytes_per_second = {bps:.0f}\n") + f.write(f" total_cycles = {total}\n") + f.write(f" cpu_cycles = {cpu_count}\n") + f.write(f" vic_cycles = {vic_count}\n") + if huge: + f.write(f" NOTE: trace > 2M cycles; PC histogram computed over " + f"last {scan_window} cycles only.\n") + f.write(f"\nTop 20 CPU-read addresses (approx PC hotspots):\n") + for addr, count in top_pcs: + f.write(f" ${addr:04X} {count}\n") + f.write(f"\nTop 10 UCI register accesses ($DF1B-$DF1F):\n") + for addr, count in top_uci: + f.write(f" ${addr:04X} {count}\n") + + # --- Write tail (last 2000 CPU cycles) --- + with open(tail_path, "w") as f: + f.write(f"# last {len(cpu_tail)} CPU cycles (of {cpu_count} total)\n") + for idx, c in cpu_tail: + rw = "R" if getattr(c, "is_read", False) else \ + ("W" if getattr(c, "is_write", False) else "?") + addr = int(getattr(c, "address", 0)) & 0xFFFF + data = int(getattr(c, "data", 0)) & 0xFF + f.write(f"{idx:>9d} {rw} ${addr:04X} = ${data:02X}\n") + + # --- Write UCI-register filter (every CPU access to $DF1B-$DF1F) --- + with open(uci_path, "w") as f: + f.write(f"# {len(uci_tail)} CPU cycles in $DF1B-$DF1F\n") + for run_idx, (idx, c) in enumerate(uci_tail): + rw = "R" if getattr(c, "is_read", False) else \ + ("W" if getattr(c, "is_write", False) else "?") + addr = int(getattr(c, "address", 0)) & 0xFFFF + data = int(getattr(c, "data", 0)) & 0xFF + f.write(f"{run_idx:>7d} @{idx:>9d} {rw} ${addr:04X} = ${data:02X}\n") + + stats["uci_total"] = len(uci_tail) + return stats + + +def main() -> int: + if not PRG_PATH.is_file(): + print(f"ERROR: PRG not found at {PRG_PATH}", file=sys.stderr) + return 2 + if not LABELS_PATH.is_file(): + print(f"ERROR: labels.txt not found", file=sys.stderr) + return 2 + if not CERT_PATH.is_file() or not KEY_PATH.is_file(): + print(f"ERROR: cert/key not found at {CERT_PATH} / {KEY_PATH}", + file=sys.stderr) + return 2 + + labels = _load_labels() + required = [ + "http_get", "http_host_ptr", "http_host_len", + "http_path_ptr", "http_path_len", "http_port", + "net_init", "net_initialized", "uci_socket_id", + "tcp_recv_head", "tcp_recv_tail", + "http_resp_buf", "http_resp_len", "http_status", + "tls_state", "tls_last_state", + ] + missing = [n for n in required if n not in labels] + if missing: + print(f"ERROR: missing labels: {missing}", file=sys.stderr) + return 2 + + for n in sorted(required): + print(f" {n:22s} = ${labels[n]:04X}") + + test_host_ip = _detect_local_ip(HOST) + print(f"\nDev host LAN IP : {test_host_ip}") + print(f"Cert / key : {CERT_PATH} / {KEY_PATH}") + + # --- Bind HTTPS listener (try default port, fall back to 4433) --- + ctx = _make_ssl_context() + srv = _try_bind(test_host_ip, DEFAULT_HTTPS_PORT) + chosen_port = DEFAULT_HTTPS_PORT + if srv is None: + if DEFAULT_HTTPS_PORT != FALLBACK_HTTPS_PORT: + print(f"NOTE: bind {test_host_ip}:{DEFAULT_HTTPS_PORT} failed" + f" (need root?), falling back to {FALLBACK_HTTPS_PORT}") + srv = _try_bind(test_host_ip, FALLBACK_HTTPS_PORT) + chosen_port = FALLBACK_HTTPS_PORT + if srv is None: + print(f"ERROR: could not bind HTTPS listener", file=sys.stderr) + return 1 + print(f"HTTPS port : {chosen_port}") + print(f"Expected body : {EXPECTED_BODY!r}") + + server_result: dict = {} + server_thread = threading.Thread( + target=_run_https_server, + args=(srv, ctx, server_result), + daemon=True, + ) + server_thread.start() + for _ in range(60): + if server_result.get("listening"): + break + time.sleep(0.05) + else: + print("ERROR: HTTPS server failed to start", file=sys.stderr) + return 1 + print(f"HTTPS server listening on {test_host_ip}:{chosen_port}") + + # --- Build routine (port patched in at build time) --- + routine_bytes_raw, host_len_patch = _build_http_routine(labels, chosen_port) + routine_bytes = bytearray(routine_bytes_raw) + host_ip_bytes = test_host_ip.encode("ascii") + routine_bytes[host_len_patch] = len(host_ip_bytes) + routine_bytes = bytes(routine_bytes) + + print(f"Routine size : {len(routine_bytes)} bytes @ ${ROUTINE_ADDR:04X}") + + host_str = host_ip_bytes + b"\x00" + path_str = b"/\x00" + + prg = PRG_PATH.read_bytes() + + lock = DeviceLock(HOST) + if not lock.acquire(timeout=60.0): + print(f"ERROR: could not acquire DeviceLock({HOST})", file=sys.stderr) + return 3 + print(f"Acquired DeviceLock({HOST})") + + # --- Per-run debug artifact directory + rotation --- + run_dir: Path | None = None + if DEBUG_CAPTURE_ENABLED: + removed = _prune_old_run_dirs(UCI_DEBUG_BASE_DIR, UCI_DEBUG_KEEP) + for d in removed: + print(f"Pruning old debug artifacts: {d}") + run_dir = _create_run_dir(UCI_DEBUG_BASE_DIR) + print(f"Debug artifacts dir: {run_dir}") + + client: Ultimate64Client | None = None + uci_enabled = False + debug_cap: DebugCapture | None = None + debug_started_on_u64 = False + outcome: str = "UNKNOWN" + exit_code: int = 1 + run_start = time.time() + try: + client = Ultimate64Client(host=HOST, timeout=15.0) + transport = Ultimate64Transport(host=HOST, timeout=15.0, client=client) + + print("Enabling UCI...") + enable_uci(client) + uci_enabled = True + + print("Resetting machine...") + client.reset() + time.sleep(2.5) + + print("run_prg(PRG)...") + client.run_prg(prg) + # Wait for auto-init (entropy, REU stash, DHCP) + time.sleep(22.0) + + init_flag = transport.read_memory(labels["net_initialized"], 1)[0] + print(f"net_initialized = ${init_flag:02X}") + if init_flag == 0: + print("WARNING: net_initialized is 0 — auto-init may have failed") + + # --- Flip to 48 MHz turbo BEFORE we trigger the stub --- + print(f"Setting turbo to {TURBO_MHZ} MHz...") + set_turbo_mhz(client, TURBO_MHZ) + time.sleep(0.5) + + # --- Start 6510 debug-stream capture (after turbo, before trigger) --- + if DEBUG_CAPTURE_ENABLED: + try: + set_debug_stream_mode(client, DEBUG_MODE_6510) + debug_cap = DebugCapture( + port=DEFAULT_DEBUG_PORT, + recv_buf_size=1024 * 1024, + max_bytes=50 * 1024 * 1024, # ~50 MB rolling filtered window + filter=_keep_cycle, + ) + debug_cap.start() + debug_dest = f"{test_host_ip}:{DEFAULT_DEBUG_PORT}" + print(f"Starting U64 6510 debug stream -> {debug_dest}") + client.stream_debug_start(debug_dest) + debug_started_on_u64 = True + time.sleep(0.3) + except Exception as exc: + print(f"WARNING: debug capture failed to start: {exc}") + debug_cap = None + else: + print("DEBUG_CAPTURE=0 — 6510 stream disabled") + + # Quit PRG main_loop back to BASIC + print("Sending 'Q' to exit PRG main_loop...") + send_text(transport, "q\r") + time.sleep(2.0) + + # DMA-write the routine + data + CHUNK = 64 + for i in range(0, len(routine_bytes), CHUNK): + transport.write_memory( + ROUTINE_ADDR + i, + routine_bytes[i:i + CHUNK], + ) + transport.write_memory(HOST_STR_ADDR, host_str.ljust(32, b"\x00")) + transport.write_memory(PATH_STR_ADDR, path_str.ljust(8, b"\x00")) + + # Clear sentinel area + transport.write_memory(SENTINEL_ADDR, bytes(16)) + + # Trigger via SYS + sys_line = f"sys{ROUTINE_ADDR}\r" + print(f"Triggering: {sys_line.strip()}") + send_text(transport, sys_line) + + # Poll sentinel + deadline = time.time() + SENTINEL_POLL_TIMEOUT + last_progress = -1 + start = time.time() + while time.time() < deadline: + time.sleep(0.5) + blob = transport.read_memory(SENTINEL_ADDR, 2) + sentinel = blob[0] + progress = blob[1] + if progress != last_progress: + elapsed = time.time() - start + print(f" [{elapsed:6.1f}s] progress=0x{progress:02X}") + last_progress = progress + if sentinel == SENTINEL_VALUE: + print(" sentinel set — routine complete") + break + else: + print(f"TIMEOUT: sentinel not set after " + f"{SENTINEL_POLL_TIMEOUT:.0f}s " + f"(progress=0x{last_progress:02X})", file=sys.stderr) + server_thread.join(timeout=1.0) + _dump_full(transport, labels, server_result, run_dir=run_dir) + outcome = "TIMEOUT" + exit_code = 1 + return exit_code + + # --- Results --- + # Join server thread briefly so server_result is populated + server_thread.join(timeout=5.0) + _dump_full(transport, labels, server_result, run_dir=run_dir) + + # Reread resp_data + screen_text for the assertion logic below. + resp_len_raw = transport.read_memory(labels["http_resp_len"], 2) + resp_len = resp_len_raw[0] | (resp_len_raw[1] << 8) + read_len = min(resp_len, 200) if resp_len > 0 else 200 + resp_data = bytes(transport.read_memory(labels["http_resp_buf"], + read_len)) + screen = bytes(transport.read_memory(0x0400, 1000)) + screen_text = _decode_screen_ram(screen) + + # --- Assertions --- + body_ascii = "" + try: + body_ascii = resp_data.decode("ascii", errors="replace") + except Exception: + pass + + if EXPECTED_BODY in body_ascii: + print(f"\nPASS: http_resp_buf contains '{EXPECTED_BODY}'") + outcome = "PASS" + exit_code = 0 + return exit_code + + if "HELLO" in screen_text.upper(): + print(f"\nPASS: screen RAM contains HELLO " + f"(body in resp_buf may differ in encoding)") + outcome = "PASS" + exit_code = 0 + return exit_code + + print(f"\nFAIL: expected '{EXPECTED_BODY}' not found in response" + f" or screen", file=sys.stderr) + outcome = "FAIL" + exit_code = 1 + return exit_code + + finally: + # --- Stop 6510 debug stream; post-process + persist trace --- + if debug_started_on_u64 and client is not None: + try: + client.stream_debug_stop() + except Exception as exc: + print(f"WARNING: stream_debug_stop failed: {exc}") + trace_bytes_on_disk = 0 + if debug_cap is not None and run_dir is not None: + cap_result = None + try: + cap_result = debug_cap.stop() + except Exception as exc: + print(f"WARNING: debug capture stop failed: {exc}") + if cap_result is not None: + try: + stats = _process_debug_trace( + cap_result, + summary_path=str(run_dir / "summary.txt"), + tail_path=str(run_dir / "tail.txt"), + uci_path=str(run_dir / "uci_accesses.txt"), + ) + print(f"\nDebug capture: {stats.get('packets', 0)} pkts, " + f"{stats.get('dropped', 0)} dropped, " + f"{stats.get('total', 0)} cycles, " + f"{stats.get('duration', 0.0):.1f}s " + f"(cpu={stats.get('cpu', 0)} vic={stats.get('vic', 0)}; " + f"uci_hits={stats.get('uci_total', 0)})") + except Exception as exc: + print(f"WARNING: debug trace post-process failed: {exc}") + + # Persist the raw trace as packed u32-LE, with a JSON + # sidecar describing the bit layout. + try: + trace_bin = run_dir / "trace.bin" + trace_meta = run_dir / "trace.bin.meta.json" + tstats = _serialize_trace_packed( + cap_result, trace_bin, trace_meta) + trace_bytes_on_disk = tstats["bytes_on_disk"] + mb = trace_bytes_on_disk / (1024 * 1024) + print(f" raw trace : {trace_bin} " + f"({tstats['cycle_count']} cycles, {mb:.2f} MB)") + except Exception as exc: + print(f"WARNING: raw trace serialize failed: {exc}") + + # --- Persist server-side listener state --- + if run_dir is not None: + try: + _serialize_server_result( + server_result, run_dir / "server_result.json") + except Exception as exc: + print(f"WARNING: server_result dump failed: {exc}") + + # --- Write run metadata --- + if run_dir is not None: + try: + _write_run_info( + run_dir / "run_info.txt", + outcome=outcome, + duration=time.time() - run_start, + exit_code=exit_code, + extra={ + "trace_bytes": trace_bytes_on_disk, + "turbo_mhz": TURBO_MHZ, + "host": HOST, + }, + ) + except Exception as exc: + print(f"WARNING: run_info write failed: {exc}") + + # Print run dir prominently for operator / test harness. + print(f"\nDebug artifacts: {run_dir}") + # Optional: drop the run dir on PASS unless the operator + # asked to keep it. 5-dir rotation still applies regardless. + if outcome == "PASS" and not UCI_DEBUG_KEEP_ON_PASS: + try: + shutil.rmtree(run_dir) + print(f"(removed on PASS; set KEEP_DEBUG_ON_PASS=1 to " + f"retain)") + except Exception as exc: + print(f"WARNING: failed to remove PASS run dir: {exc}") + + if uci_enabled and client is not None: + print("\nDisabling UCI...") + try: + disable_uci(client) + except Exception as exc: + print(f"WARNING: disable_uci failed: {exc}") + lock.release() + print(f"Released DeviceLock({HOST})") + + +if __name__ == "__main__": + raise SystemExit(main())