From 20ed07c723f49ab3b14777a80f9688e756f367dd Mon Sep 17 00:00:00 2001 From: EMRG Evolution Date: Mon, 31 Aug 2026 17:43:00 +0800 Subject: [PATCH] emrg: planted-fire drill rides the real tokenizer-switch path (issue #1087) --- Agent.md | 2 +- emrg/server/daemon.py | 112 ++++++++++++++++++ scripts/calibrate_silent_drift_threshold.py | 14 +++ ...est_ci_calibrate_silent_drift_threshold.py | 23 ++++ tests/test_daemon.py | 60 ++++++++++ 5 files changed, 210 insertions(+), 1 deletion(-) diff --git a/Agent.md b/Agent.md index cf8df56d..5ab47182 100644 --- a/Agent.md +++ b/Agent.md @@ -119,7 +119,7 @@ Community needs voiced in HN agent-UI discussions map directly to EMRG's design: pkill -f "emrg.server"; rm -f ~/.emrg/emrgd.token; python -m emrg ``` -Python: `uv run pytest tests/ -v` (1191) — import check: `uv run python -c "from emrg.client.app import run_client"` +Python: `uv run pytest tests/ -v` (1195) — import check: `uv run python -c "from emrg.client.app import run_client"` GUI: `cd emrg/gui && npm test` (95: 45 daemon_client + 20 conn-manager + 8 integration + 6 build-config + 7 gui-state + 3 preload-api + 4 boot-contract + 2 theme-guard) — syntax: `node --check main.js preload.js daemon_client.js` Renderer: `cd emrg/gui/renderer && npm run typecheck && npm test` (480: 5 snapshot-store + 9 utils + 3 ErrorBoundary + 2 App smoke + 11 commands + 4 copywriting + 11 i18n + 11 markdown + 17 transcript + 10 TranscriptView + 15 history + 22 composer + 34 Composer + 6 LinkDialog + 12 sidebar + 17 Sidebar + 9 fileTree + 9 FileTree + 16 resultPanel + 8 ResultPanel + 27 workspaceView + 10 WorkspaceView + 10 dialog + 6 Dialog + 9 ConfirmDialog + 9 RenameDialog + 10 dialogLists + 3 HelpDialog + 9 MemoryDialog + 6 SkillsDialog + 9 openSession + 6 WelcomeDialog + 8 OpenSessionDialog + 7 NewSessionDialog + 7 rewind + 8 RewindDialog + 7 GithubDeviceDialog + 15 daemonBridge + 7 DaemonBridgeProvider + 26 Shell + 15 DialogHost + 20 SettingsPanel + 6 TaskFormDialog + 5 RantDialog + 4 vendorMarkdown) + `npm run build` → `renderer/dist/` CI: `uv run pytest` (ubuntu + **windows-2025 matrix** — Windows pytest 回归在 PR CI 即失败，v0.2.29 教训 #725) + GUI tests + **actionlint workflow lint** (`rhysd/actionlint@v1.7.12` gate, #444 — workflow 解析错误在 PR CI 即失败，如 `if:` secrets 上下文) diff --git a/emrg/server/daemon.py b/emrg/server/daemon.py index 4dd45c67..364ed6d8 100644 --- a/emrg/server/daemon.py +++ b/emrg/server/daemon.py @@ -27,6 +27,7 @@ import traceback from datetime import datetime from pathlib import Path +from types import SimpleNamespace from typing import Optional from urllib.parse import urlparse @@ -431,6 +432,12 @@ async def serve(self) -> None: self._planted_fire_alarm_task = asyncio.create_task( self._planted_fire_alarm_loop()) + # Issue #1087: scheduled planted-fire drill (daily). Rides the REAL + # tokenizer-switch path (_refresh_usage_anchor → detector) so a drill + # passing means the production path plus the detector work end to end. + self._planted_fire_drill_task = asyncio.create_task( + self._planted_fire_drill_loop()) + try: await self._server.serve_forever() except asyncio.CancelledError: @@ -3153,6 +3160,99 @@ async def _planted_fire_alarm_loop(self) -> None: except Exception: logger.debug("planted-fire alarm tick failed", exc_info=True) + async def _planted_fire_drill_loop(self) -> None: + """Issue #1087: scheduled planted-fire drill (same-door constraint). + Daily cadence — the drill rides the REAL tokenizer-switch path, so a + drill that passes means the production switch path (plus the + detector) works end to end; a drill that fails means the planted fire + is dead even though real switches may not have happened recently. + Failures are logged at debug and never crash the daemon.""" + while True: + await asyncio.sleep(_PLANTED_FIRE_DRILL_INTERVAL) + try: + self._run_planted_fire_drill() + except Exception: + logger.debug("planted-fire drill tick failed", exc_info=True) + + def _run_planted_fire_drill(self) -> bool: + """Issue #1087 (heinrichneb, Dev.to 3doei): ride the REAL + tokenizer-switch path — same door as a genuine provider/tokenizer + change. Instead of calling the detector directly (a bypass that + proves less), fabricate a synthetic round whose real_pt deviates + beyond _SILENT_DRIFT_THRESHOLD and push it through + ``_refresh_usage_anchor`` — the exact production entry point the + detector guards. When the fabricated switch is detected, the drill + has proven the full path works; return True. A drill that does NOT + fire is the planted-fire failure mode (the guard is dead while + looking alive — the #1072/#1075 failure shape). + + The drill is identifiable in logs via the reserved session id + (``planted-fire-drill``) that flows through every heartbeat/drift + event, so operators can distinguish drill-triggered switches from + real ones. It never touches real anchors (dedicated session id) and + cleans up its synthetic anchor afterwards. + """ + sid = _PLANTED_FIRE_DRILL_SESSION + session = SimpleNamespace(session_id=sid) + messages = [{"role": "user", "content": "planted-fire drill round"}] + estimate = self._estimate_tokens(messages) + if estimate <= 0: + logger.debug("planted-fire-drill: estimate invalid, skipped") + return False + # Count existing drift events for the drill session so the drill can + # assert THIS run emitted a fresh one (not a previous drill's). + before = self._count_drill_drift_events() + # Plant a known anchor (bias 1.0) so the detector has a baseline. + self._usage_anchors[sid] = (estimate, estimate) + # Fabricate the switch: real_pt deviates beyond the threshold. + switched_real = estimate + max(1, int(estimate * _SILENT_DRIFT_THRESHOLD * 2)) + try: + # Ride the REAL path — a genuine provider change would take + # exactly this entry point. + self._refresh_usage_anchor( + session, {"prompt_tokens": switched_real}, messages) + finally: + # Never leak the synthetic anchor into real state. + self._usage_anchors.pop(sid, None) + after = self._count_drill_drift_events() + fired = after > before + if fired: + logger.warning( + "planted-fire-drill: PASS — synthetic tokenizer switch " + "(real %d vs est %d, bias_shift>%.0f%%) detected via the real " + "path (session=%s)", + switched_real, estimate, _SILENT_DRIFT_THRESHOLD * 100, sid, + ) + else: + logger.warning( + "planted-fire-drill: FAIL — synthetic tokenizer switch " + "(real %d vs est %d) NOT detected; the planted fire is dead " + "or the detector is broken (session=%s)", + switched_real, estimate, sid, + ) + return fired + + def _count_drill_drift_events(self) -> int: + """Issue #1087: count ``anchor_provider_drift`` events attributed to + the reserved drill session in the usage-anchor stats file. Best-effort: + an unreadable/missing file counts 0 (the drill must never crash the + daemon; a stats write failing is itself a signal the #1072 + measurability contract is broken, surfaced as drill FAIL).""" + n = 0 + try: + with open(_USAGE_ANCHOR_STATS_PATH, "r", encoding="utf-8") as fh: + for line in fh: + try: + ev = json.loads(line) + except ValueError: + continue + if (ev.get("type") == "anchor_provider_drift" + and ev.get("session") == _PLANTED_FIRE_DRILL_SESSION): + n += 1 + except OSError: + pass + return n + def _refresh_usage_anchor(self, session, final_usage, messages) -> None: """Round-loop usage processing (rant 2026-08-23T13:28:50 + issue #1078): refresh the usage anchor from the provider's real @@ -4467,6 +4567,18 @@ def _write_exit_record(reason: str, exit_code: int, traceback_text: str | None) # is caught within 6h of the N-day threshold instead of after a full day. _PLANTED_FIRE_ALARM_INTERVAL = 6 * 3600 # seconds +# Issue #1087 (heinrichneb, Dev.to 3doei — same-door constraint): the +# scheduled planted-fire drill uses a reserved session id so every heartbeat +# / drift event it produces is identifiable as a drill in logs and stats +# (distinguishing drill-triggered switches from real ones). +_PLANTED_FIRE_DRILL_SESSION = "planted-fire-drill" + +# Issue #1087: daily drill cadence. The drill rides the real tokenizer-switch +# path and asserts the detector fires; once a day is frequent enough to catch +# a dead planted fire within 24h without flooding the stats file with +# synthetic drift events. +_PLANTED_FIRE_DRILL_INTERVAL = 24 * 3600 # seconds + # Issue #1027: relative bias-ratio shift (real_pt / local_estimate) between # consecutive anchored rounds that is treated as a silent provider/tokenizer # change. The local estimate is provider-independent, so a stable provider diff --git a/scripts/calibrate_silent_drift_threshold.py b/scripts/calibrate_silent_drift_threshold.py index a5d5aa0f..255a23df 100644 --- a/scripts/calibrate_silent_drift_threshold.py +++ b/scripts/calibrate_silent_drift_threshold.py @@ -54,6 +54,13 @@ DEFAULT_PATH = Path.home() / ".emrg" / "logs" / "usage-anchor.jsonl" DEFAULT_CURRENT = 0.25 +# Issue #1087: the scheduled planted-fire drill (daemon.py) rides the real +# tokenizer-switch path under this reserved session id, so every +# anchor_provider_drift event it produces is a SYNTHETIC switch — it must be +# excluded from calibration (it is not real drift; including it would skew +# the empirical distribution toward the fabricated shift). +DRILL_SESSION = "planted-fire-drill" + def load_events(path: Path) -> tuple[list[dict], int]: """Parse a usage-anchor.jsonl file into events. @@ -109,6 +116,10 @@ def split_events(events: list[dict]) -> tuple[list[float], list[float]]: noise: list[float] = [] drift: list[float] = [] for ev in events: + # Issue #1087: skip synthetic drill events (reserved session id) — + # they are fabricated switches, not real drift. + if ev.get("session") == DRILL_SESSION: + continue shift = bias_abs(ev) if shift is None: continue @@ -134,6 +145,9 @@ def provider_groups(events: list[dict]) -> dict[str, dict[str, list[float]]]: """ groups: dict[str, dict[str, list[float]]] = {} for ev in events: + # Issue #1087: skip synthetic drill events (reserved session id). + if ev.get("session") == DRILL_SESSION: + continue shift = bias_abs(ev) if shift is None: continue diff --git a/tests/test_ci_calibrate_silent_drift_threshold.py b/tests/test_ci_calibrate_silent_drift_threshold.py index dbc28bd1..91db9d79 100644 --- a/tests/test_ci_calibrate_silent_drift_threshold.py +++ b/tests/test_ci_calibrate_silent_drift_threshold.py @@ -85,6 +85,17 @@ def test_missing_bias_shift_skipped(self) -> None: ]) assert noise == [0.05] and drift == [] + def test_drill_events_excluded(self) -> None: + """Issue #1087: synthetic planted-fire drill events (reserved session + id) must never enter the calibration distribution — they are + fabricated switches, not real drift.""" + noise, drift = cal.split_events([ + self._ev(type="anchor_provider_drift", bias_shift=0.9, + session=cal.DRILL_SESSION), + self._ev(bias_shift=0.06), + ]) + assert noise == [0.06] and drift == [] + class TestProviderGroups: def _obs(self, prov: str, shift: float) -> dict: @@ -116,6 +127,18 @@ def test_separates_providers_and_types(self) -> None: def test_empty_events(self) -> None: assert cal.provider_groups([]) == {} + def test_drill_events_excluded_from_groups(self) -> None: + """Issue #1087: drill events (reserved session id) are synthetic — + excluded from per-provider calibration groups too.""" + groups = cal.provider_groups([ + self._drift("api.openai.com", 0.9), + self._obs("api.openai.com", 0.06), + {**self._drift("api.openai.com", 1.2), + "session": cal.DRILL_SESSION}, + ]) + assert groups["api.openai.com"]["drift"] == [0.9] + assert groups["api.openai.com"]["noise"] == [0.06] + def test_missing_provider_falls_to_question_mark(self) -> None: groups = cal.provider_groups([ {"type": "anchor_bias_observation", "bias_shift": 0.05}, diff --git a/tests/test_daemon.py b/tests/test_daemon.py index c84aa9f3..609e9a4f 100644 --- a/tests/test_daemon.py +++ b/tests/test_daemon.py @@ -1018,6 +1018,66 @@ def test_planted_fire_unparsable_marker_no_alarm(tmp_path, monkeypatch, caplog): assert "planted-fire-stale" not in caplog.text +def test_planted_fire_drill_rides_real_path_fires(tmp_path, monkeypatch, caplog): + """Issue #1087 (same-door constraint, heinrichneb Dev.to 3doei): the + scheduled drill must ride the REAL tokenizer-switch path — it fabricates + a round whose real_pt deviates beyond _SILENT_DRIFT_THRESHOLD and pushes + it through _refresh_usage_anchor (the exact production entry point a + genuine provider change takes), then asserts the detector fired + (anchor_provider_drift event + PASS log). A drill passing means the real + path plus the detector work end to end, not just the detector call.""" + stats = tmp_path / "usage-anchor.jsonl" + monkeypatch.setattr(daemon_mod, "_USAGE_ANCHOR_STATS_PATH", stats) + marker = tmp_path / "planted-fire-heartbeat" + monkeypatch.setattr(daemon_mod, "_PLANTED_FIRE_MARKER_PATH", marker) + server = _make_server() + with caplog.at_level("DEBUG", logger="emrg.server.daemon"): + fired = server._run_planted_fire_drill() + assert fired is True + assert "planted-fire-drill: PASS" in caplog.text + # The fabricated switch produced a countable drift event attributed to + # the reserved drill session (distinguishable from real switches). + lines = stats.read_text(encoding="utf-8").strip().splitlines() + drill_events = [ + json.loads(ln) for ln in lines + if json.loads(ln).get("session") == daemon_mod._PLANTED_FIRE_DRILL_SESSION + ] + assert any(ev["type"] == "anchor_provider_drift" for ev in drill_events) + # The synthetic anchor must not leak into real state. + assert daemon_mod._PLANTED_FIRE_DRILL_SESSION not in server._usage_anchors + + +def test_planted_fire_drill_no_detection_reports_fail(tmp_path, monkeypatch, caplog): + """Issue #1087 negative: when the fabricated switch is NOT detected (the + detector is dead — the #1072/#1075 failure shape), the drill must report + FAIL and emit NO drift event: a silent guard is surfaced loudly rather + than swallowed.""" + stats = tmp_path / "usage-anchor.jsonl" + monkeypatch.setattr(daemon_mod, "_USAGE_ANCHOR_STATS_PATH", stats) + marker = tmp_path / "planted-fire-heartbeat" + monkeypatch.setattr(daemon_mod, "_PLANTED_FIRE_MARKER_PATH", marker) + server = _make_server() + # Simulate the detector being broken: the real path runs but produces no + # drift event (guard stopped running while looking alive). + monkeypatch.setattr(server, "_detect_silent_anchor_drift", lambda *a, **k: None) + with caplog.at_level("DEBUG", logger="emrg.server.daemon"): + fired = server._run_planted_fire_drill() + assert fired is False + assert "planted-fire-drill: FAIL" in caplog.text + # No drift event leaked into the stats for the drill session (the file + # may not even exist — nothing was written). + if stats.exists(): + lines = stats.read_text(encoding="utf-8").strip().splitlines() + drill_drift = [ + json.loads(ln) for ln in lines + if json.loads(ln).get("type") == "anchor_provider_drift" + and json.loads(ln).get("session") == daemon_mod._PLANTED_FIRE_DRILL_SESSION + ] + assert drill_drift == [] + # Synthetic anchor still cleaned up on the fail path. + assert daemon_mod._PLANTED_FIRE_DRILL_SESSION not in server._usage_anchors + + def test_usage_anchor_cross_provider_window_attributable(tmp_path, monkeypatch): """Issue #1011 — heinrichneb: 'a counter that can't say WHICH provider went silent is half a counter'. The loss window's identity must survive