From b5fed761c7c6d90233c09914e6d59cfa1f7a517c Mon Sep 17 00:00:00 2001 From: EMRG Evolution Date: Sun, 30 Aug 2026 11:32:31 +0800 Subject: [PATCH 1/2] emrg: calibrate silent-drift threshold from accumulated sub-threshold bias shifts --- Agent.md | 2 +- emrg/server/daemon.py | 26 ++ scripts/calibrate_silent_drift_threshold.py | 287 ++++++++++++++++++ ...est_ci_calibrate_silent_drift_threshold.py | 190 ++++++++++++ tests/test_daemon.py | 38 ++- 5 files changed, 534 insertions(+), 9 deletions(-) create mode 100644 scripts/calibrate_silent_drift_threshold.py create mode 100644 tests/test_ci_calibrate_silent_drift_threshold.py diff --git a/Agent.md b/Agent.md index 2e3e3a91..8945ca52 100644 --- a/Agent.md +++ b/Agent.md @@ -119,7 +119,7 @@ Community needs voiced in HN agent-UI discussions map directly to EMRG's design: pkill -f "emrg.server"; rm -f ~/.emrg/emrgd.token; python -m emrg ``` -Python: `uv run pytest tests/ -v` (1147) — import check: `uv run python -c "from emrg.client.app import run_client"` +Python: `uv run pytest tests/ -v` (1172) — import check: `uv run python -c "from emrg.client.app import run_client"` GUI: `cd emrg/gui && npm test` (95: 45 daemon_client + 20 conn-manager + 8 integration + 6 build-config + 7 gui-state + 3 preload-api + 4 boot-contract + 2 theme-guard) — syntax: `node --check main.js preload.js daemon_client.js` Renderer: `cd emrg/gui/renderer && npm run typecheck && npm test` (476: 5 snapshot-store + 9 utils + 3 ErrorBoundary + 2 App smoke + 11 commands + 4 copywriting + 11 i18n + 11 markdown + 16 transcript + 10 TranscriptView + 15 history + 22 composer + 31 Composer + 6 LinkDialog + 12 sidebar + 17 Sidebar + 9 fileTree + 9 FileTree + 16 resultPanel + 8 ResultPanel + 27 workspaceView + 10 WorkspaceView + 10 dialog + 6 Dialog + 9 ConfirmDialog + 9 RenameDialog + 10 dialogLists + 3 HelpDialog + 9 MemoryDialog + 6 SkillsDialog + 9 openSession + 6 WelcomeDialog + 8 OpenSessionDialog + 7 NewSessionDialog + 7 rewind + 8 RewindDialog + 7 GithubDeviceDialog + 15 daemonBridge + 7 DaemonBridgeProvider + 26 Shell + 15 DialogHost + 20 SettingsPanel + 6 TaskFormDialog + 5 RantDialog + 4 vendorMarkdown) + `npm run build` → `renderer/dist/` CI: `uv run pytest` (ubuntu + **windows-2025 matrix** — Windows pytest 回归在 PR CI 即失败，v0.2.29 教训 #725) + GUI tests + **actionlint workflow lint** (`rhysd/actionlint@v1.7.12` gate, #444 — workflow 解析错误在 PR CI 即失败，如 `if:` secrets 上下文) diff --git a/emrg/server/daemon.py b/emrg/server/daemon.py index 57074406..0e535138 100644 --- a/emrg/server/daemon.py +++ b/emrg/server/daemon.py @@ -3134,6 +3134,32 @@ def _detect_silent_anchor_drift(self, session, real_pt: int, estimate: int) -> N if shift < _SILENT_DRIFT_THRESHOLD else "DRIFT — emitting event", ) if shift < _SILENT_DRIFT_THRESHOLD: + # Issue #1075 (reidmarlow, Dev.to 3dn2b): "A guard that has never + # fired and a guard that stopped running look identical on disk". + # The sub-threshold distribution IS the calibration data — without + # it, _SILENT_DRIFT_THRESHOLD stays an a-priori 25% guess and the + # detector's silence cannot be told apart from its death. Accumulate + # every within-threshold shift as a countable + # anchor_bias_observation so + # scripts/calibrate_silent_drift_threshold.py can tune the + # threshold from the empirical noise floor (same event shape as the + # drift event above, minus the type, so the script sees both sides + # of the boundary). + try: + _append_usage_anchor_event({ + "type": "anchor_bias_observation", + "session": session.session_id, + "prev_real": old_real, + "real_pt": real_pt, + "prev_est": old_est, + "estimate": estimate, + # signed ratio: +0.6 = tokenizer counts ~60% more than before + "bias_shift": round((new_bias - old_bias) / old_bias, 4), + "model": self.llm.config.model, + "provider": _provider_slug(self.llm.config.base_url), + }) + except OSError as exc: + logger.warning("usage-anchor stats append failed: %s", exc) return try: _append_usage_anchor_event({ diff --git a/scripts/calibrate_silent_drift_threshold.py b/scripts/calibrate_silent_drift_threshold.py new file mode 100644 index 00000000..f2ac3e52 --- /dev/null +++ b/scripts/calibrate_silent_drift_threshold.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +"""Calibrate _SILENT_DRIFT_THRESHOLD from the empirical bias-shift +distribution accumulated in usage-anchor.jsonl (issue #1075). + +Reader comment (reidmarlow, Dev.to 3dn2b) on "A guard that has never fired +and a guard that stopped running look identical on disk": the silent-drift +detector's threshold (emrg/server/daemon.py:4300, default 0.25 = 25% +relative bias-ratio shift) was an a-priori guess. To tune it empirically the +guard must first ACCUMULATE the sub-threshold observations — every anchored +round whose |bias_shift| stays under the threshold now appends an +``anchor_bias_observation`` event (issue #1075, daemon change), while +over-threshold rounds append ``anchor_provider_drift`` events as before. + +This script reads that file and reports: + +- how many sub-threshold observations and real-drift events exist (and the + file's time span), so a missing distribution is distinguishable from a + quiet one — the "stopped running" case; +- the sub-threshold |bias_shift| distribution (mean / p50 / p90 / p95 / p99 / + max); +- a threshold recommendation: + * no observations yet → keep the current threshold (nothing to calibrate); + * noise tail crowding the boundary (p99 >= 0.9 * threshold) → recommend + raising to p99 * 1.5 (the guard is set too tight; per-round noise can + trip it); + * noise far below the boundary → keep the current threshold (the guard + demonstrably fires only on real drift); + * if the recommended value would land above the smallest observed real + drift → "no clean separation": noise and drift overlap, a single fixed + threshold cannot separate them (keep the current one, investigate + providers / per-provider thresholds instead). + +The recommendation never lowers the threshold below its current value: +lowering a working guard only risks false alarms, and drift events already +prove the current boundary catches real drift. + +Usage: + python scripts/calibrate_silent_drift_threshold.py [--path ~/.emrg/logs/usage-anchor.jsonl] [--current 0.25] + +Exit codes: 0 on success (including "not enough data"), 1 on an unreadable +file. +""" + +from __future__ import annotations + +import argparse +import json +import math +import statistics +import sys +from pathlib import Path +from typing import Any + +DEFAULT_PATH = Path.home() / ".emrg" / "logs" / "usage-anchor.jsonl" +DEFAULT_CURRENT = 0.25 + + +def load_events(path: Path) -> tuple[list[dict], int]: + """Parse a usage-anchor.jsonl file into events. + + Returns (events, malformed_count). A missing or unreadable file raises + SystemExit(1) with a message — the script must not guess when its input + is gone (the "guard stopped running" failure mode). + """ + try: + fh = open(path, "r", encoding="utf-8") + except OSError as exc: + print(f"error: cannot read {path}: {exc}", file=sys.stderr) + raise SystemExit(1) from exc + events: list[dict] = [] + malformed = 0 + with fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + events.append(json.loads(line)) + except (ValueError, TypeError): + malformed += 1 + return events, malformed + + +def bias_abs(record: dict) -> float | None: + """Absolute bias_shift of an event, or None when missing/non-numeric. + + The events store a SIGNED ratio (+0.6 = tokenizer counts ~60% more than + before); calibration cares about magnitude only, so both directions fold + onto the same axis. + """ + val = record.get("bias_shift") + try: + magnitude = abs(float(val)) + except (TypeError, ValueError): + return None + if math.isnan(magnitude) or math.isinf(magnitude): + return None + return magnitude + + +def split_events(events: list[dict]) -> tuple[list[float], list[float]]: + """Partition events into (noise, drift) by |bias_shift|. + + noise = sub-threshold observations (anchor_bias_observation, the + calibration sample — censored below the current threshold by + construction), drift = over-threshold events (anchor_provider_drift, the + real-drift sightings). Events without a numeric bias_shift are skipped. + """ + noise: list[float] = [] + drift: list[float] = [] + for ev in events: + shift = bias_abs(ev) + if shift is None: + continue + ev_type = ev.get("type") + if ev_type == "anchor_bias_observation": + noise.append(shift) + elif ev_type == "anchor_provider_drift": + drift.append(shift) + return noise, drift + + +def percentile(sorted_vals: list[float], p: float) -> float: + """Nearest-rank percentile (0 < p <= 100) of an already-sorted list. + + Deterministic and stable for small samples (no interpolation), which + matters here: the calibration sample is often only tens of points. + """ + if not sorted_vals: + raise ValueError("percentile of an empty list") + rank = max(1, min(len(sorted_vals), math.ceil(p / 100.0 * len(sorted_vals)))) + return sorted_vals[rank - 1] + + +def recommend_threshold( + noise: list[float], drift: list[float], current: float = DEFAULT_CURRENT +) -> dict[str, Any]: + """Empirical threshold recommendation from the censored noise sample. + + noise is the sub-threshold |bias_shift| distribution (censored below + `current` by construction), drift the observed over-threshold magnitudes. + + Returns a dict with distribution stats, the recommendation and a reason + key (one of: no-sub-threshold-observations | noise-well-below-boundary | + noise-crowding-boundary | no-clean-separation). + """ + noise_sorted = sorted(noise) + drift_sorted = sorted(drift) + if not noise_sorted: + return { + "n_noise": 0, + "n_drift": len(drift_sorted), + "recommended": current, + "reason": "no-sub-threshold-observations", + } + mean = statistics.fmean(noise_sorted) + p50 = percentile(noise_sorted, 50) + p90 = percentile(noise_sorted, 90) + p95 = percentile(noise_sorted, 95) + p99 = percentile(noise_sorted, 99) + worst = noise_sorted[-1] + # The noise tail with 1.5x margin is the smallest defensible boundary — + # but never below the current value (lowering a working guard only adds + # false alarms; drift events already prove the boundary catches real + # drift). + lower = max(current, p99 * 1.5) + ceiling = 0.5 # hard cap: >50% shift on one round is a tokenizer change + if drift_sorted: + # The boundary must sit below the smallest real drift (with headroom) + # or the two populations are indistinguishable. + ceiling = min(ceiling, drift_sorted[0] * 0.75) + crowding = p99 >= 0.9 * current + if lower <= ceiling: + recommended = round(lower, 4) + reason = "noise-crowding-boundary" if crowding else "noise-well-below-boundary" + else: + recommended = current + reason = "no-clean-separation" + return { + "n_noise": len(noise_sorted), + "n_drift": len(drift_sorted), + "min_drift": drift_sorted[0] if drift_sorted else None, + "mean": mean, + "p50": p50, + "p90": p90, + "p95": p95, + "p99": p99, + "worst": worst, + "crowding": crowding, + "recommended": recommended, + "reason": reason, + } + + +def _fmt(x: float) -> str: + return f"{x:.4f}" + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + description=( + "Calibrate _SILENT_DRIFT_THRESHOLD from the sub-threshold " + "bias-shift distribution in usage-anchor.jsonl (issue #1075)." + ) + ) + ap.add_argument("--path", type=Path, default=DEFAULT_PATH, + help="usage-anchor.jsonl to analyze (default: %(default)s)") + ap.add_argument("--current", type=float, default=DEFAULT_CURRENT, + help="current _SILENT_DRIFT_THRESHOLD (default: %(default)s)") + args = ap.parse_args(argv) + + events, malformed = load_events(args.path) + noise, drift = split_events(events) + stats = recommend_threshold(noise, drift, current=args.current) + + first_ts = next((e.get("timestamp") for e in events if e.get("timestamp")), "?") + last_ts = next((e.get("timestamp") for e in reversed(events) if e.get("timestamp")), "?") + by_type: dict[str, int] = {} + for ev in events: + by_type[ev.get("type", "?")] = by_type.get(ev.get("type", "?"), 0) + 1 + + print(f"usage-anchor events: {len(events)} (malformed: {malformed})") + print(f" first: {first_ts} last: {last_ts}") + for ev_type in sorted(by_type): + print(f" {ev_type}: {by_type[ev_type]}") + if malformed: + print(f" !! {malformed} malformed line(s) skipped") + + reason = stats["reason"] + if reason == "no-sub-threshold-observations": + print( + f"\nNo sub-threshold observations accumulated yet " + f"(n_noise=0, n_drift={stats['n_drift']}). The detector writes an " + f"anchor_bias_observation per anchored round — re-run after the " + f"daemon has seen some usage. Current threshold stays " + f"{args.current}." + ) + return 0 + + print( + f"\nsub-threshold |bias_shift| distribution (n={stats['n_noise']}): " + f"mean={_fmt(stats['mean'])} p50={_fmt(stats['p50'])} " + f"p90={_fmt(stats['p90'])} p95={_fmt(stats['p95'])} " + f"p99={_fmt(stats['p99'])} max={_fmt(stats['worst'])}" + ) + if stats["n_drift"]: + print(f"real-drift events seen: {stats['n_drift']} (over-threshold sightings)") + else: + print("real-drift events seen: 0 (guard has never fired)") + + if reason == "noise-crowding-boundary": + print( + f"\nNOISE CROWDING THE BOUNDARY: p99 ({_fmt(stats['p99'])}) is within " + f"10% of the current threshold {args.current} — per-round estimate " + f"noise is close to tripping the guard. Recommended " + f"_SILENT_DRIFT_THRESHOLD: {stats['recommended']} " + f"(= max(current, p99 * 1.5))." + ) + elif reason == "noise-well-below-boundary": + print( + f"\nNoise is well below the boundary (p99={_fmt(stats['p99'])}, " + f"current={args.current}) — the guard demonstrably fires only on " + f"real drift. Recommended _SILENT_DRIFT_THRESHOLD: " + f"{stats['recommended']} (unchanged)." + ) + elif reason == "no-clean-separation": + print( + f"\nNO CLEAN SEPARATION: noise tail p99 ({_fmt(stats['p99'])}) * 1.5 " + f"exceeds the smallest observed drift " + f"({_fmt(stats['min_drift']) if stats['min_drift'] is not None else 'n/a'} " + f"* 0.75 headroom) — a single fixed threshold cannot separate the " + f"two populations. Keeping current threshold {args.current}; " + f"consider per-provider thresholds or investigating the noisy " + f"providers." + ) + else: # pragma: no cover - defensive + print(f"\nRecommendation: {stats['recommended']} (reason={reason})") + + print( + "\nTo apply: edit _SILENT_DRIFT_THRESHOLD in " + "emrg/server/daemon.py (~line 4300)." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_ci_calibrate_silent_drift_threshold.py b/tests/test_ci_calibrate_silent_drift_threshold.py new file mode 100644 index 00000000..d6bdcfa6 --- /dev/null +++ b/tests/test_ci_calibrate_silent_drift_threshold.py @@ -0,0 +1,190 @@ +"""Unit tests for scripts/calibrate_silent_drift_threshold.py — empirical +tuning of _SILENT_DRIFT_THRESHOLD from the sub-threshold bias-shift +distribution (issue #1075, Dev.to 3dn2b reidmarlow). + +The script is module-friendly; pure functions are tested here without any +live daemon state. Both positive and negative states are covered per the +evolution verification rules (#455/#461/#464 lessons): the boundary decision +is exercised on both sides (crowding vs quiet), plus the no-data and +no-separation states. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) +import calibrate_silent_drift_threshold as cal # type: ignore[import-not-found] + + +class TestPercentile: + def test_median_even_count(self) -> None: + # nearest-rank p50 of 1..4: ceil(0.5*4)=2 → index 1 → 2 + assert cal.percentile([1, 2, 3, 4], 50) == 2.0 + + def test_p100_returns_max(self) -> None: + assert cal.percentile([1, 2, 3, 4], 100) == 4.0 + + def test_p1_returns_min(self) -> None: + assert cal.percentile([1, 2, 3, 4], 1) == 1.0 + + def test_single_value_any_percentile(self) -> None: + assert cal.percentile([0.3], 99) == 0.3 + + def test_empty_raises(self) -> None: + with pytest.raises(ValueError): + cal.percentile([], 50) + + +class TestBiasAbs: + def test_positive_signed(self) -> None: + assert cal.bias_abs({"bias_shift": 0.6}) == 0.6 + + def test_negative_signed_folds(self) -> None: + assert cal.bias_abs({"bias_shift": -0.4}) == 0.4 + + def test_zero(self) -> None: + assert cal.bias_abs({"bias_shift": 0}) == 0.0 + + def test_missing_field(self) -> None: + assert cal.bias_abs({"type": "anchor_loss"}) is None + + def test_non_numeric(self) -> None: + assert cal.bias_abs({"bias_shift": "NaN"}) is None + + +class TestSplitEvents: + def _ev(self, **kw) -> dict: + return {"type": "anchor_bias_observation", "bias_shift": 0.05, **kw} + + def test_observation_goes_to_noise(self) -> None: + noise, drift = cal.split_events([self._ev(bias_shift=0.06)]) + assert noise == [0.06] and drift == [] + + def test_drift_event_goes_to_drift(self) -> None: + noise, drift = cal.split_events([ + self._ev(type="anchor_provider_drift", bias_shift=-0.8) + ]) + assert noise == [] and drift == [0.8] + + def test_unrelated_event_types_skipped(self) -> None: + noise, drift = cal.split_events([ + {"type": "anchor_loss", "est": 100}, + {"type": "anchor_drift", "delta": 50}, + ]) + assert noise == [] and drift == [] + + def test_missing_bias_shift_skipped(self) -> None: + noise, drift = cal.split_events([ + {"type": "anchor_bias_observation"}, # no bias_shift + self._ev(), + ]) + assert noise == [0.05] and drift == [] + + +class TestLoadEvents: + def test_valid_and_malformed_mixed(self, tmp_path) -> None: + f = tmp_path / "usage-anchor.jsonl" + f.write_text( + '{"type": "anchor_bias_observation", "bias_shift": 0.05}\n' + "not-json\n" + '{"type": "anchor_provider_drift", "bias_shift": 0.8}\n', + encoding="utf-8", + ) + events, malformed = cal.load_events(f) + assert len(events) == 2 and malformed == 1 + + def test_blank_lines_ignored(self, tmp_path) -> None: + f = tmp_path / "usage-anchor.jsonl" + f.write_text('{"type": "anchor_loss"}\n\n\n', encoding="utf-8") + events, malformed = cal.load_events(f) + assert len(events) == 1 and malformed == 0 + + def test_missing_file_exits(self, tmp_path) -> None: + with pytest.raises(SystemExit) as exc: + cal.load_events(tmp_path / "nope.jsonl") + assert exc.value.code == 1 + + +class TestRecommendThreshold: + def test_no_observations_keeps_current(self) -> None: + stats = cal.recommend_threshold([], [], current=0.25) + assert stats["reason"] == "no-sub-threshold-observations" + assert stats["recommended"] == 0.25 + assert stats["n_noise"] == 0 + + def test_quiet_noise_keeps_current(self) -> None: + # per-round wobble a few % — p99*1.5 stays under the current 0.25 + noise = [0.01, 0.02, 0.03, 0.04, 0.05] + stats = cal.recommend_threshold(noise, [], current=0.25) + assert stats["reason"] == "noise-well-below-boundary" + assert stats["recommended"] == 0.25 + assert stats["p99"] == pytest.approx(0.05) + assert stats["crowding"] is False + + def test_noise_crowding_boundary_raises(self) -> None: + # p99 = 0.24 is within 10% of the 0.25 boundary → recommend 0.36 + noise = [0.05, 0.1, 0.15, 0.2, 0.22, 0.24] + stats = cal.recommend_threshold(noise, [], current=0.25) + assert stats["reason"] == "noise-crowding-boundary" + assert stats["recommended"] == pytest.approx(0.36) + assert stats["crowding"] is True + + def test_no_clean_separation_keeps_current(self) -> None: + # noise tail 0.24*1.5 = 0.36 > smallest drift 0.30*0.75 = 0.225 — + # a single threshold cannot separate the populations + noise = [0.05, 0.1, 0.15, 0.2, 0.22, 0.24] + drift = [0.30] + stats = cal.recommend_threshold(noise, drift, current=0.25) + assert stats["reason"] == "no-clean-separation" + assert stats["recommended"] == 0.25 # keep, don't gamble + assert stats["min_drift"] == pytest.approx(0.30) + + def test_drift_floor_keeps_quiet_separation(self) -> None: + # noise tiny (p99=0.12), drift far away at 0.45 → unchanged, quiet + noise = [0.05, 0.08, 0.1, 0.12] + drift = [0.45] + stats = cal.recommend_threshold(noise, drift, current=0.25) + assert stats["reason"] == "noise-well-below-boundary" + assert stats["recommended"] == 0.25 + + def test_recommendation_never_lowers_below_current(self) -> None: + # even a pathological quiet sample must not lower the working guard + stats = cal.recommend_threshold([0.001, 0.002], [], current=0.25) + assert stats["recommended"] >= 0.25 + + +class TestMain: + def test_end_to_end_report(self, tmp_path, capsys) -> None: + f = tmp_path / "usage-anchor.jsonl" + rows = [ + {"type": "anchor_bias_observation", "bias_shift": 0.02}, + {"type": "anchor_bias_observation", "bias_shift": -0.04}, + {"type": "anchor_bias_observation", "bias_shift": 0.05}, + {"type": "anchor_provider_drift", "bias_shift": 0.8}, + ] + f.write_text( + "\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8" + ) + rc = cal.main(["--path", str(f)]) + assert rc == 0 + out = capsys.readouterr().out + assert "anchor_bias_observation: 3" in out + assert "anchor_provider_drift: 1" in out + assert "distribution (n=3)" in out + assert "Recommended _SILENT_DRIFT_THRESHOLD: 0.25" in out + + def test_no_data_report(self, tmp_path, capsys) -> None: + f = tmp_path / "usage-anchor.jsonl" + f.write_text( + '{"type": "anchor_loss", "est": 100}\n', encoding="utf-8" + ) + rc = cal.main(["--path", str(f)]) + assert rc == 0 + out = capsys.readouterr().out + assert "No sub-threshold observations accumulated yet" in out + assert "Current threshold stays 0.25" in out diff --git a/tests/test_daemon.py b/tests/test_daemon.py index bd0b2c29..f3c5f4a8 100644 --- a/tests/test_daemon.py +++ b/tests/test_daemon.py @@ -812,16 +812,24 @@ def test_silent_provider_drift_emits_event(tmp_path, monkeypatch, caplog): assert "usage anchor silent drift session=silent-drift" in caplog.text # Re-anchor on the new provider's real number (natural anchor overwrite) server._usage_anchors[sid] = (360_000, 150_000) - # Same provider bias again → no second event (stable after re-anchor) + # Same provider bias again → no SECOND DRIFT event (stable after + # re-anchor); issue #1075: the stable round accumulates a sub-threshold + # observation instead of vanishing. server._detect_silent_anchor_drift(_SidSession(sid), 365_000, 152_000) - assert len(stats.read_text(encoding="utf-8").strip().splitlines()) == 1 + lines = stats.read_text(encoding="utf-8").strip().splitlines() + assert len(lines) == 2 + assert json.loads(lines[1])["type"] == "anchor_bias_observation" def test_silent_drift_below_threshold_silent(tmp_path, monkeypatch, caplog): """Issue #1027 — a small per-round bias wobble (same provider, estimate - noise) must NOT emit a drift event: the threshold separates real + noise) must NOT emit a DRIFT event: the threshold separates real tokenizer changes from normal variance. Issue #1072 — the unconditional - heartbeat still proves the detector ran (no event, no warning).""" + heartbeat still proves the detector ran (no drift event, no warning). + + Issue #1075 (reidmarlow, Dev.to 3dn2b) — the sub-threshold distribution + IS the calibration data, so the within-threshold shift now accumulates as + an anchor_bias_observation event instead of vanishing.""" stats = tmp_path / "usage-anchor.jsonl" monkeypatch.setattr(daemon_mod, "_USAGE_ANCHOR_STATS_PATH", stats) server = _make_server() @@ -830,18 +838,32 @@ def test_silent_drift_below_threshold_silent(tmp_path, monkeypatch, caplog): # Same provider: real 240K vs est 150K → bias 1.6 → shift 6.7% < 25% with caplog.at_level("DEBUG", logger="emrg.server.daemon"): server._detect_silent_anchor_drift(_SidSession(sid), 240_000, 150_000) - assert not stats.exists() or stats.read_text(encoding="utf-8").strip() == "" - # Heartbeat present with "within threshold" verdict; no drift warning + lines = stats.read_text(encoding="utf-8").strip().splitlines() + assert len(lines) == 1 + ev = json.loads(lines[0]) + assert ev["type"] == "anchor_bias_observation" + assert ev["session"] == sid + assert ev["bias_shift"] == pytest.approx(0.0667, abs=0.001) + assert ev["provider"] == "localhost" + # Heartbeat present with "within threshold" verdict; no drift event/warning assert "anchor-bias-heartbeat session=no-drift" in caplog.text assert "bias_shift=0.0667" in caplog.text assert "within threshold, no drift" in caplog.text assert "usage anchor silent drift" not in caplog.text - # No anchor at all → no-op (first round, nothing to compare against) + # Negative states — no observation may be written: + # 1) No anchor at all → no-op (first round, nothing to compare against) caplog.clear() server._detect_silent_anchor_drift(_SidSession("fresh"), 10_000, 9_000) - assert not stats.exists() or stats.read_text(encoding="utf-8").strip() == "" + assert len(stats.read_text(encoding="utf-8").strip().splitlines()) == 1 assert "anchor-bias-heartbeat" not in caplog.text # no old anchor → skip + # 2) Non-positive estimate → no-op (guard: estimate <= 0) + server._detect_silent_anchor_drift(_SidSession(sid), 240_000, 0) + assert len(stats.read_text(encoding="utf-8").strip().splitlines()) == 1 + # 3) Non-positive old bias → no-op (guard: old_bias <= 0) + server._usage_anchors["bad-bias"] = (0, 148_000) + server._detect_silent_anchor_drift(_SidSession("bad-bias"), 240_000, 150_000) + assert len(stats.read_text(encoding="utf-8").strip().splitlines()) == 1 def test_usage_anchor_cross_provider_window_attributable(tmp_path, monkeypatch): From eafc71dc9c23cb4ad034d74eb325c77531b27dfe Mon Sep 17 00:00:00 2001 From: EMRG Evolution Date: Sun, 30 Aug 2026 12:18:51 +0800 Subject: [PATCH 2/2] =?UTF-8?q?emrg:=20calibration=20script=20=E2=80=94=20?= =?UTF-8?q?per-provider=20bias-shift=20breakdown=20(issue=20#1075)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Agent.md | 2 +- scripts/calibrate_silent_drift_threshold.py | 51 ++++++++++++++ ...est_ci_calibrate_silent_drift_threshold.py | 67 +++++++++++++++++-- 3 files changed, 115 insertions(+), 5 deletions(-) diff --git a/Agent.md b/Agent.md index 8945ca52..96eaef59 100644 --- a/Agent.md +++ b/Agent.md @@ -119,7 +119,7 @@ Community needs voiced in HN agent-UI discussions map directly to EMRG's design: pkill -f "emrg.server"; rm -f ~/.emrg/emrgd.token; python -m emrg ``` -Python: `uv run pytest tests/ -v` (1172) — import check: `uv run python -c "from emrg.client.app import run_client"` +Python: `uv run pytest tests/ -v` (1177) — import check: `uv run python -c "from emrg.client.app import run_client"` GUI: `cd emrg/gui && npm test` (95: 45 daemon_client + 20 conn-manager + 8 integration + 6 build-config + 7 gui-state + 3 preload-api + 4 boot-contract + 2 theme-guard) — syntax: `node --check main.js preload.js daemon_client.js` Renderer: `cd emrg/gui/renderer && npm run typecheck && npm test` (476: 5 snapshot-store + 9 utils + 3 ErrorBoundary + 2 App smoke + 11 commands + 4 copywriting + 11 i18n + 11 markdown + 16 transcript + 10 TranscriptView + 15 history + 22 composer + 31 Composer + 6 LinkDialog + 12 sidebar + 17 Sidebar + 9 fileTree + 9 FileTree + 16 resultPanel + 8 ResultPanel + 27 workspaceView + 10 WorkspaceView + 10 dialog + 6 Dialog + 9 ConfirmDialog + 9 RenameDialog + 10 dialogLists + 3 HelpDialog + 9 MemoryDialog + 6 SkillsDialog + 9 openSession + 6 WelcomeDialog + 8 OpenSessionDialog + 7 NewSessionDialog + 7 rewind + 8 RewindDialog + 7 GithubDeviceDialog + 15 daemonBridge + 7 DaemonBridgeProvider + 26 Shell + 15 DialogHost + 20 SettingsPanel + 6 TaskFormDialog + 5 RantDialog + 4 vendorMarkdown) + `npm run build` → `renderer/dist/` CI: `uv run pytest` (ubuntu + **windows-2025 matrix** — Windows pytest 回归在 PR CI 即失败，v0.2.29 教训 #725) + GUI tests + **actionlint workflow lint** (`rhysd/actionlint@v1.7.12` gate, #444 — workflow 解析错误在 PR CI 即失败，如 `if:` secrets 上下文) diff --git a/scripts/calibrate_silent_drift_threshold.py b/scripts/calibrate_silent_drift_threshold.py index f2ac3e52..a5d5aa0f 100644 --- a/scripts/calibrate_silent_drift_threshold.py +++ b/scripts/calibrate_silent_drift_threshold.py @@ -120,6 +120,35 @@ def split_events(events: list[dict]) -> tuple[list[float], list[float]]: return noise, drift +def provider_groups(events: list[dict]) -> dict[str, dict[str, list[float]]]: + """Group events by provider, split into per-provider noise/drift lists. + + A global threshold hides provider-specific noise floors: OpenAI's + tokenizer may sit at a ~1.5x bias with tiny per-round wobble while a + local endpoint wobbles 20%+ — aggregating them into one distribution can + produce a "no-clean-separation" verdict even though every provider is + cleanly separable on its own (the script's own no-clean-separation + message says "consider per-provider thresholds" — this table is the data + to do that). Events without a numeric bias_shift are skipped; events + without a provider field fall under "?". + """ + groups: dict[str, dict[str, list[float]]] = {} + for ev in events: + shift = bias_abs(ev) + if shift is None: + continue + ev_type = ev.get("type") + if ev_type not in ("anchor_bias_observation", "anchor_provider_drift"): + continue + prov = ev.get("provider") or "?" + group = groups.setdefault(prov, {"noise": [], "drift": []}) + if ev_type == "anchor_bias_observation": + group["noise"].append(shift) + else: + group["drift"].append(shift) + return groups + + def percentile(sorted_vals: list[float], p: float) -> float: """Nearest-rank percentile (0 < p <= 100) of an already-sorted list. @@ -211,6 +240,7 @@ def main(argv: list[str] | None = None) -> int: events, malformed = load_events(args.path) noise, drift = split_events(events) + groups = provider_groups(events) stats = recommend_threshold(noise, drift, current=args.current) first_ts = next((e.get("timestamp") for e in events if e.get("timestamp")), "?") @@ -248,6 +278,27 @@ def main(argv: list[str] | None = None) -> int: else: print("real-drift events seen: 0 (guard has never fired)") + if groups: + print("\nper-provider |bias_shift| (noise n, drift n, noise p90/p99):") + for prov in sorted(groups, key=lambda p: (-len(groups[p]["noise"]), p)): + group = groups[prov] + n = sorted(group["noise"]) + n_drift = len(group["drift"]) + if n: + p90 = percentile(n, 90) + p99 = percentile(n, 99) + flag = " <-- CROWDING (p99 >= 90% of current threshold)" \ + if p99 >= 0.9 * args.current else "" + print( + f" {prov}: noise n={len(n)} drift n={n_drift} " + f"p90={_fmt(p90)} p99={_fmt(p99)}{flag}" + ) + else: + print( + f" {prov}: noise n=0 drift n={n_drift} " + f"(no sub-threshold observations)" + ) + if reason == "noise-crowding-boundary": print( f"\nNOISE CROWDING THE BOUNDARY: p99 ({_fmt(stats['p99'])}) is within " diff --git a/tests/test_ci_calibrate_silent_drift_threshold.py b/tests/test_ci_calibrate_silent_drift_threshold.py index d6bdcfa6..dbc28bd1 100644 --- a/tests/test_ci_calibrate_silent_drift_threshold.py +++ b/tests/test_ci_calibrate_silent_drift_threshold.py @@ -86,6 +86,58 @@ def test_missing_bias_shift_skipped(self) -> None: assert noise == [0.05] and drift == [] +class TestProviderGroups: + def _obs(self, prov: str, shift: float) -> dict: + return { + "type": "anchor_bias_observation", + "provider": prov, + "bias_shift": shift, + } + + def _drift(self, prov: str, shift: float) -> dict: + return { + "type": "anchor_provider_drift", + "provider": prov, + "bias_shift": shift, + } + + def test_separates_providers_and_types(self) -> None: + groups = cal.provider_groups([ + self._obs("api.openai.com", 0.05), + self._obs("api.openai.com", 0.07), + self._obs("localhost", 0.2), + self._drift("localhost", 0.9), + ]) + assert groups["api.openai.com"]["noise"] == [0.05, 0.07] + assert groups["api.openai.com"]["drift"] == [] + assert groups["localhost"]["noise"] == [0.2] + assert groups["localhost"]["drift"] == [0.9] + + def test_empty_events(self) -> None: + assert cal.provider_groups([]) == {} + + def test_missing_provider_falls_to_question_mark(self) -> None: + groups = cal.provider_groups([ + {"type": "anchor_bias_observation", "bias_shift": 0.05}, + ]) + assert groups["?"]["noise"] == [0.05] + + def test_unrelated_and_biasless_events_skipped(self) -> None: + groups = cal.provider_groups([ + {"type": "anchor_loss", "provider": "x", "est": 100}, + {"type": "anchor_bias_observation", "provider": "x"}, # no bias + {"type": "anchor_bias_observation", "bias_shift": 0.03}, # no prov + ]) + assert groups == {"?": {"noise": [0.03], "drift": []}} + + def test_negative_shift_folds_per_provider(self) -> None: + groups = cal.provider_groups([ + self._obs("p", -0.06), + self._obs("p", 0.04), + ]) + assert sorted(groups["p"]["noise"]) == [0.04, 0.06] + + class TestLoadEvents: def test_valid_and_malformed_mixed(self, tmp_path) -> None: f = tmp_path / "usage-anchor.jsonl" @@ -162,10 +214,14 @@ class TestMain: def test_end_to_end_report(self, tmp_path, capsys) -> None: f = tmp_path / "usage-anchor.jsonl" rows = [ - {"type": "anchor_bias_observation", "bias_shift": 0.02}, - {"type": "anchor_bias_observation", "bias_shift": -0.04}, - {"type": "anchor_bias_observation", "bias_shift": 0.05}, - {"type": "anchor_provider_drift", "bias_shift": 0.8}, + {"type": "anchor_bias_observation", "bias_shift": 0.02, + "provider": "api.openai.com"}, + {"type": "anchor_bias_observation", "bias_shift": -0.04, + "provider": "api.openai.com"}, + {"type": "anchor_bias_observation", "bias_shift": 0.05, + "provider": "api.openai.com"}, + {"type": "anchor_provider_drift", "bias_shift": 0.8, + "provider": "localhost"}, ] f.write_text( "\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8" @@ -177,6 +233,9 @@ def test_end_to_end_report(self, tmp_path, capsys) -> None: assert "anchor_provider_drift: 1" in out assert "distribution (n=3)" in out assert "Recommended _SILENT_DRIFT_THRESHOLD: 0.25" in out + # per-provider table present with both providers, one without noise + assert "api.openai.com: noise n=3 drift n=0" in out + assert "localhost: noise n=0 drift n=1" in out def test_no_data_report(self, tmp_path, capsys) -> None: f = tmp_path / "usage-anchor.jsonl"