diff --git a/engine/hooks/verdict-flip-watch/README.md b/engine/hooks/verdict-flip-watch/README.md new file mode 100644 index 00000000..da3b2366 --- /dev/null +++ b/engine/hooks/verdict-flip-watch/README.md @@ -0,0 +1,60 @@ +# verdict-flip-watch + +Third and last layer of the self-correction guard. Advisory Stop hook. + +A verifier that printed `ok` earlier in the session and failed later means any +status reported off the earlier run is stale — as a matter of record, not +interpretation. This hook reads the transcript's own Bash commands and their +`tool_result` output, pairs them, and notes the first target whose verdict +flipped from pass to fail. It stays silent when the outgoing message already +mentions it (`wrong`, `vacuous`, `stale`, `retract`, `now fails`, …). + +## Why a third layer + +The two layers above it both depend on the model: + +1. `wrong-check-reflect` matches the shape of a retraction in the outgoing + text — so it only helps once the model has decided to admit something. +2. `principle-flag-your-own-corrections` carries the judgment no regex can + enumerate — but still needs the model to notice the claim went stale. + +This hook needs neither. It catches the silent switch to the corrected value, +which is the failure `principle-flag-your-own-corrections` names: the user +cannot tell a silent correction from consistency. + +The live case it was written for: `check_skill_test_coverage.py` printed `ok` +for a stacked slice it never compared, that result was reported as "fully +green", and the same script failed once it was given the slice refs. Nothing +mechanical connected the two runs. + +## Scope + +Only verifier-shaped commands count — `check_*`, `test_*`, `run_*`, pytest, +unittest, npm/pnpm test, cargo/go test, make test/check, jest, vitest, +preflight. Tracking every `ls` would make a flip meaningless. + +`fail` wins over `pass` when both appear in one output, because a run can +print `ok` lines for early gates and still fail overall. + +Advisory on purpose: exit 0, stderr. A gate can legitimately start failing +because the turn broke it deliberately, and the hook cannot know intent. Once +per transcript per target. Fail-open on any parse or IO error. + +## Files + +- `detect.py` — command/result pairing, verdict classification, `decide()` +- `claude_stop_check.py` — Claude `Stop` entrypoint (stderr, exit 0) +- `claude.hook.json` / `install_claude_hook.py` — settings.json merge (idempotent) +- `tests/fixtures/` — transcripts that fire and stay silent +- `tests/test_hooks.py` + +## Install + +`./install.sh` from the repo root. Restart the harness. + +## Tests + +```sh +python3 -m unittest discover -s engine/hooks/verdict-flip-watch/tests -v +python3 scripts/check_hook_test_coverage.py engine/hooks/verdict-flip-watch +``` diff --git a/engine/hooks/verdict-flip-watch/claude.hook.json b/engine/hooks/verdict-flip-watch/claude.hook.json new file mode 100644 index 00000000..d26edc8d --- /dev/null +++ b/engine/hooks/verdict-flip-watch/claude.hook.json @@ -0,0 +1,16 @@ +{ + "hooks": { + "Stop": [ + { + "matcher": "*", + "hooks": [ + { + "type": "command", + "command": "python3 $HOME/.claude/hooks/verdict-flip-watch/claude_stop_check.py", + "timeout": 10 + } + ] + } + ] + } +} diff --git a/engine/hooks/verdict-flip-watch/claude_stop_check.py b/engine/hooks/verdict-flip-watch/claude_stop_check.py new file mode 100644 index 00000000..b5a8ddd6 --- /dev/null +++ b/engine/hooks/verdict-flip-watch/claude_stop_check.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Claude Stop hook: note a verifier that passed earlier and failed later. + +Advisory — stderr plus exit 0 — because a gate can legitimately start failing +when the turn broke it on purpose. Fail-open on any read/parse error. +""" +from __future__ import annotations + +import json +import sys + +from detect import decide + + +def main() -> None: + try: + payload = json.load(sys.stdin) + except (json.JSONDecodeError, OSError): + return + try: + message = decide(payload if isinstance(payload, dict) else {}) + except Exception: + return + if message: + sys.stderr.write(message + "\n") + + +if __name__ == "__main__": + main() diff --git a/engine/hooks/verdict-flip-watch/detect.py b/engine/hooks/verdict-flip-watch/detect.py new file mode 100644 index 00000000..8ad2180a --- /dev/null +++ b/engine/hooks/verdict-flip-watch/detect.py @@ -0,0 +1,209 @@ +"""Backstop layer: a check that passed earlier and failed later makes any claim +made off the earlier run stale -- whether or not the model noticed. + +Three layers guard self-correction, weakest dependency last: + +1. `wrong-check-reflect` matches the SHAPE of a retraction in the outgoing + text. It only fires once the model has already decided to admit something. +2. `principle-flag-your-own-corrections` carries the judgment no regex can + enumerate. It still requires the model to realise a claim went stale. +3. This hook requires neither. It reads the transcript's own command results. + If `scripts/x.py` printed ok at one point and failed later, the earlier + "green" was wrong as a matter of record, and the only question is whether + this message says so. + +That third property is the point: it catches the silent switch to the +corrected value, which `principle-flag-your-own-corrections` names as the real +failure -- the user cannot tell a silent correction from consistency. + +Advisory: stderr plus exit 0. A gate can legitimately start failing because +the turn broke something on purpose, so this informs rather than blocks. +Fail-open on any parse or IO error. + +VERIFIER_RE is what counts as a command worth tracking: a checker, test runner, +or build. Tracking every `ls` and `git status` would make a flip meaningless. + +ACKNOWLEDGED_RE is the outgoing message already owning the flip: a correction +marker, or the word stale/vacuous near the verdict. It reuses the +wrong-check-reflect vocabulary. +""" +from __future__ import annotations + +import hashlib +import json +import os +import re + +STATE_DIR = os.environ.get( + "VERDICT_FLIP_WATCH_STATE_DIR", + os.path.join(os.path.expanduser("~"), ".cache", "catstack-verdict-flip-watch"), +) + +VERIFIER_RE = re.compile( + r"(?:^|[\s/])(?:check_|test_|run_)|" + r"\b(?:pytest|unittest|npm\s+(?:test|run\s+\w*test\w*)|pnpm\s+(?:test|run)|" + r"cargo\s+test|go\s+test|make\s+(?:test|check)|tox|jest|vitest|preflight)\b", + re.IGNORECASE, +) +SCRIPTISH_RE = re.compile(r"[\w./-]+\.(?:py|sh|mjs|js|ts)\b") + +FAIL_RE = re.compile( + r"(?:^|\n)\s*(?:fail|FAILED|ERROR)\b|\bTraceback\b|\bFAILED\s*\(|" + r"\bexit(?:\s*code)?[\s=:]+[1-9]\b|\bassertionerror\b", + re.IGNORECASE, +) +PASS_RE = re.compile( + r"(?:^|\n)\s*ok\b|\bOK\s*$|\bpassed\b|\ball\s+\d+\s+\w+\s+(?:passed|ok)\b|" + r"\bexit(?:\s*code)?[\s=:]+0\b", + re.IGNORECASE | re.MULTILINE, +) + +ACKNOWLEDGED_RE = re.compile( + r"(?i)\b(?:wrong|incorrect|vacuous|stale|retract(?:ing|ed)?|mistaken|" + r"misread|premature|no\s+longer\s+(?:true|holds)|" + r"does(?:n'?t|\s+not)\s+hold|now\s+fails|started\s+failing|" + r"earlier\s+(?:run|pass|result|claim))\b" +) + +MESSAGE = ( + "verdict-flip-watch: `{target}` passed earlier in this session and failed " + "later, and this message does not mention it. Any status reported off the " + "earlier run is stale. Say which claim is affected and what changed, then " + "follow principle-flag-your-own-corrections (the admission is a reflect " + "trigger, not just a sentence). If the flip is expected because this turn " + "broke it on purpose, say that instead." +) + + +def _blocks(data: dict) -> list: + message = data.get("message") + content = message.get("content") if isinstance(message, dict) else None + return content if isinstance(content, list) else [] + + +def normalize_target(command: str) -> str | None: + """The thing being verified, so two runs of it can be compared.""" + if not VERIFIER_RE.search(command or ""): + return None + script = SCRIPTISH_RE.search(command) + if script: + return script.group(0) + words = (command or "").split() + return " ".join(words[:2]) if words else None + + +def classify(output: str) -> str: + """'fail' wins over 'pass': a run can print ok lines and still fail.""" + if FAIL_RE.search(output or ""): + return "fail" + if PASS_RE.search(output or ""): + return "pass" + return "unknown" + + +def result_text(data: dict) -> str: + out: list[str] = [] + for block in _blocks(data): + if not isinstance(block, dict) or block.get("type") != "tool_result": + continue + content = block.get("content") + if isinstance(content, str): + out.append(content) + elif isinstance(content, list): + for part in content: + if isinstance(part, dict) and isinstance(part.get("text"), str): + out.append(part["text"]) + return "\n".join(out) + + +def verdicts(transcript_path: str) -> list[tuple[str, str]]: + """(target, 'pass'|'fail') in transcript order. [] when unreadable.""" + try: + with open(transcript_path, encoding="utf-8") as handle: + lines = handle.readlines() + except OSError: + return [] + pending: list[str] = [] + found: list[tuple[str, str]] = [] + for raw in lines: + try: + data = json.loads(raw) + except json.JSONDecodeError: + continue + if not isinstance(data, dict): + continue + if data.get("type") == "assistant": + for block in _blocks(data): + if ( + isinstance(block, dict) + and block.get("type") == "tool_use" + and block.get("name") == "Bash" + ): + command = (block.get("input") or {}).get("command") + target = normalize_target(command) if isinstance(command, str) else None + if target: + pending.append(target) + continue + text = result_text(data) + if not text or not pending: + continue + verdict = classify(text) + target = pending.pop(0) + if verdict != "unknown": + found.append((target, verdict)) + return found + + +def find_flip(transcript_path: str) -> str | None: + """A target that passed and then later failed, else None.""" + passed: set[str] = set() + for target, verdict in verdicts(transcript_path): + if verdict == "pass": + passed.add(target) + elif verdict == "fail" and target in passed: + return target + return None + + +def _state_file(transcript_path: str, target: str) -> str: + key = f"{os.path.abspath(transcript_path or 'none')}::{target}" + digest = hashlib.sha1(key.encode()).hexdigest()[:16] + return os.path.join(STATE_DIR, f"{digest}.noted") + + +def already_noted(transcript_path: str, target: str) -> bool: + return os.path.isfile(_state_file(transcript_path, target)) + + +def mark_noted(transcript_path: str, target: str) -> None: + path = _state_file(transcript_path, target) + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + handle.write(target + "\n") + except OSError: + pass + + +def decide(payload: dict) -> str | None: + if payload.get("stop_hook_active"): + return None + message = payload.get("last_assistant_message") or "" + if ACKNOWLEDGED_RE.search(message): + return None + transcript_path = ( + payload.get("agent_transcript_path") + or payload.get("transcript_path") + or payload.get("transcriptPath") + or "" + ) + if not transcript_path: + return None + try: + target = find_flip(transcript_path) + except Exception: + return None + if not target or already_noted(transcript_path, target): + return None + mark_noted(transcript_path, target) + return MESSAGE.format(target=target) diff --git a/engine/hooks/verdict-flip-watch/install_claude_hook.py b/engine/hooks/verdict-flip-watch/install_claude_hook.py new file mode 100644 index 00000000..585df1fb --- /dev/null +++ b/engine/hooks/verdict-flip-watch/install_claude_hook.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Merge verdict-flip-watch into ~/.claude/settings.json Stop hooks. Idempotent.""" +from __future__ import annotations + +import json +import os + +HERE = os.path.dirname(os.path.abspath(__file__)) +SETTINGS_PATH = os.path.expanduser("~/.claude/settings.json") +FRAGMENT_PATH = os.path.join(HERE, "claude.hook.json") +MARKER = "verdict-flip-watch/claude_stop_check.py" +EVENT = "Stop" + + +def _is_ours(entry: dict) -> bool: + return any(MARKER in h.get("command", "") for h in entry.get("hooks", [])) + + +def merge_hook(settings: dict, fragment: dict) -> bool: + entry_list = settings.setdefault("hooks", {}).setdefault(EVENT, []) + new_entries = fragment.get("hooks", {}).get(EVENT, []) + before = json.dumps(entry_list, sort_keys=True) + kept = [e for e in entry_list if not _is_ours(e)] + entry_list[:] = kept + new_entries + return json.dumps(entry_list, sort_keys=True) != before + + +def main() -> None: + settings: dict = {} + if os.path.exists(SETTINGS_PATH): + with open(SETTINGS_PATH) as handle: + settings = json.load(handle) + with open(FRAGMENT_PATH) as handle: + fragment = json.load(handle) + if not merge_hook(settings, fragment): + print("ok claude Stop verdict-flip-watch already up to date") + return + os.makedirs(os.path.dirname(SETTINGS_PATH), exist_ok=True) + with open(SETTINGS_PATH, "w") as handle: + json.dump(settings, handle, indent=2) + handle.write("\n") + print("added claude Stop verdict-flip-watch") + + +if __name__ == "__main__": + main() diff --git a/engine/hooks/verdict-flip-watch/tests/fixtures/fixed.jsonl b/engine/hooks/verdict-flip-watch/tests/fixtures/fixed.jsonl new file mode 100644 index 00000000..8e515b2f --- /dev/null +++ b/engine/hooks/verdict-flip-watch/tests/fixtures/fixed.jsonl @@ -0,0 +1,5 @@ +{"type": "user", "message": {"role": "user", "content": "fix the gate"}} +{"type": "assistant", "message": {"role": "assistant", "content": [{"type": "tool_use", "name": "Bash", "input": {"command": "python3 -m unittest discover -s engine/hooks/x/tests"}}]}} +{"type": "user", "message": {"role": "user", "content": [{"type": "tool_result", "content": "FAILED (failures=1)"}]}} +{"type": "assistant", "message": {"role": "assistant", "content": [{"type": "tool_use", "name": "Bash", "input": {"command": "python3 -m unittest discover -s engine/hooks/x/tests"}}]}} +{"type": "user", "message": {"role": "user", "content": [{"type": "tool_result", "content": "Ran 12 tests in 0.2s\n\nOK"}]}} diff --git a/engine/hooks/verdict-flip-watch/tests/fixtures/flip.jsonl b/engine/hooks/verdict-flip-watch/tests/fixtures/flip.jsonl new file mode 100644 index 00000000..d7d24068 --- /dev/null +++ b/engine/hooks/verdict-flip-watch/tests/fixtures/flip.jsonl @@ -0,0 +1,6 @@ +{"type": "user", "message": {"role": "user", "content": "verify the stacked slice"}} +{"type": "assistant", "message": {"role": "assistant", "content": [{"type": "tool_use", "name": "Bash", "input": {"command": "python3 scripts/check_skill_test_coverage.py"}}]}} +{"type": "user", "message": {"role": "user", "content": [{"type": "tool_result", "content": "ok skill test coverage"}]}} +{"type": "user", "message": {"role": "user", "content": "run it against the slice refs"}} +{"type": "assistant", "message": {"role": "assistant", "content": [{"type": "tool_use", "name": "Bash", "input": {"command": "python3 scripts/check_skill_test_coverage.py --base skill-scenarios --head HEAD"}}]}} +{"type": "user", "message": {"role": "user", "content": [{"type": "tool_result", "content": "fail engine/skills/make-pr: changed without a corresponding test change"}]}} diff --git a/engine/hooks/verdict-flip-watch/tests/fixtures/noise.jsonl b/engine/hooks/verdict-flip-watch/tests/fixtures/noise.jsonl new file mode 100644 index 00000000..5e956983 --- /dev/null +++ b/engine/hooks/verdict-flip-watch/tests/fixtures/noise.jsonl @@ -0,0 +1,5 @@ +{"type": "user", "message": {"role": "user", "content": "look around"}} +{"type": "assistant", "message": {"role": "assistant", "content": [{"type": "tool_use", "name": "Bash", "input": {"command": "ls engine/hooks"}}]}} +{"type": "user", "message": {"role": "user", "content": [{"type": "tool_result", "content": "ok"}]}} +{"type": "assistant", "message": {"role": "assistant", "content": [{"type": "tool_use", "name": "Bash", "input": {"command": "ls engine/hooks/missing"}}]}} +{"type": "user", "message": {"role": "user", "content": [{"type": "tool_result", "content": "ls: cannot access: No such file or directory\nexit=2"}]}} diff --git a/engine/hooks/verdict-flip-watch/tests/fixtures/stable.jsonl b/engine/hooks/verdict-flip-watch/tests/fixtures/stable.jsonl new file mode 100644 index 00000000..e2d4d5d0 --- /dev/null +++ b/engine/hooks/verdict-flip-watch/tests/fixtures/stable.jsonl @@ -0,0 +1,5 @@ +{"type": "user", "message": {"role": "user", "content": "verify the slice"}} +{"type": "assistant", "message": {"role": "assistant", "content": [{"type": "tool_use", "name": "Bash", "input": {"command": "python3 scripts/check_skill_test_coverage.py --base a --head b"}}]}} +{"type": "user", "message": {"role": "user", "content": [{"type": "tool_result", "content": "ok skill test coverage"}]}} +{"type": "assistant", "message": {"role": "assistant", "content": [{"type": "tool_use", "name": "Bash", "input": {"command": "python3 scripts/check_skill_test_coverage.py --base a --head b"}}]}} +{"type": "user", "message": {"role": "user", "content": [{"type": "tool_result", "content": "ok skill test coverage"}]}} diff --git a/engine/hooks/verdict-flip-watch/tests/test_hooks.py b/engine/hooks/verdict-flip-watch/tests/test_hooks.py new file mode 100644 index 00000000..0a337958 --- /dev/null +++ b/engine/hooks/verdict-flip-watch/tests/test_hooks.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Tests for verdict-flip-watch. + +The `flip` fixture is the real case: check_skill_test_coverage.py printed ok +with its default scope, that was reported as "fully green", and the same script +failed once it was given the slice refs. Nothing mechanical connected the two +runs, so a stale claim stood. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +HOOK_DIR = os.path.dirname(HERE) +sys.path.insert(0, HOOK_DIR) +import detect # noqa: E402 + +FIXTURES = os.path.join(HERE, "fixtures") +CLEAN_REPLY = "Opened the PR; here is the link." + + +def fixture(name: str) -> str: + return os.path.join(FIXTURES, f"{name}.jsonl") + + +def payload(name: str, reply: str = CLEAN_REPLY) -> dict: + return {"last_assistant_message": reply, "transcript_path": fixture(name)} + + +class IsolatedState(unittest.TestCase): + """Each test gets its own state dir so once-per-target does not leak.""" + + def setUp(self): + self._tmp = tempfile.mkdtemp() + self._prev = detect.STATE_DIR + detect.STATE_DIR = self._tmp + + def tearDown(self): + detect.STATE_DIR = self._prev + + +class TestFindFlip(IsolatedState): + def test_fires_when_a_verifier_passed_then_failed(self): + self.assertEqual( + detect.find_flip(fixture("flip")), + "scripts/check_skill_test_coverage.py", + ) + + def test_silent_when_the_same_verifier_stays_green(self): + self.assertIsNone(detect.find_flip(fixture("stable"))) + + def test_silent_on_red_then_green_because_that_is_a_fix(self): + self.assertIsNone(detect.find_flip(fixture("fixed"))) + + def test_silent_when_the_flipping_command_is_not_a_verifier(self): + """An `ls` that succeeds then fails is not a stale verdict.""" + self.assertIsNone(detect.find_flip(fixture("noise"))) + + def test_silent_on_unreadable_transcript(self): + self.assertIsNone(detect.find_flip("/nonexistent/transcript.jsonl")) + + +class TestClassify(unittest.TestCase): + def test_fail_wins_over_pass_in_mixed_output(self): + mixed = "ok ecosystem boundaries\nfail engine/skills/make-pr: no test change" + self.assertEqual(detect.classify(mixed), "fail") + + def test_plain_ok_is_a_pass(self): + self.assertEqual(detect.classify("ok skill test coverage"), "pass") + + def test_unittest_ok_is_a_pass(self): + self.assertEqual(detect.classify("Ran 14 tests in 0.2s\n\nOK"), "pass") + + def test_traceback_is_a_fail(self): + self.assertEqual(detect.classify("Traceback (most recent call last):"), "fail") + + def test_unrecognised_output_is_unknown(self): + self.assertEqual(detect.classify("building..."), "unknown") + + +class TestNormalizeTarget(unittest.TestCase): + def test_script_path_is_the_target_regardless_of_flags(self): + self.assertEqual( + detect.normalize_target("python3 scripts/check_x.py --base a --head b"), + "scripts/check_x.py", + ) + + def test_runner_without_a_script_falls_back_to_two_words(self): + self.assertEqual(detect.normalize_target("npm test --silent"), "npm test") + + def test_non_verifier_command_is_not_tracked(self): + self.assertIsNone(detect.normalize_target("git status --short")) + + +class TestDecide(IsolatedState): + def test_blocks_once_then_stays_silent_for_the_same_target(self): + first = detect.decide(payload("flip")) + self.assertIsNotNone(first) + self.assertIn("check_skill_test_coverage.py", first) + self.assertIsNone(detect.decide(payload("flip"))) + + def test_silent_when_the_reply_already_owns_the_flip(self): + reply = "Correction: that earlier green was vacuous — the gate never compared the slice." + self.assertIsNone(detect.decide(payload("flip", reply))) + + def test_silent_when_stop_hook_active(self): + data = payload("flip") + data["stop_hook_active"] = True + self.assertIsNone(detect.decide(data)) + + def test_silent_without_a_transcript_path(self): + self.assertIsNone(detect.decide({"last_assistant_message": CLEAN_REPLY})) + + +class TestHarnessWrapper(IsolatedState): + def _run(self, data: dict): + env = dict(os.environ, VERDICT_FLIP_WATCH_STATE_DIR=detect.STATE_DIR) + return subprocess.run( + [sys.executable, os.path.join(HOOK_DIR, "claude_stop_check.py")], + input=json.dumps(data), capture_output=True, text=True, env=env, + ) + + def test_advisory_exits_zero_and_writes_to_stderr(self): + res = self._run(payload("flip")) + self.assertEqual(res.returncode, 0, res.stderr) + self.assertIn("verdict-flip-watch", res.stderr) + + def test_clean_transcript_is_quiet(self): + res = self._run(payload("stable")) + self.assertEqual(res.returncode, 0) + self.assertEqual(res.stderr.strip(), "") + + def test_malformed_stdin_fails_open(self): + res = subprocess.run( + [sys.executable, os.path.join(HOOK_DIR, "claude_stop_check.py")], + input="not json", capture_output=True, text=True, + ) + self.assertEqual(res.returncode, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/install.sh b/install.sh index b0522460..5ef60408 100755 --- a/install.sh +++ b/install.sh @@ -243,6 +243,7 @@ echo "--- claude hooks: wait / hedge / callout stack ---" link_item "wait-needs-wakeup" "$REPO_DIR/engine/hooks/wait-needs-wakeup" "$HOME/.claude/hooks/wait-needs-wakeup" link_item "hedge-runs-prove-it" "$REPO_DIR/engine/hooks/hedge-runs-prove-it" "$HOME/.claude/hooks/hedge-runs-prove-it" link_item "incidence-needs-repetition" "$REPO_DIR/engine/hooks/incidence-needs-repetition" "$HOME/.claude/hooks/incidence-needs-repetition" +link_item "verdict-flip-watch" "$REPO_DIR/engine/hooks/verdict-flip-watch" "$HOME/.claude/hooks/verdict-flip-watch" link_item "new-file-callout" "$REPO_DIR/engine/hooks/new-file-callout" "$HOME/.claude/hooks/new-file-callout" link_item "agent-relay-attribution" "$REPO_DIR/engine/hooks/agent-relay-attribution" "$HOME/.claude/hooks/agent-relay-attribution" link_item "scratchpad-collision" "$REPO_DIR/engine/hooks/scratchpad-collision" "$HOME/.claude/hooks/scratchpad-collision" @@ -350,6 +351,7 @@ echo "--- claude settings: wait / hedge / callout stack ---" python3 "$REPO_DIR/engine/hooks/wait-needs-wakeup/install_claude_hook.py" python3 "$REPO_DIR/engine/hooks/hedge-runs-prove-it/install_claude_hook.py" python3 "$REPO_DIR/engine/hooks/incidence-needs-repetition/install_claude_hook.py" +python3 "$REPO_DIR/engine/hooks/verdict-flip-watch/install_claude_hook.py" python3 "$REPO_DIR/engine/hooks/new-file-callout/install_claude_hook.py" python3 "$REPO_DIR/engine/hooks/agent-relay-attribution/install_claude_hook.py" python3 "$REPO_DIR/engine/hooks/scratchpad-collision/install_claude_hook.py" diff --git a/scripts/run_skill_scenarios.py b/scripts/run_skill_scenarios.py index 2d3c4b01..1b6d97d1 100755 --- a/scripts/run_skill_scenarios.py +++ b/scripts/run_skill_scenarios.py @@ -67,11 +67,29 @@ def bash_line(command: str) -> dict: } +def result_line(text: str) -> dict: + return { + "type": "user", + "message": {"role": "user", "content": [{"type": "tool_result", "content": text}]}, + } + + def transcript_for(scenario: dict) -> list[dict]: - """prior entries, then the user's turn, then any tool calls made in it.""" + """prior entries, then the user's turn, then any tool calls made in it. + + A `ran` entry is either a command string, or {"cmd": ..., "output": ...} + when a hook needs the command's real result -- verdict-flip-watch compares + two runs of the same verifier, so it cannot work from commands alone. + """ lines = list(scenario.get("prior") or []) lines.append(user_line(scenario.get("user") or "do the thing")) - lines.extend(bash_line(c) for c in scenario.get("ran") or []) + for entry in scenario.get("ran") or []: + if isinstance(entry, dict): + lines.append(bash_line(entry["cmd"])) + if entry.get("output") is not None: + lines.append(result_line(entry["output"])) + else: + lines.append(bash_line(entry)) return lines diff --git a/tests/scenarios/verdict-flip.json b/tests/scenarios/verdict-flip.json new file mode 100644 index 00000000..98d11c29 --- /dev/null +++ b/tests/scenarios/verdict-flip.json @@ -0,0 +1,36 @@ +[ + { + "name": "stale-green-caught-without-any-admission", + "situation": "The backstop layer. The same gate printed ok with its default scope and failed once given the slice refs, and the outgoing message reports the new result without saying the earlier one was wrong. Neither the regex layer nor the model noticing is involved — the transcript alone proves the earlier green is stale.", + "user": "verify the stacked slice", + "ran": [ + {"cmd": "python3 scripts/check_skill_test_coverage.py", "output": "ok skill test coverage"}, + {"cmd": "python3 scripts/check_skill_test_coverage.py --base skill-scenarios --head HEAD", "output": "fail engine/skills/make-pr: changed without a corresponding test change"} + ], + "reply": "Coverage for the slice needs a colocated test under engine/skills/make-pr/tests/. Adding one now.", + "expect_fire": ["verdict-flip-watch"], + "expect_silent": ["wrong-check-reflect"] + }, + { + "name": "stale-green-already-owned-by-the-reply", + "situation": "Same transcript, but the reply names the earlier result as vacuous. The backstop stays quiet because the correction is already made.", + "user": "verify the stacked slice", + "ran": [ + {"cmd": "python3 scripts/check_skill_test_coverage.py", "output": "ok skill test coverage"}, + {"cmd": "python3 scripts/check_skill_test_coverage.py --base skill-scenarios --head HEAD", "output": "fail engine/skills/make-pr: changed without a corresponding test change"} + ], + "reply": "The earlier green was vacuous — that run compared against origin/main, not the slice. With the slice refs it fails, so the coverage claim I gave you was wrong.", + "expect_silent": ["verdict-flip-watch"] + }, + { + "name": "red-then-green-is-a-fix-not-a-stale-claim", + "situation": "A verifier that failed and then passed is ordinary repair work. Firing here would punish fixing things.", + "user": "fix the failing hook test", + "ran": [ + {"cmd": "python3 -m unittest discover -s engine/hooks/verdict-flip-watch/tests", "output": "FAILED (failures=1)"}, + {"cmd": "python3 -m unittest discover -s engine/hooks/verdict-flip-watch/tests", "output": "Ran 20 tests in 0.18s\n\nOK"} + ], + "reply": "Fixed — the classifier was treating mixed output as a pass. 20 tests pass now.", + "expect_silent": ["verdict-flip-watch"] + } +]