From 40fe66ece226ff1e50eb110a8c539fb22d656bb9 Mon Sep 17 00:00:00 2001 From: Edbert Chan Date: Tue, 8 Sep 2026 23:18:21 -0700 Subject: [PATCH] hook(wrong-check-reflect): catch a concession that the user found the miss A reply that opens by conceding and then says it is verifying now, that it had not checked, or that it should have, is the wrong-check class this detector already covers in its explicit forms. "Your instinct was right" fires on its own: conceding that the user's suspicion beat the agent's checks already means the checks missed something. Bare agreement stays silent. The reflect skill's invoke list names the same shapes, so the written trigger and the detector agree. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KU2pPKob4MJ1NqjsfTNyYJ Change-Id: I5b8f852a9ba5097517405ae1959f21378fab3cbf --- engine/hooks/wrong-check-reflect/detect.py | 11 ++++++++ .../wrong-check-reflect/tests/test_hooks.py | 25 +++++++++++++++++++ engine/skills/reflect/SKILL.md | 1 + 3 files changed, 37 insertions(+) diff --git a/engine/hooks/wrong-check-reflect/detect.py b/engine/hooks/wrong-check-reflect/detect.py index 9644470..c41aa79 100644 --- a/engine/hooks/wrong-check-reflect/detect.py +++ b/engine/hooks/wrong-check-reflect/detect.py @@ -64,6 +64,17 @@ re.compile( r"(?i)\bi\s+misread\s+(it|that|this|my\s+own|the)\b" ), + re.compile( + r"(?i)\byour\s+(?:instinct|hunch|gut|suspicion|read)\s+(?:was|were)\s+right\b" + ), + re.compile( + r"(?i)^\s*[*_#\s>-]*(?:you'?re\s+right|you\s+are\s+right|good\s+catch)\b" + r".{0,200}?(?:verifying\s+(?:it\s+|that\s+)?now|checking\s+(?:it\s+|that\s+)?now|" + r"i\s+hadn'?t\b|i\s+had\s+not\b|i\s+didn'?t\b|i\s+did\s+not\b|" + r"i\s+should\s+have\b|instead\s+of\s+(?:labeling|labelling|assuming|guessing)|" + r"i\s+never\s+(?:ran|checked|read|verified))", + re.DOTALL, + ), ] # Hypothetical / product-blame shapes that must stay silent even if a diff --git a/engine/hooks/wrong-check-reflect/tests/test_hooks.py b/engine/hooks/wrong-check-reflect/tests/test_hooks.py index 3b05883..b27d876 100644 --- a/engine/hooks/wrong-check-reflect/tests/test_hooks.py +++ b/engine/hooks/wrong-check-reflect/tests/test_hooks.py @@ -92,6 +92,31 @@ def test_hit_i_misread_without_youre_right_prefix(self): detect.find_admission("I misread the front matter on that skill.") ) + def test_hit_youre_right_verifying_it_now(self): + self.assertIsNotNone(detect.find_admission( + "You're right. Verifying it now instead of labeling it." + )) + + def test_hit_good_catch_i_should_have_checked(self): + self.assertIsNotNone(detect.find_admission( + "Good catch on the hook. I should have run the two greps before sending that." + )) + + def test_no_hit_youre_right_agreeing_with_a_choice(self): + self.assertIsNone(detect.find_admission( + "You're right that the second option is cheaper, so I will build that one." + )) + + def test_hit_your_instinct_was_right_stands_alone(self): + self.assertIsNotNone(detect.find_admission( + "Your instinct was right — the size cap was silently skipping files." + )) + + def test_hit_your_hunch_was_right(self): + self.assertIsNotNone(detect.find_admission( + "Your hunch was right, the wrapper path was never resolved." + )) + def test_no_hit_product_test_was_wrong(self): self.assertIsNone(detect.find_admission("the test was wrong")) diff --git a/engine/skills/reflect/SKILL.md b/engine/skills/reflect/SKILL.md index 9658c4c..ddc33f0 100644 --- a/engine/skills/reflect/SKILL.md +++ b/engine/skills/reflect/SKILL.md @@ -52,6 +52,7 @@ Every invocation of this skill — single-transcript or multi-conversation mode - A non-trivial workflow emerged that isn't captured anywhere. - A session, or a corpus-scan bucket, shows heavy user involvement — many corrections, clarifying answers typed out by hand, repeated manual confirmations — over a short span. That is a **FAILURE**, not a preference ping: the user had to stay in the loop because the agent missed a named constraint. Route to `automate-me` (step 4). Do not write a one-off task-skill edit and call it done. - The user said "you fucked up", "you messed up", "I told you", "you're ignoring me", or equivalent agent-blame. Treat this reflect pass as FAIL. The class is *ignored named constraint*, not the swear word. Product-blame ("the UI is messed up") is not this class. +- The reply conceded the user's suspicion — "your instinct was right", "your hunch was right", "good catch" followed by a correction. Conceding means the user found what the agent's own checks did not, so the miss is the finding, not the concession. Treat as FAIL; the `wrong-check-reflect` detector fires on these shapes. - The user had to keep iterating, restate requirements, or change product direction because the agent missed something already named. FAIL, then `automate-me`. A genuine mind-change (user learned new facts, then redirected) is not failure. A forced restatement of an already-named constraint is. - The same *type* of complaint appears in 2+ turns or 2+ sessions (repro-then-fix, UI proof before done, e2e before claiming pass, obey the named verb). That class is a bug. **Must** invoke `automate-me` — not optional, do not wait for the user to say "automate me." `token_audit.py`'s `intervention-must-automate` flag is the mechanical catch; human-message only, never tool_result / skill-injection / `/loop` polls. - It's been a while since the corpus-wide pass (`top_sessions.py` + this skill's lenses across the worst offenders) last ran. No fixed cadence and no cron — just periodically worth doing by hand.