diff --git a/engine/skills/reflect/references/cost-audit.md b/engine/skills/reflect/references/cost-audit.md index 4b6c48d9..b978339d 100644 --- a/engine/skills/reflect/references/cost-audit.md +++ b/engine/skills/reflect/references/cost-audit.md @@ -26,7 +26,7 @@ It reports, per session: total tokens by category and cache-read share, turns wh ## Frustration signals -Both `claude` and `omp` modes also emit `frustration-signals` — the mechanical feed for the Frustration lens (see lenses.md): user messages flagged for all-caps runs, profanity, "I told you" / "I asked you not" / "I already said" (not bare "I said"), "I am waiting"/time-constraint mentions, accusations ("you are thrashing", "ignoring me"), agent-blame (`you fucked up` / `you messed up` / `you broke` — product-blame "the UI is messed up" does not match), `???`, and verbatim repeats within 10 minutes (the strongest single signal), plus an interruption count (claude: `[Request interrupted by user` markers; omp: `interrupted-thinking` events and "Skipped due to queued user message" tool results). Human messages only — claude-mode system-injected user turns (``, ``, continuation summaries, skill injections) are excluded; counting them poisons the stats, found on a real 124MB transcript where task-notifications echoing the word "thrashing" inflated the count from 139 to 151. Backtested against the session that motivated it: 13/58 messages flagged, matching the hand audit. +Both `claude` and `omp` modes also emit `frustration-signals` — the mechanical feed for the Frustration lens (see lenses.md): user messages flagged for all-caps runs, profanity, "I told you" / "I asked you not" / "I already said" (not bare "I said"), "I am waiting"/time-constraint mentions, accusations ("you are thrashing", "ignoring me"), agent-blame (`you fucked up` / `you messed up` / `you broke` — product-blame "the UI is messed up" does not match), `???`, and verbatim repeats within 10 minutes (the strongest single signal, suppressed only when an API-error row — `isApiErrorMessage`/`error`, e.g. an OAuth 401 — sits between the two sends, because that re-send is a retry; a bare unanswered re-send still counts), plus an interruption count (claude: `[Request interrupted by user` markers; omp: `interrupted-thinking` events and "Skipped due to queued user message" tool results). Human messages only — claude-mode system-injected user turns (``, ``, continuation summaries, skill injections) are excluded; counting them poisons the stats, found on a real 124MB transcript where task-notifications echoing the word "thrashing" inflated the count from 139 to 151. Backtested against the session that motivated it: 13/58 messages flagged, matching the hand audit. They also emit `intervention-must-automate`: yes when a verbatim re-send fired, any intervention kind (`told-you`, `accusation`, `agent-blame`) appears ≥2 times, or ≥2 distinct intervention kinds appear in the session. One "I told you" is frustration only; the same class twice is FAIL and must route to `automate-me`. `/loop` polls and Stop-hook injection text are not the human complaining. diff --git a/engine/skills/reflect/scripts/tests/test_token_audit.py b/engine/skills/reflect/scripts/tests/test_token_audit.py index d4281344..50d20b5d 100644 --- a/engine/skills/reflect/scripts/tests/test_token_audit.py +++ b/engine/skills/reflect/scripts/tests/test_token_audit.py @@ -1151,6 +1151,68 @@ def test_repeat_outside_ten_minute_window_not_flagged(self): finally: os.unlink(path) + def test_verbatim_repeat_suppressed_only_by_a_real_api_error_between_sends(self): + """Real session: the prompt hit an OAuth 401, so no response was ever + generated, the user ran /login and re-sent the same prompt. Claude + records that failure as its own assistant row carrying + isApiErrorMessage + error, so suppression keys off that row. A bare + re-send with no such row is still a verbatim-repeat -- an unanswered + restatement is the frustration signal itself, not evidence of an + outage.""" + usage = {"input_tokens": 0, "output_tokens": 0, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0} + prompt = "please search for auth tokens in the codebase" + api_error = { + "type": "assistant", + "timestamp": "2026-09-01T10:01:00Z", + "error": "authentication_failed", + "isApiErrorMessage": True, + "message": {"id": "err1", "model": "", "role": "assistant", "usage": usage, + "content": [{"type": "text", "text": "Login expired \u00b7 Please run /login"}]}, + } + + def kinds_for(lines): + path = write_jsonl(lines) + try: + with redirect_stdout(io.StringIO()): + result = token_audit.audit_claude(path) + return {k for f in result["frustration"]["flagged"] for k in f["kinds"]} + finally: + os.unlink(path) + + after_api_error = kinds_for([ + claude_user_text_line(prompt, ts="2026-09-01T10:00:00Z"), + api_error, + claude_user_text_line(prompt, ts="2026-09-01T10:02:00Z"), + claude_assistant_line("m1", "u1", [{"type": "text", "text": "ok"}], usage), + ]) + self.assertNotIn("verbatim-repeat", after_api_error) + + after_a_real_answer = kinds_for([ + claude_user_text_line(prompt, ts="2026-09-01T10:00:00Z"), + claude_assistant_line("m1", "u1", [{"type": "text", "text": "searching now"}], usage), + claude_user_text_line(prompt, ts="2026-09-01T10:02:00Z"), + claude_assistant_line("m2", "u2", [{"type": "text", "text": "ok done"}], usage), + ]) + self.assertIn("verbatim-repeat", after_a_real_answer) + + unanswered_resend = kinds_for([ + claude_user_text_line(prompt, ts="2026-09-01T10:00:00Z"), + claude_user_text_line(prompt, ts="2026-09-01T10:02:00Z"), + claude_assistant_line("m1", "u1", [{"type": "text", "text": "ok"}], usage), + ]) + self.assertIn("verbatim-repeat", unanswered_resend) + + def test_verbatim_repeat_still_works_for_non_claude_modes(self): + """Codex/OMP/Cursor don't pass failed_turn_indices, so + verbatim-repeat detection must still work (no suppression).""" + msgs = [ + (0, "2026-09-01T10:00:00Z", "please fix the build"), + (1, "2026-09-01T10:02:00Z", "please fix the build"), + ] + result = token_audit.frustration_signals(msgs, interruptions=0) + kinds = {k for f in result["flagged"] for k in f["kinds"]} + self.assertIn("verbatim-repeat", kinds) + def test_agent_blame_and_same_type_must_automate(self): """You-messed-up is agent-blame; product 'the ui is messed up' is not. Two told-yous fire intervention-must-automate; one told-you does not.""" @@ -1495,6 +1557,27 @@ def test_thrash_inside_subagents_is_reported(self): self.assertEqual(flags["subagent-thrash"]["value"], "yes") self.assertEqual(flags["subagent-thrash"]["count"], 1) + def test_subagent_thrash_detail_includes_files_and_failure_signatures(self): + """B2: the --out JSON report must include which files were redundantly + read and what the recurring failure error messages were, not just + counts and flag names.""" + u = {"input_tokens": 1, "output_tokens": 1, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0} + agent_c = os.path.join(self.subagents_dir, "agent-c.jsonl") + self._write(agent_c, [ + _sidechain_user_line("run the tests", "c"), + claude_assistant_line("c1", "cu1", [{"type": "tool_use", "id": "tc1", "name": "Bash", "input": {"command": "pytest"}}], u), + claude_error_line("tc1", "ModuleNotFoundError: No module named 'foo'"), + claude_assistant_line("c2", "cu2", [{"type": "tool_use", "id": "tc2", "name": "Bash", "input": {"command": "pytest"}}], u), + claude_error_line("tc2", "ModuleNotFoundError: No module named 'foo'"), + ]) + res, _ = self._audit() + thrash = res["subagents"]["thrash"] + self.assertIn("agent-a.jsonl:/x.py", thrash["redundant_read_files"]) + self.assertTrue(any( + d["agent"] == "agent-c.jsonl" and d["tool"] == "Bash" + for d in thrash["recurring_failure_details"] + )) + def test_subagent_first_user_turn_is_never_a_human_intervention(self): res, _ = self._audit() self.assertEqual(res["frustration"]["n_user_messages"], 1) diff --git a/engine/skills/reflect/scripts/token_audit.py b/engine/skills/reflect/scripts/token_audit.py index c0530844..2599ed65 100644 --- a/engine/skills/reflect/scripts/token_audit.py +++ b/engine/skills/reflect/scripts/token_audit.py @@ -47,7 +47,7 @@ running an audit against a remote host is a separate, explicitly-confirmed step outside this script. """ -import json, sys, hashlib, os, re, importlib.util +import bisect, json, sys, hashlib, os, re, importlib.util from datetime import datetime from collections import Counter @@ -195,16 +195,43 @@ def _is_allcaps(text): return sum(c.isupper() for c in letters) / len(letters) > 0.6 -def frustration_signals(user_msgs, interruptions=0): +def _is_api_error_line(row): + """A turn that never produced a response. Claude writes these as an + assistant row carrying isApiErrorMessage plus an `error` code (observed + shape: model "", zero usage, text "Login expired - Please run + /login"), which is why an ordinary assistant row cannot stand in for it.""" + if not isinstance(row, dict) or row.get("type") != "assistant": + return False + return bool(row.get("isApiErrorMessage") or row.get("error")) + + +def _has_index_between(sorted_indices, prev_idx, curr_idx): + """True if any element of sorted_indices falls strictly between prev_idx + and curr_idx (exclusive on both ends). bisect keeps this O(log n) on + large sessions.""" + lo = bisect.bisect_right(sorted_indices, prev_idx) + return lo < len(sorted_indices) and sorted_indices[lo] < curr_idx + + +def frustration_signals(user_msgs, interruptions=0, failed_turn_indices=None): """user_msgs: [(ordinal, iso_timestamp_or_None, text)] — HUMAN-authored messages only (never tool_results, never interruption markers). Returns flagged messages with their signal kinds, plus a verbatim-repeat check: the same normalized text re-sent within 10 minutes is the single strongest frustration signal (the user re-sent it because nothing visibly changed). `interruptions` is counted by the caller (tool-specific shape). + + `failed_turn_indices` (optional): JSONL line indices carrying positive + evidence that a turn never produced a response — an API-error line + (OAuth 401, rate limit, network). When one of those sits between two + identical sends, the re-send is a retry, not frustration, and + verbatim-repeat is suppressed. Absence of an assistant reply is NOT + such evidence: an unanswered restatement is the frustration signal + itself, and Claude writes auth failures as their own assistant line. """ flagged = [] - seen = [] # (epoch_seconds_or_None, normalized_text) + seen = [] + _failed = sorted(failed_turn_indices) if failed_turn_indices else None for idx, ts, text in user_msgs: t = (text or "").strip() if not t: @@ -218,11 +245,13 @@ def frustration_signals(user_msgs, interruptions=0): secs = _ts_seconds(ts) norm = re.sub(r"\s+", " ", t).casefold() if len(norm) >= 12: - for prev_secs, prev_norm in seen: + for prev_secs, prev_norm, prev_idx in seen: if prev_norm == norm and (secs is None or prev_secs is None or 0 <= secs - prev_secs <= 600): + if _failed is not None and _has_index_between(_failed, prev_idx, idx): + continue kinds.append("verbatim-repeat") break - seen.append((secs, norm)) + seen.append((secs, norm, idx)) if kinds: flagged.append({"index": idx, "ts": ts, "kinds": sorted(set(kinds)), "excerpt": t[:100]}) peak = None @@ -400,8 +429,10 @@ def audit_subagents(path): totals = {"input": 0, "output": 0, "cache_read": 0, "cache_creation": 0, "total": 0} thrash = { "redundant_reads": 0, + "redundant_read_files": [], "tool_errors": 0, "recurring_failure_signatures": 0, + "recurring_failure_details": [], "longest_edit_streak_no_verify": 0, "self_retraction": 0, "by_agent": {}, @@ -423,8 +454,15 @@ def audit_subagents(path): if fired: thrash["by_agent"][fname] = fired thrash["redundant_reads"] += flags["redundant-reads"]["count"] + for fp in stats.get("redundant_read_files", []): + thrash["redundant_read_files"].append(f"{fname}:{fp}") thrash["tool_errors"] += stats["n_errors"] thrash["recurring_failure_signatures"] += stats["n_recurring_failures"] + for detail in stats.get("recurring_failure_details", []): + thrash["recurring_failure_details"].append( + {"agent": fname, "tool": detail["tool"], "signature": detail["signature"], + "occurrences": detail["occurrences"]} + ) thrash["longest_edit_streak_no_verify"] = max( thrash["longest_edit_streak_no_verify"], stats["longest_edit_streak_no_verify"] ) @@ -493,16 +531,18 @@ def audit_claude(path, out_path=None, include_subagents=True): # different lines of the same message — found by e2e sample fixtures. msg_tool_names = {} # mid -> [tool name, ...] msg_output_tokens = {} # mid -> output_tokens (from first line of that message) + _human_utterances = transcript_provenance.direct_human_utterances(path, "claude") user_msgs = [ - (index, utterance.timestamp, utterance.text) - for index, utterance in enumerate( - transcript_provenance.direct_human_utterances(path, "claude") - ) + (utterance.index, utterance.timestamp, utterance.text) + for utterance in _human_utterances ] - n_interruptions = 0 # "[Request interrupted by user" markers - assistant_texts = [] # text blocks for self-retraction scan + n_interruptions = 0 + assistant_texts = [] + failed_turn_indices = [] - for d in lines: + for line_idx, d in enumerate(lines): + if _is_api_error_line(d): + failed_turn_indices.append(line_idx) if d.get("type") == "assistant": msg = d.get("message", {}) mid = msg.get("id") @@ -638,7 +678,9 @@ def audit_claude(path, out_path=None, include_subagents=True): seen.add(c) spikes.append((s, c, r)) - frustration = frustration_signals(user_msgs, n_interruptions) + frustration = frustration_signals( + user_msgs, n_interruptions, failed_turn_indices=failed_turn_indices + ) flags = [ _flag( @@ -695,6 +737,12 @@ def audit_claude(path, out_path=None, include_subagents=True): flags.append(_subagent_thrash_flag(subagents)) combined_total = grand + (subagents["totals"]["total"] if subagents else 0) + redundant_read_files = sorted({fp for fp, _, _, _ in redundant}) + recurring_failure_details = [ + {"tool": name, "signature": norm, "occurrences": len(seqs)} + for (name, norm), seqs in sorted(recurring.items(), key=lambda x: -len(x[1])) + ] + result = { "input": total_input, "output": total_output, @@ -712,6 +760,8 @@ def audit_claude(path, out_path=None, include_subagents=True): "self_retraction": retraction_hits, "subagents": subagents, "combined_total": combined_total, + "redundant_read_files": redundant_read_files, + "recurring_failure_details": recurring_failure_details, } if out_path: