diff --git a/docs/plans/scripts/agent-minutes-manifest.json b/docs/plans/scripts/agent-minutes-manifest.json new file mode 100644 index 00000000000..77b237a8573 --- /dev/null +++ b/docs/plans/scripts/agent-minutes-manifest.json @@ -0,0 +1,49 @@ +{ + "_comment": "Workflow ids per wave for agent-minutes.py. Labels are best-effort from the prompts; the ids are the ground truth. Transcripts live only on the orchestrator machine (workflows_dir) and are never committed. expected_agent_minutes are the totals published in 2026-09-05-zs-throughput-plan.md; the regression check (--check) must reproduce them within the tolerance.", + "workflows_dir": "~/.claude/projects/-Users-zero-suminc-/a0fab389-7ac7-4f57-88e6-599d452e8e3a/subagents/workflows", + "waves": { + "wave-1": { + "description": "Seven tickets under the original loop, one workflow, 2026-09-04T08:35Z to 22:09Z.", + "workflows": { + "wf_d039d459-774": "wave 1, seven tickets (T2, T5, T8, T11 and others)" + }, + "expected_agent_minutes": 2588, + "tolerance": 2 + }, + "waves-2-3": { + "description": "Four code tickets and three memos under the revised loop, 2026-09-04T22:55Z to 2026-09-05T10:09Z.", + "workflows": { + "wf_4991aff2-011": "T3 port/4316 build", + "wf_f8590a1f-179": "T6-config", + "wf_b85b65b5-727": "T9 PDF export", + "wf_231309b6-0bd": "T7a registry core", + "wf_a2d9707f-d29": "T3 Sol-finding verifiers (driver-added)", + "wf_72bf0947-3d5": "T3 port check and fix", + "wf_ab7d7b5c-0a4": "T12a memo", + "wf_a6a94c02-398": "T7a round 3", + "wf_5ebaa29c-46d": "shared round 3 (T9 and T7a) and T12a", + "wf_d9ff25f2-d0d": "T12a round" + }, + "excluded": { + "wf_fc2164f3-4da": "13 short agents on 2026-09-05T04:57Z to 05:28Z, not ticket-loop work; left out of the published table", + "wf_c4f4430b-f95": "5 agents on 2026-09-05T06:00Z to 06:40Z, not ticket-loop work; left out of the published table" + }, + "note": "wf_4991aff2-011 and wf_b85b65b5-727 each carry more than one ticket (T3 with the T7 and T11 memos; T9 with two T12a agents). The published per-ticket table split them by prompt; per-workflow totals here differ from it by per-agent rounding only.", + "expected_agent_minutes": 1443, + "tolerance": 5 + }, + "wave-4": { + "description": "First like-for-like test: full feature tickets under the revised loop. Measured when all three land.", + "workflows": { + "wf_d128b3c8-44d": "T3b", + "wf_771a8364-c74": "T12", + "wf_5eb938b7-85e": "T7b" + }, + "excluded": { + "wf_2f417f8f-23a": "T15 CLI document attachments: concurrent load, not part of the cohort", + "wf_a2bacbdf-50f": "studio-kb: concurrent load, not part of the cohort" + }, + "expected_agent_minutes": null + } + } +} diff --git a/docs/plans/scripts/agent-minutes.py b/docs/plans/scripts/agent-minutes.py new file mode 100755 index 00000000000..d300cc69b1d --- /dev/null +++ b/docs/plans/scripts/agent-minutes.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Agent-minutes per wave, per workflow and per stage, from Claude Code workflow transcripts. + +This is the measurement behind the "Measured baseline" and "Second baseline" sections of +docs/plans/2026-09-05-zs-throughput-plan.md. Definitions (unchanged since the first baseline): + + one agent = one agent-*.jsonl transcript inside a workflow directory + agent-minutes = last timestamp minus first timestamp of that transcript, rounded per agent + stage = the role line of the agent's prompt (first record of the transcript), matched + in the priority order of STAGES below (specific roles before the generic BUILDER) + wave = the set of workflow ids listed for it in the manifest + +The transcripts are machine-local and are never committed; only this script and the manifest +(workflow ids, labels, expected totals) live in the repo. + +Usage: + agent-minutes.py --manifest agent-minutes-manifest.json --wave waves-2-3 [--workflows-dir DIR] + agent-minutes.py --manifest agent-minutes-manifest.json --check # regression: every wave with an expected total +""" +import argparse +import datetime as dt +import glob +import json +import os +import re +import sys +from collections import OrderedDict, defaultdict + +STAGES = [ + ("delta audit", r"DELTA AUDIT RUNNER"), + ("verifier", r"adversarial verifier"), + ("port check", r"PORT CHECK"), + ("pr opener", r"PR OPENER"), + ("critic", r"BLIND CRITIC"), + ("tester", r"TEST RUNNER"), + ("fix", r"FIX agent"), + ("audit", r"adversarial senior code reviewer|Review the memo"), + ("builder", r"BUILDER"), +] + + +def parse_ts(s): + return dt.datetime.fromisoformat(s.replace("Z", "+00:00")) + + +def read_agent(path): + """Return (first_ts, last_ts, stage) for one transcript, or None if it has no timestamps.""" + first = last = None + stage = None + with open(path, encoding="utf-8") as fh: + for i, line in enumerate(fh): + try: + rec = json.loads(line) + except json.JSONDecodeError: + continue + ts = rec.get("timestamp") + if ts: + t = parse_ts(ts) + first = t if first is None else min(first, t) + last = t if last is None else max(last, t) + if stage is None and i < 3: + blob = json.dumps(rec) + for name, pat in STAGES: + if re.search(pat, blob): + stage = name + break + if first is None: + return None + return first, last, stage or "unclassified" + + +def measure(workflows_dir, wf_ids): + rows = [] + for wf in wf_ids: + matches = glob.glob(os.path.join(workflows_dir, wf + "*")) + if not matches: + print(f"warning: no directory for {wf} under {workflows_dir}", file=sys.stderr) + continue + for path in sorted(glob.glob(os.path.join(matches[0], "agent-*.jsonl"))): + r = read_agent(path) + if r is None: + continue + first, last, stage = r + rows.append({ + "wf": wf, + "agent": os.path.basename(path)[6:-6], + "first": first, + "last": last, + "minutes": round((last - first).total_seconds() / 60), + "stage": stage, + }) + return rows + + +def report(name, wave, rows): + by_wf = OrderedDict() + by_stage = defaultdict(lambda: [0, 0]) + for r in rows: + d = by_wf.setdefault(r["wf"], {"n": 0, "min": 0, "first": r["first"], "last": r["last"]}) + d["n"] += 1 + d["min"] += r["minutes"] + d["first"] = min(d["first"], r["first"]) + d["last"] = max(d["last"], r["last"]) + by_stage[r["stage"]][0] += 1 + by_stage[r["stage"]][1] += r["minutes"] + total = sum(r["minutes"] for r in rows) + print(f"== {name}: {len(rows)} agents, {total} agent-minutes ==") + print("workflow label agents agent-min first last") + for wf, d in by_wf.items(): + label = wave["workflows"].get(wf, "")[:36] + print(f"{wf:17s} {label:37s} {d['n']:5d} {d['min']:10d} {d['first']:%m-%dT%H:%MZ} {d['last']:%m-%dT%H:%MZ}") + print("stage runs agent-min") + for stage, (n, m) in sorted(by_stage.items(), key=lambda kv: -kv[1][1]): + print(f"{stage:13s} {n:4d} {m:10d}") + return total + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--manifest", required=True) + ap.add_argument("--wave", help="wave name from the manifest") + ap.add_argument("--workflows-dir", help="directory holding wf_* transcript directories (default: manifest workflows_dir)") + ap.add_argument("--check", action="store_true", help="regression: compare every wave that has expected_agent_minutes") + args = ap.parse_args() + + with open(args.manifest, encoding="utf-8") as fh: + manifest = json.load(fh) + wdir = os.path.expanduser(args.workflows_dir or manifest["workflows_dir"]) + if not os.path.isdir(wdir): + sys.exit(f"workflows dir not found: {wdir}") + + if args.check: + failed = False + for name, wave in manifest["waves"].items(): + exp = wave.get("expected_agent_minutes") + if exp is None: + continue + total = report(name, wave, measure(wdir, list(wave["workflows"]))) + tol = wave.get("tolerance", 0) + ok = abs(total - exp) <= tol + failed |= not ok + print(f"CHECK {name}: measured {total}, expected {exp} (tolerance {tol}) -> {'OK' if ok else 'FAIL'}\n") + sys.exit(1 if failed else 0) + + if not args.wave: + sys.exit("pass --wave NAME or --check") + wave = manifest["waves"][args.wave] + report(args.wave, wave, measure(wdir, list(wave["workflows"]))) + + +if __name__ == "__main__": + main()