docs(zs): agent-minutes script and manifest behind the throughput baselines

docs/plans/scripts/agent-minutes-manifest.json

@@ -0,0 +1,49 @@
+{
+  "_comment": "Workflow ids per wave for agent-minutes.py. Labels are best-effort from the prompts; the ids are the ground truth. Transcripts live only on the orchestrator machine (workflows_dir) and are never committed. expected_agent_minutes are the totals published in 2026-09-05-zs-throughput-plan.md; the regression check (--check) must reproduce them within the tolerance.",
+  "workflows_dir": "~/.claude/projects/-Users-zero-suminc-/a0fab389-7ac7-4f57-88e6-599d452e8e3a/subagents/workflows",
+  "waves": {
+    "wave-1": {
+      "description": "Seven tickets under the original loop, one workflow, 2026-09-04T08:35Z to 22:09Z.",
+      "workflows": {
+        "wf_d039d459-774": "wave 1, seven tickets (T2, T5, T8, T11 and others)"
+      },
+      "expected_agent_minutes": 2588,
+      "tolerance": 2
+    },
+    "waves-2-3": {
+      "description": "Four code tickets and three memos under the revised loop, 2026-09-04T22:55Z to 2026-09-05T10:09Z.",
+      "workflows": {
+        "wf_4991aff2-011": "T3 port/4316 build",
+        "wf_f8590a1f-179": "T6-config",
+        "wf_b85b65b5-727": "T9 PDF export",
+        "wf_231309b6-0bd": "T7a registry core",
+        "wf_a2d9707f-d29": "T3 Sol-finding verifiers (driver-added)",
+        "wf_72bf0947-3d5": "T3 port check and fix",
+        "wf_ab7d7b5c-0a4": "T12a memo",
+        "wf_a6a94c02-398": "T7a round 3",
+        "wf_5ebaa29c-46d": "shared round 3 (T9 and T7a) and T12a",
+        "wf_d9ff25f2-d0d": "T12a round"
+      },
+      "excluded": {
+        "wf_fc2164f3-4da": "13 short agents on 2026-09-05T04:57Z to 05:28Z, not ticket-loop work; left out of the published table",
+        "wf_c4f4430b-f95": "5 agents on 2026-09-05T06:00Z to 06:40Z, not ticket-loop work; left out of the published table"
+      },
+      "note": "wf_4991aff2-011 and wf_b85b65b5-727 each carry more than one ticket (T3 with the T7 and T11 memos; T9 with two T12a agents). The published per-ticket table split them by prompt; per-workflow totals here differ from it by per-agent rounding only.",
+      "expected_agent_minutes": 1443,
+      "tolerance": 5
+    },
+    "wave-4": {
+      "description": "First like-for-like test: full feature tickets under the revised loop. Measured when all three land.",
+      "workflows": {
+        "wf_d128b3c8-44d": "T3b",
+        "wf_771a8364-c74": "T12",
+        "wf_5eb938b7-85e": "T7b"
+      },
+      "excluded": {
+        "wf_2f417f8f-23a": "T15 CLI document attachments: concurrent load, not part of the cohort",
+        "wf_a2bacbdf-50f": "studio-kb: concurrent load, not part of the cohort"
+      },
+      "expected_agent_minutes": null
+    }
+  }
+}

docs/plans/scripts/agent-minutes.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""Agent-minutes per wave, per workflow and per stage, from Claude Code workflow transcripts.
+
+This is the measurement behind the "Measured baseline" and "Second baseline" sections of
+docs/plans/2026-09-05-zs-throughput-plan.md. Definitions (unchanged since the first baseline):
+
+  one agent        = one agent-*.jsonl transcript inside a workflow directory
+  agent-minutes    = last timestamp minus first timestamp of that transcript, rounded per agent
+  stage            = the role line of the agent's prompt (first record of the transcript), matched
+                     in the priority order of STAGES below (specific roles before the generic BUILDER)
+  wave             = the set of workflow ids listed for it in the manifest
+
+The transcripts are machine-local and are never committed; only this script and the manifest
+(workflow ids, labels, expected totals) live in the repo.
+
+Usage:
+  agent-minutes.py --manifest agent-minutes-manifest.json --wave waves-2-3 [--workflows-dir DIR]
+  agent-minutes.py --manifest agent-minutes-manifest.json --check      # regression: every wave with an expected total
+"""
+import argparse
+import datetime as dt
+import glob
+import json
+import os
+import re
+import sys
+from collections import OrderedDict, defaultdict
+
+STAGES = [
+    ("delta audit", r"DELTA AUDIT RUNNER"),
+    ("verifier", r"adversarial verifier"),
+    ("port check", r"PORT CHECK"),
+    ("pr opener", r"PR OPENER"),
+    ("critic", r"BLIND CRITIC"),
+    ("tester", r"TEST RUNNER"),
+    ("fix", r"FIX agent"),
+    ("audit", r"adversarial senior code reviewer|Review the memo"),
+    ("builder", r"BUILDER"),
+]
+
+
+def parse_ts(s):
+    return dt.datetime.fromisoformat(s.replace("Z", "+00:00"))
+
+
+def read_agent(path):
+    """Return (first_ts, last_ts, stage) for one transcript, or None if it has no timestamps."""
+    first = last = None
+    stage = None
+    with open(path, encoding="utf-8") as fh:
+        for i, line in enumerate(fh):
+            try:
+                rec = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            ts = rec.get("timestamp")
+            if ts:
+                t = parse_ts(ts)
+                first = t if first is None else min(first, t)
+                last = t if last is None else max(last, t)
+            if stage is None and i < 3:
+                blob = json.dumps(rec)
+                for name, pat in STAGES:
+                    if re.search(pat, blob):
+                        stage = name
+                        break
+    if first is None:
+        return None
+    return first, last, stage or "unclassified"
+
+
+def measure(workflows_dir, wf_ids):
+    rows = []
+    for wf in wf_ids:
+        matches = glob.glob(os.path.join(workflows_dir, wf + "*"))
+        if not matches:
+            print(f"warning: no directory for {wf} under {workflows_dir}", file=sys.stderr)
+            continue
+        for path in sorted(glob.glob(os.path.join(matches[0], "agent-*.jsonl"))):
+            r = read_agent(path)
+            if r is None:
+                continue
+            first, last, stage = r
+            rows.append({
+                "wf": wf,
+                "agent": os.path.basename(path)[6:-6],
+                "first": first,
+                "last": last,
+                "minutes": round((last - first).total_seconds() / 60),
+                "stage": stage,
+            })
+    return rows
+
+
+def report(name, wave, rows):
+    by_wf = OrderedDict()
+    by_stage = defaultdict(lambda: [0, 0])
+    for r in rows:
+        d = by_wf.setdefault(r["wf"], {"n": 0, "min": 0, "first": r["first"], "last": r["last"]})
+        d["n"] += 1
+        d["min"] += r["minutes"]
+        d["first"] = min(d["first"], r["first"])
+        d["last"] = max(d["last"], r["last"])
+        by_stage[r["stage"]][0] += 1
+        by_stage[r["stage"]][1] += r["minutes"]
+    total = sum(r["minutes"] for r in rows)
+    print(f"== {name}: {len(rows)} agents, {total} agent-minutes ==")
+    print("workflow          label                                 agents  agent-min  first            last")
+    for wf, d in by_wf.items():
+        label = wave["workflows"].get(wf, "")[:36]
+        print(f"{wf:17s} {label:37s} {d['n']:5d} {d['min']:10d}  {d['first']:%m-%dT%H:%MZ}  {d['last']:%m-%dT%H:%MZ}")
+    print("stage         runs  agent-min")
+    for stage, (n, m) in sorted(by_stage.items(), key=lambda kv: -kv[1][1]):
+        print(f"{stage:13s} {n:4d} {m:10d}")
+    return total
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--manifest", required=True)
+    ap.add_argument("--wave", help="wave name from the manifest")
+    ap.add_argument("--workflows-dir", help="directory holding wf_* transcript directories (default: manifest workflows_dir)")
+    ap.add_argument("--check", action="store_true", help="regression: compare every wave that has expected_agent_minutes")
+    args = ap.parse_args()
+
+    with open(args.manifest, encoding="utf-8") as fh:
+        manifest = json.load(fh)
+    wdir = os.path.expanduser(args.workflows_dir or manifest["workflows_dir"])
+    if not os.path.isdir(wdir):
+        sys.exit(f"workflows dir not found: {wdir}")
+
+    if args.check:
+        failed = False
+        for name, wave in manifest["waves"].items():
+            exp = wave.get("expected_agent_minutes")
+            if exp is None:
+                continue
+            total = report(name, wave, measure(wdir, list(wave["workflows"])))
+            tol = wave.get("tolerance", 0)
+            ok = abs(total - exp) <= tol
+            failed |= not ok
+            print(f"CHECK {name}: measured {total}, expected {exp} (tolerance {tol}) -> {'OK' if ok else 'FAIL'}\n")
+        sys.exit(1 if failed else 0)
+
+    if not args.wave:
+        sys.exit("pass --wave NAME or --check")
+    wave = manifest["waves"][args.wave]
+    report(args.wave, wave, measure(wdir, list(wave["workflows"])))
+
+
+if __name__ == "__main__":
+    main()