Feat/agent bench production grade

.gitignore

@@ -4,6 +4,10 @@
 node_modules
 .pnp
 .pnp.js
+# Content-addressable package store. Default is ~/.local/share/pnpm/store;
+# if pnpm is pointed at the repo (store-dir / PNPM_STORE_DIR), it lands here
+# and must never be committed.
+.pnpm-store/
 
 # Local env files
 .env

AGENTS.md

@@ -21,7 +21,8 @@ This codebase follows `docs/superpowers/specs/2026-05-25-architecture-v4.md` (su
 | **Trace (commands)** | **`TRACE.md`** — `freecode trace` flags, how to read the waterfall |
 | Eval harness | `specs/2026-08-23-eval-harness.md` (Phases 0–5, built) + `specs/2026-08-29-eval-case-registry.md` |
 | **Eval (commands)** | **`EVAL.md`** — which command, which flag, when to run it |
-| Agent comparison | `specs/2026-09-03-agent-comparison-benchmark.md` — freecode vs Claude Code/Codex/OpenCode. **Design only, nothing built.** Deliberately outside `eval/`: only `scorers/outcome.ts` is agent-agnostic |
+| **Agent comparison (commands)** | **`AGENT-BENCH.md`** — `pnpm bench:agents` (+ `bench:grade`, `bench:bundle`, `--isolate`) vs Claude Code / OpenCode; metering, grading, isolation, `/benchmark`. Spec `2026-09-03-agent-comparison-benchmark.md`. **Not `pnpm eval`.** |
+| Agent comparison (design) | `specs/2026-09-03-agent-comparison-benchmark.md` — harness-vs-harness on SWE-bench Lite. Deliberately outside `eval/`. Runtime RAM is `Benchmark.md` |
 | Hooks | `apps/core/src/hooks/hooks-system.md` |
 
 ## Implemented Subsystems

Benchmark.md

@@ -1,6 +1,21 @@
 # Benchmarking FreeCode
 
-> Runtime performance harness comparing **freecode** against other AI coding
+> Two different instruments. Pick the question, then the command.
+
+| Question | Command | Operator page |
+| --- | --- | --- |
+| How much RAM, how fast to first frame? | `pnpm bench:memory` | **this file** |
+| Does it fix real bugs vs other agents, and for how much? | `pnpm bench:agents` | **[`AGENT-BENCH.md`](AGENT-BENCH.md)** — same shape as `EVAL.md` |
+| Did my last change make *our* agent worse? | `pnpm eval` | [`EVAL.md`](EVAL.md) |
+
+The rest of this file is the **runtime** harness (PSS, time-to-visible). It
+does not measure whether anyone fixed a bug.
+
+---
+
+# Runtime performance
+
+> Comparing **freecode** against other AI coding
 > agents (Claude Code, Codex CLI, OpenCode, pi, GitHub Copilot, Cursor Agent,
 > Antigravity). Mirrors the methodology used by
 > [jcode's](https://github.com/1jehuang/jcode) `scripts/bench_memory_cli.py`.

CLAUDE.md

@@ -21,7 +21,8 @@ This codebase follows `docs/superpowers/specs/2026-05-25-architecture-v4.md` (su
 | **Trace (commands)** | **`TRACE.md`** — `freecode trace` flags, how to read the waterfall |
 | Eval harness | `specs/2026-08-23-eval-harness.md` (Phases 0–5, built) + `specs/2026-08-29-eval-case-registry.md` |
 | **Eval (commands)** | **`EVAL.md`** — which command, which flag, when to run it |
-| Agent comparison | `specs/2026-09-03-agent-comparison-benchmark.md` — freecode vs Claude Code/Codex/OpenCode. **Design only, nothing built.** Deliberately outside `eval/`: only `scorers/outcome.ts` is agent-agnostic |
+| **Agent comparison (commands)** | **`AGENT-BENCH.md`** — `pnpm bench:agents` (+ `bench:grade`, `bench:bundle`, `--isolate`) vs Claude Code / OpenCode; metering, grading, isolation, `/benchmark`. Spec `2026-09-03-agent-comparison-benchmark.md`. **Not `pnpm eval`.** |
+| Agent comparison (design) | `specs/2026-09-03-agent-comparison-benchmark.md` — harness-vs-harness on SWE-bench Lite. Deliberately outside `eval/`. Runtime RAM is `Benchmark.md` |
 | Hooks | `apps/core/src/hooks/hooks-system.md` |
 
 ## Implemented Subsystems

EVAL.md

@@ -162,6 +162,7 @@ for a real suite run.
 ```bash
 freecode trace [id] [--follow|--slow N|--tools|--json|--list|--otlp] # where a turn's time went
 pnpm bench:recall # memory retrieval benchmark
+pnpm bench:agents # vs other agents — AGENT-BENCH.md
 ```
 
 ---

README.md

@@ -178,7 +178,8 @@ Full docs: **[freecode.website](https://freecode.website)**
  providers, memory, compaction, permissions, eval
 
 In-repo references: [`CLAUDE.md`](CLAUDE.md) (contributor guide),
-[`EVAL.md`](EVAL.md), [`TRACE.md`](TRACE.md), and the design specs under
+[`EVAL.md`](EVAL.md), [`TRACE.md`](TRACE.md), [`AGENT-BENCH.md`](AGENT-BENCH.md),
+[`Benchmark.md`](Benchmark.md) (runtime RAM / TTF), and the design specs under
 [`docs/superpowers/specs/`](docs/superpowers/specs/).
 
 ## License

apps/core/src/providers/catalogue.test.ts

@@ -3,6 +3,7 @@ import assert from "node:assert/strict";
 import {
  resolveCatalogue,
  envKeysFor,
+ baseURLFor,
  FEATURED_PROVIDER_IDS,
 } from "./catalogue.js";
 import { CATALOGUE_SNAPSHOT } from "./catalogue-snapshot.js";
@@ -86,6 +87,20 @@ test("an id the catalogue does not carry yields no keys rather than guessing", (
  assert.deepEqual(envKeysFor("not-a-real-provider"), []);
 });
 
+test("MINIMAX_BASE_URL overrides the catalogue endpoint", () => {
+ const minimax = resolveCatalogue().find((e) => e.id === "minimax");
+ assert.ok(minimax);
+ assert.equal(baseURLFor(minimax), minimax.baseURL);
+ const prev = process.env.MINIMAX_BASE_URL;
+ process.env.MINIMAX_BASE_URL = "http://127.0.0.1:9/v1";
+ try {
+ assert.equal(baseURLFor(minimax), "http://127.0.0.1:9/v1");
+ } finally {
+ if (prev === undefined) delete process.env.MINIMAX_BASE_URL;
+ else process.env.MINIMAX_BASE_URL = prev;
+ }
+});
+
 test("resolution is memoized, and the memo can be dropped", async () => {
  const { invalidateCatalogue } = await import("./catalogue.js");
  const first = resolveCatalogue();

apps/core/src/providers/catalogue.ts

@@ -211,3 +211,20 @@ export function envKeysFor(id: string): string[] {
  }
  return envKeyIndex.get(id) ?? [];
 }
+
+/**
+ * Catalogue `baseURL`, or `$<ID>_BASE_URL` when set.
+ *
+ * The recording proxy in `bench/agent-bench` points every agent at one meter
+ * (spec §6.4). Claude Code already honours `ANTHROPIC_BASE_URL`; freecode
+ * previously ignored it because the SDK is constructed with the catalogue
+ * URL. `MINIMAX_BASE_URL` (and the same pattern for any other id) is the
+ * equivalent hook — a bench-only override, not a documented user setting.
+ */
+export function baseURLFor(entry: {
+ id: string;
+ baseURL?: string;
+}): string | undefined {
+ const envName = `${entry.id.replace(/-/g, "_").toUpperCase()}_BASE_URL`;
+ return process.env[envName] || entry.baseURL;
+}

apps/core/src/providers/generic-provider.ts

@@ -28,7 +28,10 @@ import { normalizeAiSdkStream } from "./streaming.js";
 import { mapUsage } from "./provider-shared.js";
 import { applyEffort } from "./effort.js";
 import { loadSdkFactory } from "./sdk-factories.js";
-import type { ProviderCatalogueEntry } from "./catalogue.js";
+import {
+ baseURLFor,
+ type ProviderCatalogueEntry,
+} from "./catalogue.js";
 
 /**
  * Which branch of request-shaping an SDK package needs.
@@ -230,7 +233,7 @@ export function createGenericProvider(entry: ProviderCatalogueEntry): AIProvider
  apiKey: oauth
  ? "oauth-subscription"
  : getApiKey(entry.id, entry.envKeys),
- baseURL: entry.baseURL,
+ baseURL: baseURLFor(entry),
  fetch: oauth
  ? createAnthropicOAuthFetch(createTimeoutFetch())
  : createTimeoutFetch(),

apps/web/app/components/AgentBenchmark.tsx

@@ -7,6 +7,9 @@ import { BenchBarList, type BenchBar } from "./BenchBarList";
 
 const pct = (n: number) => `${Math.round(n * 100)}%`;
 const secs = (ms: number) => `${(ms / 1000).toFixed(0)}s`;
+const tok = (n: number) =>
+ n >= 1_000_000 ? `${(n / 1_000_000).toFixed(1)}M` : `${Math.round(n / 1000)}k`;
+const usd = (n: number) => `$${n.toFixed(4)}`;
 
 export function AgentBenchmark({ views }: { views: BenchView[] }) {
  const [slug, setSlug] = useState(views[0]?.slug ?? "");
@@ -52,6 +55,38 @@ export function AgentBenchmark({ views }: { views: BenchView[] }) {
  highlight: a.isFreeCode,
  }));
 
+ // Metering (spec §6.4/§7): shown only for agents the proxy actually saw.
+ // An unmetered agent gets a zero-width bar labelled as such, never a zero —
+ // a blank is not the cheapest run in the table.
+ const anyMetered = view.agents.some((a) => a.meteredTrials > 0);
+ const tokenBars: BenchBar[] = view.agents.map((a) => ({
+ label: a.id,
+ value: a.meanTokens ?? 0,
+ display: a.meanTokens !== undefined ? tok(a.meanTokens) : "unmetered",
+ note:
+ a.meteredTrials > 0
+ ? `mean of ${a.meteredTrials} metered trial${a.meteredTrials === 1 ? "" : "s"}${a.meanTurns !== undefined ? ` · ~${Math.round(a.meanTurns)} turns` : ""}`
+ : "proxy saw no traffic",
+ highlight: a.isFreeCode,
+ }));
+ const costBars: BenchBar[] = view.agents.map((a) => ({
+ label: a.id,
+ value: typeof a.meanUsd === "number" ? a.meanUsd : 0,
+ display:
+ typeof a.meanUsd === "number"
+ ? usd(a.meanUsd)
+ : a.meanUsd === null
+ ? "unpriced"
+ : "unmetered",
+ note:
+ a.worstUsd !== undefined
+ ? `worst single trial ${usd(a.worstUsd)}`
+ : a.meteredTrials > 0
+ ? "no rate-card row for this model"
+ : "proxy saw no traffic",
+ highlight: a.isFreeCode,
+ }));
+
  // Headline cards, freecode against the strongest rival on each axis. The
  // comparison is deliberately unflattering — "slower" goes in the headline in
  // red, because a benchmark we publish only when we win is an advertisement.
@@ -329,6 +364,36 @@ export function AgentBenchmark({ views }: { views: BenchView[] }) {
  bars={sizeBars}
  />
 
+ {anyMetered && (
+ <BenchBarList
+ id={`tokens-${view.slug}`}
+ title="Tokens per trial"
+ description="Input + output counted off one recording proxy for every agent — the same meter, the same rules, never each vendor's own accounting. Fewer is cheaper, but an agent that gives up early is also cheap; read this next to the outcome bar, not instead of it."
+ bars={tokenBars}
+ />
+ )}
+
+ {anyMetered && (
+ <BenchBarList
+ id={`cost-${view.slug}`}
+ title="Cost per trial"
+ description="USD from a committed rate card applied to the proxy's counts — identical pricing for every agent. Cache reads are discounted, not added. The worst-single-trial figure is there because a good mean can hide one runaway."
+ bars={costBars}
+ footnote={
+ view.cost
+ ? view.cost.suppressed
+ ? `Cost on the intersection of solved bugs (spec §7.2) is suppressed: only ${view.cost.instances.length} instance${view.cost.instances.length === 1 ? "" : "s"} were resolved by every agent, and below 3 the mean is an anecdote.`
+ : `On the ${view.cost.instances.length} bugs every agent solved: ${view.cost.perAgent
+ .map(
+ (p) =>
+ `${p.id} ${typeof p.meanUsd === "number" ? usd(p.meanUsd) : "unpriced"}${p.meanTokens !== null ? ` (${tok(p.meanTokens)} tok)` : ""}`,
+ )
+ .join(" · ")}. Averaging over failures would make the quitter cheapest — this mean covers solved bugs only.`
+ : "Per-solved-bug cost (spec §7.2) appears once the matchup is graded — averaging cost over failed attempts would make the agent that gives up fastest look cheapest."
+ }
+ />
+ )}
+
  <div className="rounded-md border border-border bg-card p-6 md:p-8">
  <h3 className="text-lg font-medium text-foreground">Per instance</h3>
  <p className="text-sm text-muted-foreground mt-1 mb-6">
@@ -357,7 +422,7 @@ export function AgentBenchmark({ views }: { views: BenchView[] }) {
  {row.cells.map((cell) => (
  <span
  key={`${cell.agent}-${cell.trial}`}
- title={`${cell.reason} · ${cell.patchBytes}B`}
+ title={`${cell.reason} · ${cell.patchBytes}B${cell.tokens !== undefined ? ` · ${tok(cell.tokens)} tok` : ""}${typeof cell.usd === "number" ? ` · ${usd(cell.usd)}` : ""}${cell.auditOk === false ? " · UNMETERED/LEAK" : ""}`}
  className={`inline-flex items-center gap-1.5 rounded border px-2 py-1 font-mono text-[11px] ${
  cell.ok
  ? "border-border bg-muted text-foreground/80"