From ffcdcb6417de6346d1ce56a469c29b7955df027a Mon Sep 17 00:00:00 2001 From: Ayan De Date: Sun, 6 Sep 2026 04:21:28 +0530 Subject: [PATCH 01/10] chore: ignore .pnpm-store A repo-pointed pnpm store must never be committed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ELXKXKpNy38kvVfjwJC52f --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 852f6bc9..d2ee4b77 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,10 @@ node_modules .pnp .pnp.js +# Content-addressable package store. Default is ~/.local/share/pnpm/store; +# if pnpm is pointed at the repo (store-dir / PNPM_STORE_DIR), it lands here +# and must never be committed. +.pnpm-store/ # Local env files .env From d861cf59fbe95e49721ac8ee33021e1035e72bf7 Mon Sep 17 00:00:00 2001 From: Ayan De Date: Sun, 6 Sep 2026 04:21:28 +0530 Subject: [PATCH 02/10] feat(providers): let $_BASE_URL override a catalogue baseURL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent-bench recording proxy (spec §6.4) points every agent at one meter. Claude Code honours ANTHROPIC_BASE_URL; freecode ignored the equivalent because the SDK is constructed with the catalogue URL. baseURLFor() gives every provider id the same hook (MINIMAX_BASE_URL for the bench) — a bench-only override, not a documented user setting. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ELXKXKpNy38kvVfjwJC52f --- apps/core/src/providers/catalogue.test.ts | 15 +++++++++++++++ apps/core/src/providers/catalogue.ts | 17 +++++++++++++++++ apps/core/src/providers/generic-provider.ts | 7 +++++-- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/apps/core/src/providers/catalogue.test.ts b/apps/core/src/providers/catalogue.test.ts index 7a28a64b..35c3c5da 100644 --- a/apps/core/src/providers/catalogue.test.ts +++ b/apps/core/src/providers/catalogue.test.ts @@ -3,6 +3,7 @@ import assert from "node:assert/strict"; import { resolveCatalogue, envKeysFor, + baseURLFor, FEATURED_PROVIDER_IDS, } from "./catalogue.js"; import { CATALOGUE_SNAPSHOT } from "./catalogue-snapshot.js"; @@ -86,6 +87,20 @@ test("an id the catalogue does not carry yields no keys rather than guessing", ( assert.deepEqual(envKeysFor("not-a-real-provider"), []); }); +test("MINIMAX_BASE_URL overrides the catalogue endpoint", () => { + const minimax = resolveCatalogue().find((e) => e.id === "minimax"); + assert.ok(minimax); + assert.equal(baseURLFor(minimax), minimax.baseURL); + const prev = process.env.MINIMAX_BASE_URL; + process.env.MINIMAX_BASE_URL = "http://127.0.0.1:9/v1"; + try { + assert.equal(baseURLFor(minimax), "http://127.0.0.1:9/v1"); + } finally { + if (prev === undefined) delete process.env.MINIMAX_BASE_URL; + else process.env.MINIMAX_BASE_URL = prev; + } +}); + test("resolution is memoized, and the memo can be dropped", async () => { const { invalidateCatalogue } = await import("./catalogue.js"); const first = resolveCatalogue(); diff --git a/apps/core/src/providers/catalogue.ts b/apps/core/src/providers/catalogue.ts index eb0f9065..ca4583eb 100644 --- a/apps/core/src/providers/catalogue.ts +++ b/apps/core/src/providers/catalogue.ts @@ -211,3 +211,20 @@ export function envKeysFor(id: string): string[] { } return envKeyIndex.get(id) ?? []; } + +/** + * Catalogue `baseURL`, or `$_BASE_URL` when set. + * + * The recording proxy in `bench/agent-bench` points every agent at one meter + * (spec §6.4). Claude Code already honours `ANTHROPIC_BASE_URL`; freecode + * previously ignored it because the SDK is constructed with the catalogue + * URL. `MINIMAX_BASE_URL` (and the same pattern for any other id) is the + * equivalent hook — a bench-only override, not a documented user setting. + */ +export function baseURLFor(entry: { + id: string; + baseURL?: string; +}): string | undefined { + const envName = `${entry.id.replace(/-/g, "_").toUpperCase()}_BASE_URL`; + return process.env[envName] || entry.baseURL; +} diff --git a/apps/core/src/providers/generic-provider.ts b/apps/core/src/providers/generic-provider.ts index bbbbce3f..0280b24c 100644 --- a/apps/core/src/providers/generic-provider.ts +++ b/apps/core/src/providers/generic-provider.ts @@ -28,7 +28,10 @@ import { normalizeAiSdkStream } from "./streaming.js"; import { mapUsage } from "./provider-shared.js"; import { applyEffort } from "./effort.js"; import { loadSdkFactory } from "./sdk-factories.js"; -import type { ProviderCatalogueEntry } from "./catalogue.js"; +import { + baseURLFor, + type ProviderCatalogueEntry, +} from "./catalogue.js"; /** * Which branch of request-shaping an SDK package needs. @@ -230,7 +233,7 @@ export function createGenericProvider(entry: ProviderCatalogueEntry): AIProvider apiKey: oauth ? "oauth-subscription" : getApiKey(entry.id, entry.envKeys), - baseURL: entry.baseURL, + baseURL: baseURLFor(entry), fetch: oauth ? createAnthropicOAuthFetch(createTimeoutFetch()) : createTimeoutFetch(), From 703bd1d5eadbbc1028cc45f5f281514d3f3b5a52 Mon Sep 17 00:00:00 2001 From: Ayan De Date: Sun, 6 Sep 2026 04:21:43 +0530 Subject: [PATCH 03/10] =?UTF-8?q?feat(bench):=20recording=20proxy=20?= =?UTF-8?q?=E2=80=94=20one=20meter,=20not=20four=20self-reports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass-through HTTP proxy (spec §6.4): no cache, no retries, no body rewrite. Tokens parsed off Anthropic Messages JSON/SSE and OpenAI Chat Completions, normalized to an INCLUSIVE inputTokens (Anthropic's wire excludes cache fields, OpenAI's includes them) so the rate card's cache discount is correct for both shapes. USD from a committed rate card with a vintage — an unknown model prices as undefined, never zero. The log doubles as the isolation audit: a non-model path is a leak, and an empty log is unmetered, not clean. No secrets on disk (no headers, no bodies). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ELXKXKpNy38kvVfjwJC52f --- bench/agent-bench/proxy/audit.test.ts | 28 +++++ bench/agent-bench/proxy/audit.ts | 34 ++++++ bench/agent-bench/proxy/env.test.ts | 19 +++ bench/agent-bench/proxy/env.ts | 22 ++++ bench/agent-bench/proxy/fold.test.ts | 64 ++++++++++ bench/agent-bench/proxy/fold.ts | 73 ++++++++++++ bench/agent-bench/proxy/forward.ts | 35 ++++++ bench/agent-bench/proxy/price.test.ts | 49 ++++++++ bench/agent-bench/proxy/price.ts | 45 ++++++++ bench/agent-bench/proxy/server.test.ts | 154 +++++++++++++++++++++++++ bench/agent-bench/proxy/server.ts | 151 ++++++++++++++++++++++++ bench/agent-bench/proxy/usage.test.ts | 71 ++++++++++++ bench/agent-bench/proxy/usage.ts | 113 ++++++++++++++++++ 13 files changed, 858 insertions(+) create mode 100644 bench/agent-bench/proxy/audit.test.ts create mode 100644 bench/agent-bench/proxy/audit.ts create mode 100644 bench/agent-bench/proxy/env.test.ts create mode 100644 bench/agent-bench/proxy/env.ts create mode 100644 bench/agent-bench/proxy/fold.test.ts create mode 100644 bench/agent-bench/proxy/fold.ts create mode 100644 bench/agent-bench/proxy/forward.ts create mode 100644 bench/agent-bench/proxy/price.test.ts create mode 100644 bench/agent-bench/proxy/price.ts create mode 100644 bench/agent-bench/proxy/server.test.ts create mode 100644 bench/agent-bench/proxy/server.ts create mode 100644 bench/agent-bench/proxy/usage.test.ts create mode 100644 bench/agent-bench/proxy/usage.ts diff --git a/bench/agent-bench/proxy/audit.test.ts b/bench/agent-bench/proxy/audit.test.ts new file mode 100644 index 00000000..1c7ea747 --- /dev/null +++ b/bench/agent-bench/proxy/audit.test.ts @@ -0,0 +1,28 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { destUrl } from "./forward.js"; +import { audit, isModelEndpoint } from "./audit.js"; + +test("keeps the upstream path prefix when joining /v1/messages", () => { + assert.equal( + destUrl("https://api.minimax.io/anthropic", "/v1/messages").href, + "https://api.minimax.io/anthropic/v1/messages", + ); +}); + +test("messages and chat/completions are the model; a GitHub path is not", () => { + assert.equal(isModelEndpoint("/v1/messages"), true); + assert.equal(isModelEndpoint("/messages"), true); + assert.equal(isModelEndpoint("/v1/chat/completions"), true); + assert.equal(isModelEndpoint("/repos/django/django"), false); +}); + +test("audit fails closed on any non-model path", () => { + const result = audit([ + { modelEndpoint: true, method: "POST", path: "/v1/messages", host: "api.minimax.io", status: 200 }, + { modelEndpoint: false, method: "GET", path: "/repos/django/django", host: "github.com", status: 200 }, + ]); + assert.equal(result.ok, false); + assert.equal(result.modelCalls, 1); + assert.equal(result.leaks[0]?.path, "/repos/django/django"); +}); diff --git a/bench/agent-bench/proxy/audit.ts b/bench/agent-bench/proxy/audit.ts new file mode 100644 index 00000000..9c89bb4d --- /dev/null +++ b/bench/agent-bench/proxy/audit.ts @@ -0,0 +1,34 @@ +export function isModelEndpoint(pathname: string): boolean { + return /\/(v\d+\/)?(messages|chat\/completions)\/?$/.test(pathname); +} + +export interface IsolationAudit { + ok: boolean; + modelCalls: number; + leaks: { method: string; path: string; host: string; status: number }[]; +} + +/** The proxy log is the list of everything that left. A non-model path is a leak. */ +export function audit( + calls: { + modelEndpoint: boolean; + method: string; + path: string; + host: string; + status: number; + }[], +): IsolationAudit { + const leaks = calls + .filter((c) => !c.modelEndpoint) + .map((c) => ({ + method: c.method, + path: c.path, + host: c.host, + status: c.status, + })); + return { + ok: leaks.length === 0, + modelCalls: calls.filter((c) => c.modelEndpoint).length, + leaks, + }; +} diff --git a/bench/agent-bench/proxy/env.test.ts b/bench/agent-bench/proxy/env.test.ts new file mode 100644 index 00000000..6871e192 --- /dev/null +++ b/bench/agent-bench/proxy/env.test.ts @@ -0,0 +1,19 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { DEFAULT_UPSTREAM, meterEnv, upstreamFor } from "./env.js"; + +test("Claude gets the origin, freecode gets the origin plus /v1", () => { + assert.deepEqual(meterEnv("http://127.0.0.1:9"), { + ANTHROPIC_BASE_URL: "http://127.0.0.1:9", + MINIMAX_BASE_URL: "http://127.0.0.1:9/v1", + }); +}); + +test("upstream is the adapter's Anthropic URL when it is a literal", () => { + assert.equal( + upstreamFor({ env: { ANTHROPIC_BASE_URL: "https://api.minimax.io/anthropic" } }), + "https://api.minimax.io/anthropic", + ); + assert.equal(upstreamFor({ env: { ANTHROPIC_BASE_URL: "${MINIMAX_API_KEY}" } }), DEFAULT_UPSTREAM); + assert.equal(upstreamFor({}), DEFAULT_UPSTREAM); +}); diff --git a/bench/agent-bench/proxy/env.ts b/bench/agent-bench/proxy/env.ts new file mode 100644 index 00000000..4b08fb99 --- /dev/null +++ b/bench/agent-bench/proxy/env.ts @@ -0,0 +1,22 @@ +export const DEFAULT_UPSTREAM = "https://api.minimax.io/anthropic"; + +/** + * Point every current adapter at the recording proxy. + * + * Claude Code reads `ANTHROPIC_BASE_URL` (no `/v1` suffix — it adds it). + * Freecode's MiniMax SDK is constructed with the catalogue URL + * (`.../anthropic/v1`), so `MINIMAX_BASE_URL` carries that suffix. + */ +export function meterEnv(origin: string): Record { + const base = origin.replace(/\/$/, ""); + return { + ANTHROPIC_BASE_URL: base, + MINIMAX_BASE_URL: `${base}/v1`, + }; +} + +export function upstreamFor(spec: { env?: Record }): string { + const raw = spec.env?.ANTHROPIC_BASE_URL; + if (raw && raw.startsWith("http") && !raw.includes("${")) return raw; + return DEFAULT_UPSTREAM; +} diff --git a/bench/agent-bench/proxy/fold.test.ts b/bench/agent-bench/proxy/fold.test.ts new file mode 100644 index 00000000..ade498fd --- /dev/null +++ b/bench/agent-bench/proxy/fold.test.ts @@ -0,0 +1,64 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { foldTrial, readLog } from "./fold.js"; +import type { LoggedCall } from "./server.js"; + +function call(partial: Partial & Pick): LoggedCall { + return { + ts: "t", + method: "POST", + path: "/v1/messages", + host: "api.minimax.io", + status: 200, + durationMs: 1, + requestBytes: 0, + responseBytes: 0, + ...partial, + }; +} + +test("folds two model calls into turns, tokens, and MiniMax-M3 USD", () => { + const calls: LoggedCall[] = [ + call({ + modelEndpoint: true, + model: "MiniMax-M3", + usage: { + inputTokens: 1_000_000, + outputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + }), + call({ + modelEndpoint: true, + usage: { + inputTokens: 0, + outputTokens: 1_000_000, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + }), + ]; + const folded = foldTrial(calls, "MiniMax-M3"); + assert.equal(folded.turns, 2); + assert.equal(folded.inputTokens, 1_000_000); + assert.equal(folded.outputTokens, 1_000_000); + assert.equal(folded.usd, 1.5); + assert.equal(folded.leaks, 0); +}); + +test("a leak does not count as a turn and flags the fold", () => { + const folded = foldTrial( + [call({ modelEndpoint: false, path: "/repos/x" })], + "MiniMax-M3", + ); + assert.equal(folded.turns, 0); + assert.equal(folded.leaks, 1); + assert.equal(folded.auditOk, false); +}); + +test("an empty log is unmetered, not a cheap clean run", () => { + const folded = foldTrial([], "MiniMax-M3"); + assert.equal(folded.turns, 0); + assert.equal(folded.auditOk, false); +}); diff --git a/bench/agent-bench/proxy/fold.ts b/bench/agent-bench/proxy/fold.ts new file mode 100644 index 00000000..cc3e6b8e --- /dev/null +++ b/bench/agent-bench/proxy/fold.ts @@ -0,0 +1,73 @@ +import * as fs from "fs"; +import * as path from "path"; +import { audit } from "./audit.js"; +import { priceUsd } from "./price.js"; +import type { LoggedCall } from "./server.js"; +import type { Usage } from "./usage.js"; + +export interface TrialMeter { + turns: number; + inputTokens: number; + outputTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + usd: number | undefined; + leaks: number; + auditOk: boolean; +} + +const ZERO: Usage = { + inputTokens: 0, + outputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, +}; + +export function readLog(file: string): LoggedCall[] { + if (!fs.existsSync(file)) return []; + return fs + .readFileSync(file, "utf-8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as LoggedCall); +} + +export function foldTrial(calls: LoggedCall[], model: string): TrialMeter { + const modelCalls = calls.filter((c) => c.modelEndpoint); + const usage = modelCalls.reduce( + (acc, c) => ({ + inputTokens: acc.inputTokens + (c.usage?.inputTokens ?? 0), + outputTokens: acc.outputTokens + (c.usage?.outputTokens ?? 0), + cacheReadTokens: acc.cacheReadTokens + (c.usage?.cacheReadTokens ?? 0), + cacheWriteTokens: acc.cacheWriteTokens + (c.usage?.cacheWriteTokens ?? 0), + }), + { ...ZERO }, + ); + const isolation = audit(calls); + return { + turns: modelCalls.length, + ...usage, + usd: priceUsd(model, usage), + leaks: isolation.leaks.length, + // An empty log is not a clean audit: the agent never went through the + // meter (wrong base URL, or it talked around us). + auditOk: isolation.ok && modelCalls.length > 0, + }; +} + +/** Fold the jsonl into usage.json + audit.json next to it. */ +export function persistTrialMeter(artifactDir: string, model: string): TrialMeter { + const logPath = path.join(artifactDir, "proxy.jsonl"); + const calls = readLog(logPath); + const usage = foldTrial(calls, model); + fs.writeFileSync( + path.join(artifactDir, "usage.json"), + JSON.stringify(usage, null, 2) + "\n", + ); + fs.writeFileSync( + path.join(artifactDir, "audit.json"), + JSON.stringify(audit(calls), null, 2) + "\n", + ); + return usage; +} diff --git a/bench/agent-bench/proxy/forward.ts b/bench/agent-bench/proxy/forward.ts new file mode 100644 index 00000000..c8d30074 --- /dev/null +++ b/bench/agent-bench/proxy/forward.ts @@ -0,0 +1,35 @@ +import type { IncomingMessage } from "http"; + +const HOP = new Set([ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailers", + "transfer-encoding", + "upgrade", + "host", +]); + +export function isHopByHop(name: string): boolean { + return HOP.has(name.toLowerCase()); +} + +/** Join the incoming path onto the upstream prefix without collapsing it. */ +export function destUrl(upstream: string, reqUrl: string): URL { + const prefix = new URL(upstream); + const incoming = new URL(reqUrl, "http://proxy.local"); + const base = prefix.pathname.replace(/\/$/, ""); + return new URL(base + incoming.pathname + incoming.search, prefix.origin); +} + +export function outboundHeaders(req: IncomingMessage, dest: URL): Headers { + const headers = new Headers(); + for (const [name, value] of Object.entries(req.headers)) { + if (!value || isHopByHop(name)) continue; + headers.set(name, Array.isArray(value) ? value.join(", ") : value); + } + headers.set("host", dest.host); + return headers; +} diff --git a/bench/agent-bench/proxy/price.test.ts b/bench/agent-bench/proxy/price.test.ts new file mode 100644 index 00000000..3138eb9d --- /dev/null +++ b/bench/agent-bench/proxy/price.test.ts @@ -0,0 +1,49 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { priceUsd } from "./price.js"; + +test("an unknown model prices as undefined, never as zero", () => { + assert.equal( + priceUsd("mystery-model", { + inputTokens: 1_000_000, + outputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }), + undefined, + ); +}); + +test("MiniMax-M3: 1M plain input is $0.30, 1M output is $1.20", () => { + assert.equal( + priceUsd("MiniMax-M3", { + inputTokens: 1_000_000, + outputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }), + 0.3, + ); + assert.equal( + priceUsd("minimax/MiniMax-M3", { + inputTokens: 0, + outputTokens: 1_000_000, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }), + 1.2, + ); +}); + +test("a cache read is a discount off inclusive input, not an addend", () => { + // 1M input of which 800k is a cache read: 200k × $0.30 + 800k × $0.06 = $0.108 + assert.equal( + priceUsd("MiniMax-M3", { + inputTokens: 1_000_000, + outputTokens: 0, + cacheReadTokens: 800_000, + cacheWriteTokens: 0, + }), + 0.108, + ); +}); diff --git a/bench/agent-bench/proxy/price.ts b/bench/agent-bench/proxy/price.ts new file mode 100644 index 00000000..0cd40e0a --- /dev/null +++ b/bench/agent-bench/proxy/price.ts @@ -0,0 +1,45 @@ +// One rate card for every agent. Spec §6.4 / §7: comparing four vendors' +// self-reports is comparing four rounding policies. Cache reads are a +// discount off the inclusive input count, matching providers/pricing.ts — +// adding them on top would report a cache win as a cost increase. +// +// This copy lives here on purpose: agent-bench shares no code with +// apps/core/src/eval, and importing pricing.ts would pull the models.dev +// catalogue (and whatever ~/.freecode/pricing.json the operator has) into +// a published number. A committed table with a vintage is the honest stamp. + +import type { Usage } from "./usage.js"; + +const MILLION = 1_000_000; + +/** USD per million tokens. Vintage: MiniMax pay-as-you-go, standard ≤512k, 2026-09. */ +export interface Rate { + input: number; + output: number; + cacheRead?: number; + cacheWrite?: number; +} + +export const RATES_AS_OF = "2026-09 MiniMax standard ≤512k"; + +const RATES: Record = { + "minimax-m3": { input: 0.3, output: 1.2, cacheRead: 0.06 }, +}; + +function key(model: string): string { + const bare = model.includes("/") ? model.slice(model.indexOf("/") + 1) : model; + return bare.toLowerCase(); +} + +export function priceUsd(model: string, usage: Usage): number | undefined { + const rate = RATES[key(model)]; + if (!rate) return undefined; + const cacheRead = usage.cacheReadTokens; + const cacheWrite = usage.cacheWriteTokens; + const plainInput = Math.max(0, usage.inputTokens - cacheRead - cacheWrite); + let usd = (plainInput / MILLION) * rate.input; + usd += (cacheRead / MILLION) * (rate.cacheRead ?? rate.input); + usd += (cacheWrite / MILLION) * (rate.cacheWrite ?? rate.input); + usd += (usage.outputTokens / MILLION) * rate.output; + return usd; +} diff --git a/bench/agent-bench/proxy/server.test.ts b/bench/agent-bench/proxy/server.test.ts new file mode 100644 index 00000000..ab676c4b --- /dev/null +++ b/bench/agent-bench/proxy/server.test.ts @@ -0,0 +1,154 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "fs"; +import * as http from "http"; +import * as os from "os"; +import * as path from "path"; +import { startProxy, type LoggedCall } from "./server.js"; + +function tmpLog(): string { + return path.join( + fs.mkdtempSync(path.join(os.tmpdir(), "agent-bench-proxy-")), + "proxy.jsonl", + ); +} + +function listen( + handler: http.RequestListener, +): Promise<{ origin: string; close: () => Promise }> { + const server = http.createServer(handler); + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const addr = server.address(); + if (!addr || typeof addr === "string") throw new Error("no port"); + resolve({ + origin: `http://127.0.0.1:${addr.port}`, + close: () => + new Promise((r, j) => server.close((e) => (e ? j(e) : r()))), + }); + }); + }); +} + +function readLog(file: string): LoggedCall[] { + return fs + .readFileSync(file, "utf-8") + .trim() + .split("\n") + .filter(Boolean) + .map((l) => JSON.parse(l) as LoggedCall); +} + +test("forwards the body unchanged and records Anthropic usage from the reply", async () => { + const logPath = tmpLog(); + let hits = 0; + const upstream = await listen((req, res) => { + hits++; + const chunks: Buffer[] = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => { + assert.equal(Buffer.concat(chunks).toString(), '{"model":"MiniMax-M3"}'); + assert.equal(req.headers["x-api-key"], "secret"); + res.setHeader("content-type", "application/json"); + res.end( + JSON.stringify({ + usage: { input_tokens: 12, output_tokens: 3, cache_read_input_tokens: 8 }, + }), + ); + }); + }); + const proxy = await startProxy({ upstream: upstream.origin, logPath }); + try { + const res = await fetch(`${proxy.origin}/v1/messages`, { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "secret" }, + body: '{"model":"MiniMax-M3"}', + }); + assert.equal(res.status, 200); + assert.equal(hits, 1); + const body = await res.json(); + assert.equal((body as { usage: { input_tokens: number } }).usage.input_tokens, 12); + + const [row] = readLog(logPath); + assert.equal(row.method, "POST"); + assert.equal(row.path, "/v1/messages"); + assert.equal(row.status, 200); + assert.equal(row.model, "MiniMax-M3"); + // Wire input_tokens (12) is exclusive; the log normalizes to inclusive. + assert.deepEqual(row.usage, { + inputTokens: 20, + outputTokens: 3, + cacheReadTokens: 8, + cacheWriteTokens: 0, + }); + assert.equal(row.modelEndpoint, true); + assert.equal(JSON.stringify(row).includes("secret"), false); + } finally { + await proxy.close(); + await upstream.close(); + } +}); + +test("does not retry a 500 — a retry would be an optimization of one side", async () => { + const logPath = tmpLog(); + let hits = 0; + const upstream = await listen((_req, res) => { + hits++; + res.statusCode = 500; + res.end("nope"); + }); + const proxy = await startProxy({ upstream: upstream.origin, logPath }); + try { + const res = await fetch(`${proxy.origin}/v1/messages`, { method: "POST" }); + assert.equal(res.status, 500); + assert.equal(hits, 1); + assert.equal(await res.text(), "nope"); + } finally { + await proxy.close(); + await upstream.close(); + } +}); + +test("SSE streams through and usage is taken from the last event", async () => { + const logPath = tmpLog(); + const sse = [ + "event: message_start", + 'data: {"type":"message_start","message":{"usage":{"input_tokens":4,"output_tokens":1}}}', + "", + "event: message_delta", + 'data: {"type":"message_delta","usage":{"output_tokens":9}}', + "", + ].join("\n"); + const upstream = await listen((_req, res) => { + res.setHeader("content-type", "text/event-stream"); + res.end(sse); + }); + const proxy = await startProxy({ upstream: upstream.origin, logPath }); + try { + const res = await fetch(`${proxy.origin}/v1/messages`, { method: "POST" }); + assert.equal(await res.text(), sse); + const [row] = readLog(logPath); + assert.equal(row.usage?.outputTokens, 9); + assert.equal(row.usage?.inputTokens, 4); + } finally { + await proxy.close(); + await upstream.close(); + } +}); + +test("a request that is not the model endpoint is recorded as a leak", async () => { + const logPath = tmpLog(); + const upstream = await listen((_req, res) => { + res.end("ok"); + }); + const proxy = await startProxy({ upstream: upstream.origin, logPath }); + try { + await fetch(`${proxy.origin}/repos/django/django`); + const [row] = readLog(logPath); + assert.equal(row.path, "/repos/django/django"); + assert.equal(row.modelEndpoint, false); + } finally { + await proxy.close(); + await upstream.close(); + } +}); diff --git a/bench/agent-bench/proxy/server.ts b/bench/agent-bench/proxy/server.ts new file mode 100644 index 00000000..bf1c599b --- /dev/null +++ b/bench/agent-bench/proxy/server.ts @@ -0,0 +1,151 @@ +// Pass-through recording proxy. Spec §6.4: no caching, no retries, no +// rewriting. Tokens come off the wire; the log is also the isolation audit. + +import * as fs from "fs"; +import * as http from "http"; +import * as path from "path"; +import { isModelEndpoint } from "./audit.js"; +import { destUrl, isHopByHop, outboundHeaders } from "./forward.js"; +import { parseUsage, usageFromSse, type Usage } from "./usage.js"; + +export interface LoggedCall { + ts: string; + method: string; + path: string; + host: string; + status: number; + durationMs: number; + requestBytes: number; + responseBytes: number; + model?: string; + usage?: Usage; + /** False when the path is not a model API — a leak, or an attempt at one. */ + modelEndpoint: boolean; +} + +export interface ProxyOptions { + /** Origin + path prefix the agent thinks it is talking to, e.g. MiniMax /anthropic. */ + upstream: string; + logPath: string; + /** + * Bind address. Default loopback; isolated trials bind the internal docker + * network's gateway instead, the one address an agent container can reach. + */ + host?: string; +} + +export interface RecordingProxy { + origin: string; + port: number; + close(): Promise; +} + +function readBody(req: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => resolve(Buffer.concat(chunks))); + req.on("error", reject); + }); +} + +function modelFrom(body: Buffer): string | undefined { + try { + const parsed = JSON.parse(body.toString("utf-8")); + return typeof parsed?.model === "string" ? parsed.model : undefined; + } catch { + return undefined; + } +} + +function usageFrom(body: Buffer, contentType: string | undefined): Usage | undefined { + const text = body.toString("utf-8"); + if (contentType?.includes("event-stream") || text.startsWith("event:")) { + return usageFromSse(text); + } + try { + return parseUsage(JSON.parse(text)); + } catch { + return usageFromSse(text); + } +} + +function appendLog(file: string, row: LoggedCall): void { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.appendFileSync(file, JSON.stringify(row) + "\n"); +} + +export async function startProxy(opts: ProxyOptions): Promise { + const server = http.createServer(async (req, res) => { + const started = Date.now(); + const reqUrl = req.url ?? "/"; + const dest = destUrl(opts.upstream, reqUrl); + const method = req.method ?? "GET"; + let status = 502; + let responseBytes = 0; + let requestBytes = 0; + let model: string | undefined; + let usage: Usage | undefined; + try { + const body = await readBody(req); + requestBytes = body.length; + model = modelFrom(body); + const init: RequestInit = { + method, + headers: outboundHeaders(req, dest), + }; + if (method !== "GET" && method !== "HEAD") init.body = new Uint8Array(body); + const up = await fetch(dest, init); + status = up.status; + for (const [name, value] of up.headers) { + if (isHopByHop(name)) continue; + res.setHeader(name, value); + } + res.statusCode = status; + const chunks: Buffer[] = []; + if (up.body) { + for await (const chunk of up.body) { + const buf = Buffer.from(chunk); + chunks.push(buf); + responseBytes += buf.length; + res.write(buf); + } + } + res.end(); + usage = usageFrom(Buffer.concat(chunks), up.headers.get("content-type") ?? undefined); + } catch (err) { + if (!res.headersSent) { + res.statusCode = 502; + res.end(String((err as Error).message)); + } else { + res.end(); + } + } finally { + const pathname = new URL(reqUrl, "http://proxy.local").pathname; + appendLog(opts.logPath, { + ts: new Date().toISOString(), + method, + path: pathname, + host: dest.host, + status, + durationMs: Date.now() - started, + requestBytes, + responseBytes, + model, + usage, + modelEndpoint: isModelEndpoint(pathname), + }); + } + }); + + const host = opts.host ?? "127.0.0.1"; + await new Promise((resolve) => server.listen(0, host, resolve)); + const addr = server.address(); + if (!addr || typeof addr === "string") throw new Error("proxy: no port"); + return { + origin: `http://${host}:${addr.port}`, + port: addr.port, + close: () => + new Promise((resolve, reject) => server.close((e) => (e ? reject(e) : resolve()))), + }; +} diff --git a/bench/agent-bench/proxy/usage.test.ts b/bench/agent-bench/proxy/usage.test.ts new file mode 100644 index 00000000..e70f553f --- /dev/null +++ b/bench/agent-bench/proxy/usage.test.ts @@ -0,0 +1,71 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { parseUsage, usageFromSse } from "./usage.js"; + +test("Anthropic wire is exclusive; Usage.inputTokens comes back inclusive", () => { + // Real Anthropic convention: input_tokens does NOT include the cache + // fields. 15 fresh + 80 read + 5 written = 100 inclusive. + assert.deepEqual( + parseUsage({ + usage: { + input_tokens: 15, + output_tokens: 20, + cache_read_input_tokens: 80, + cache_creation_input_tokens: 5, + }, + }), + { + inputTokens: 100, + outputTokens: 20, + cacheReadTokens: 80, + cacheWriteTokens: 5, + }, + ); +}); + +test("Anthropic with no cache fields: input passes through untouched", () => { + assert.deepEqual( + parseUsage({ usage: { input_tokens: 40, output_tokens: 3 } }), + { inputTokens: 40, outputTokens: 3, cacheReadTokens: 0, cacheWriteTokens: 0 }, + ); +}); + +test("reads OpenAI Chat Completions usage, cached_tokens as cache reads", () => { + assert.deepEqual( + parseUsage({ + usage: { + prompt_tokens: 50, + completion_tokens: 10, + prompt_tokens_details: { cached_tokens: 40 }, + }, + }), + { + inputTokens: 50, + outputTokens: 10, + cacheReadTokens: 40, + cacheWriteTokens: 0, + }, + ); +}); + +test("ignores a body with no usage", () => { + assert.equal(parseUsage({ id: "msg_1", content: [] }), undefined); +}); + +test("SSE: message_delta only has output_tokens; input and cache stay from message_start", () => { + const sse = [ + "event: message_start", + 'data: {"type":"message_start","message":{"usage":{"input_tokens":10,"output_tokens":1,"cache_read_input_tokens":4}}}', + "", + "event: message_delta", + 'data: {"type":"message_delta","usage":{"output_tokens":7}}', + "", + ].join("\n"); + // 10 fresh + 4 cache-read = 14 inclusive; the delta overlays only output. + assert.deepEqual(usageFromSse(sse), { + inputTokens: 14, + outputTokens: 7, + cacheReadTokens: 4, + cacheWriteTokens: 0, + }); +}); diff --git a/bench/agent-bench/proxy/usage.ts b/bench/agent-bench/proxy/usage.ts new file mode 100644 index 00000000..e522b3ad --- /dev/null +++ b/bench/agent-bench/proxy/usage.ts @@ -0,0 +1,113 @@ +// Parse token usage off the wire. Anthropic Messages and OpenAI Chat +// Completions only — those are the two shapes MiniMax's /anthropic shim and +// a native openai-compatible agent both emit. Spec §6.4: one meter, not four +// self-reports. + +export interface Usage { + /** + * INCLUSIVE of cache reads and writes, whatever the wire said. OpenAI's + * `prompt_tokens` already is; Anthropic's `input_tokens` is exclusive + * (cache_read/cache_creation are separate additive fields), so the + * Anthropic parser adds them in. price.ts subtracts the cache fields back + * out — normalizing here is what makes that subtraction correct for both + * shapes. + */ + inputTokens: number; + outputTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; +} + +function num(v: unknown): number { + return typeof v === "number" && Number.isFinite(v) ? Math.max(0, v) : 0; +} + +function take(usage: Record, wire: string, field: keyof Usage): Partial { + return wire in usage ? { [field]: num(usage[wire]) } : {}; +} + +/** Only fields the payload actually carried — a delta must not zero the rest. */ +function fromAnthropicPartial(usage: Record): Partial { + const partial: Partial = { + ...take(usage, "input_tokens", "inputTokens"), + ...take(usage, "output_tokens", "outputTokens"), + ...take(usage, "cache_read_input_tokens", "cacheReadTokens"), + ...take(usage, "cache_creation_input_tokens", "cacheWriteTokens"), + }; + // Anthropic's input_tokens EXCLUDES cache tokens; Usage.inputTokens is + // inclusive. The cache fields ride the same payload (message_start), so + // normalizing per-payload survives the SSE overlay in usageFromSse. + if (partial.inputTokens !== undefined) { + partial.inputTokens += + (partial.cacheReadTokens ?? 0) + (partial.cacheWriteTokens ?? 0); + } + return partial; +} + +function complete(partial: Partial): Usage { + return { + inputTokens: partial.inputTokens ?? 0, + outputTokens: partial.outputTokens ?? 0, + cacheReadTokens: partial.cacheReadTokens ?? 0, + cacheWriteTokens: partial.cacheWriteTokens ?? 0, + }; +} + +function fromOpenAi(usage: Record): Usage | undefined { + if (usage.prompt_tokens === undefined && usage.completion_tokens === undefined) { + return undefined; + } + const details = usage.prompt_tokens_details; + const cached = + details && typeof details === "object" + ? num((details as Record).cached_tokens) + : 0; + return { + inputTokens: num(usage.prompt_tokens), + outputTokens: num(usage.completion_tokens), + cacheReadTokens: cached, + cacheWriteTokens: 0, + }; +} + +function usageFields(body: unknown): Partial | undefined { + if (!body || typeof body !== "object") return undefined; + const rec = body as Record; + const usage = rec.usage; + if (usage && typeof usage === "object") { + const u = usage as Record; + if ("input_tokens" in u || "output_tokens" in u) return fromAnthropicPartial(u); + const openai = fromOpenAi(u); + if (openai) return openai; + } + const nested = rec.message; + if (nested && typeof nested === "object") return usageFields(nested); + return undefined; +} + +/** Usage object nested on a JSON response (or an SSE `data:` payload). */ +export function parseUsage(body: unknown): Usage | undefined { + const fields = usageFields(body); + if (!fields || Object.keys(fields).length === 0) return undefined; + return complete(fields); +} + +/** Walk an SSE buffer; later events overlay only the fields they carry. */ +export function usageFromSse(text: string): Usage | undefined { + let acc: Partial | undefined; + for (const block of text.split(/\r?\n\r?\n/)) { + const data = block + .split(/\r?\n/) + .filter((l) => l.startsWith("data:")) + .map((l) => l.slice(5).trimStart()) + .join(""); + if (!data || data === "[DONE]") continue; + try { + const parsed = usageFields(JSON.parse(data)); + if (parsed && Object.keys(parsed).length > 0) acc = { ...acc, ...parsed }; + } catch { + // A truncated frame is not a usage event. + } + } + return acc ? complete(acc) : undefined; +} From 38052744b9bfe0cdf4f13c93e395e7a78376f5d6 Mon Sep 17 00:00:00 2001 From: Ayan De Date: Sun, 6 Sep 2026 04:21:43 +0530 Subject: [PATCH 04/10] =?UTF-8?q?feat(bench):=20container=20isolation=20mo?= =?UTF-8?q?dule=20(spec=20=C2=A76.3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One container per trial on an --internal docker network: no route to the internet, so the only exit is the recording proxy on the network gateway — the proxy log becomes a real egress audit. Stated residue: other host services on the gateway IP stay reachable; the internet does not. The image bakes pinned agent versions (freecode from a released binary — at trial time there is no network to install with), $HOME is a throwaway so no memory carries between trials, and env values ride bare -e NAME flags so argv.json never contains a secret. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ELXKXKpNy38kvVfjwJC52f --- bench/agent-bench/isolate/Dockerfile | 35 ++++++++ bench/agent-bench/isolate/docker.test.ts | 33 ++++++++ bench/agent-bench/isolate/docker.ts | 102 +++++++++++++++++++++++ 3 files changed, 170 insertions(+) create mode 100644 bench/agent-bench/isolate/Dockerfile create mode 100644 bench/agent-bench/isolate/docker.test.ts create mode 100644 bench/agent-bench/isolate/docker.ts diff --git a/bench/agent-bench/isolate/Dockerfile b/bench/agent-bench/isolate/Dockerfile new file mode 100644 index 00000000..d514ac68 --- /dev/null +++ b/bench/agent-bench/isolate/Dockerfile @@ -0,0 +1,35 @@ +# One image, every agent. Built ONLINE (`pnpm bench:image`); trials then run +# it on an --internal network with no route out, so everything an agent needs +# must already be in here. Versions are build args so a run can pin exactly +# what it benchmarked — they default to the versions verified in +# AGENT-BENCH.md §5 and should move in lockstep with the adapter files. + +FROM node:22-bookworm-slim + +ARG CLAUDE_CODE_VERSION=2.1.251 +ARG OPENCODE_VERSION=1.18.25 +# A released freecode binary — the self-contained bun build, never the SEA one. +ARG FREECODE_VERSION=v0.30.0 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +RUN npm install -g \ + @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} \ + opencode-ai@${OPENCODE_VERSION} + +RUN curl -fsSL -o /tmp/freecode.tar.gz \ + https://github.com/ayan-de/freecode/releases/download/${FREECODE_VERSION}/freecode-linux-x86_64.tar.gz \ + && mkdir -p /opt/freecode \ + && tar -xzf /tmp/freecode.tar.gz -C /opt/freecode \ + && ln -s /opt/freecode/freecode /usr/local/bin/freecode \ + && rm /tmp/freecode.tar.gz + +# Trials run --user : of the operator, so HOME must be writable by +# anyone. Agent CLIs write their config/state here and it dies with the +# container — which is the point: no memory carried between trials (§6.3). +RUN mkdir -p /tmp/agent-home && chmod 777 /tmp/agent-home +ENV HOME=/tmp/agent-home + +WORKDIR /workspace diff --git a/bench/agent-bench/isolate/docker.test.ts b/bench/agent-bench/isolate/docker.test.ts new file mode 100644 index 00000000..2825943a --- /dev/null +++ b/bench/agent-bench/isolate/docker.test.ts @@ -0,0 +1,33 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { dockerArgv, forwardedEnvNames, type Containerize } from "./docker.js"; + +const c: Containerize = { + image: "agent-bench", + network: "agent-bench-internal", + name: "trial-x", + wsDir: "/tmp/ws", + benchDir: "/repo/bench/agent-bench", + envNames: ["ANTHROPIC_BASE_URL", "MINIMAX_API_KEY"], + uid: 1000, + gid: 1000, +}; + +test("dockerArgv: env rides as bare -e NAME — no secret ever lands in argv", () => { + const argv = dockerArgv(c, ["freecode", "run", "fix it"]); + assert.equal(argv.includes("MINIMAX_API_KEY"), true); + assert.equal(argv.some((a) => a.includes("MINIMAX_API_KEY=")), false); + assert.deepEqual(argv.slice(-3), ["freecode", "run", "fix it"]); + assert.equal(argv[argv.indexOf("--network") + 1], "agent-bench-internal"); + assert.equal(argv[argv.indexOf("--user") + 1], "1000:1000"); + assert.equal(argv.includes("/tmp/ws:/workspace"), true); + assert.equal(argv.includes("/repo/bench/agent-bench:/bench:ro"), true); +}); + +test("forwardedEnvNames: adapter names + meter names; \"\" (unset) is not forwarded", () => { + const names = forwardedEnvNames( + { XDG_CONFIG_HOME: "{benchDir}/empty-config", ANTHROPIC_API_KEY: "" }, + { MINIMAX_BASE_URL: "http://x", ANTHROPIC_BASE_URL: "http://x" }, + ); + assert.deepEqual(names, ["ANTHROPIC_BASE_URL", "MINIMAX_BASE_URL", "XDG_CONFIG_HOME"]); +}); diff --git a/bench/agent-bench/isolate/docker.ts b/bench/agent-bench/isolate/docker.ts new file mode 100644 index 00000000..581328b2 --- /dev/null +++ b/bench/agent-bench/isolate/docker.ts @@ -0,0 +1,102 @@ +// ============================================================================= +// Container isolation (spec §6.3). One container per trial on an --internal +// docker network: no route to the internet, so the only way out is the +// recording proxy listening on the network's gateway (the host side of the +// bridge). That makes the proxy log a real egress audit instead of an honor +// system — with the stated residue that OTHER host services on the gateway +// IP remain reachable; the network namespace blocks the internet, the proxy +// log still audits what was actually sent. +// +// Secrets never enter argv: env vars ride as bare `-e NAME` flags, which +// docker resolves from the spawning process's environment. argv.json stays +// publishable. +// ============================================================================= + +import { spawnSync } from "child_process"; + +export const IMAGE = "agent-bench"; +export const NETWORK = "agent-bench-internal"; +/** Where the workspace and bench dir land inside the container. */ +export const WORKSPACE = "/workspace"; +export const BENCH_MOUNT = "/bench"; + +export interface Containerize { + image: string; + network: string; + /** Unique per trial, so a timeout can `docker rm -f` exactly this one. */ + name: string; + /** Host workspace dir, mounted rw at /workspace. */ + wsDir: string; + /** Host bench/agent-bench dir, mounted ro at /bench (empty-config lives there). */ + benchDir: string; + /** Env NAMES to forward. Values come from the spawn env, never argv. */ + envNames: string[]; + uid: number; + gid: number; +} + +/** `docker run` argv around an agent's own argv. Pure — unit-testable. */ +export function dockerArgv(c: Containerize, argv: string[]): string[] { + return [ + "docker", "run", "--rm", "--init", + "--name", c.name, + "--network", c.network, + "--user", `${c.uid}:${c.gid}`, + "-v", `${c.wsDir}:${WORKSPACE}`, + "-v", `${c.benchDir}:${BENCH_MOUNT}:ro`, + "-w", WORKSPACE, + // Writable HOME for CLIs that insist on one; --user means /root is not it. + "-e", "HOME=/tmp/agent-home", + ...c.envNames.flatMap((n) => ["-e", n]), + c.image, + ...argv, + ]; +} + +/** The env names worth forwarding: the adapter's own, plus the meter's. */ +export function forwardedEnvNames( + adapterEnv: Record | undefined, + meterEnv: NodeJS.ProcessEnv | undefined, +): string[] { + const names = new Set(); + for (const [k, v] of Object.entries(adapterEnv ?? {})) { + if (v !== "") names.add(k); // "" means unset — simply do not forward it + } + for (const k of Object.keys(meterEnv ?? {})) names.add(k); + return [...names].sort(); +} + +function docker(args: string[]): { ok: boolean; out: string } { + const r = spawnSync("docker", args, { encoding: "utf-8" }); + return { ok: r.status === 0, out: (r.stdout ?? "").trim() }; +} + +export function dockerAvailable(): boolean { + return docker(["info", "--format", "{{.ServerVersion}}"]).ok; +} + +export function imageExists(image: string): boolean { + return docker(["image", "inspect", image, "--format", "ok"]).ok; +} + +/** + * Create the internal network if missing; return the gateway IP the proxy + * should bind. `--internal` is the isolation: docker programs no masquerade + * for it, so nothing routes past the bridge. + */ +export function ensureInternalNetwork(name: string): string { + const fmt = ["--format", "{{(index .IPAM.Config 0).Gateway}}"]; + let r = docker(["network", "inspect", name, ...fmt]); + if (!r.ok) { + const created = docker(["network", "create", "--internal", name]); + if (!created.ok) throw new Error(`docker network create ${name} failed`); + r = docker(["network", "inspect", name, ...fmt]); + } + if (!r.ok || !r.out) throw new Error(`no gateway on docker network ${name}`); + return r.out; +} + +/** Best-effort teardown after a timeout — the client dying does not stop a container. */ +export function removeContainer(name: string): void { + spawnSync("docker", ["rm", "-f", name], { stdio: "ignore" }); +} From 2b6a7a389da8ad510c098e0d8c0f3a6c82c480be Mon Sep 17 00:00:00 2001 From: Ayan De Date: Sun, 6 Sep 2026 04:22:10 +0530 Subject: [PATCH 05/10] feat(bench): wire metering and --isolate into the trial loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Metering on by default: each trial gets its own proxy, the runner injects the base-URL env, and the folded tokens/USD/turns/auditOk land in the trial record and the published matchup JSON. --isolate runs the trial in a container on the internal network (proxy bound on its gateway), reads the agent's version from the image, resolves {benchDir} to the ro mount, and docker-rm's a timed-out container — the docker client dying does not stop one. Also restores TrialRecord.artifactDir, which had been dropped while run.ts still assigned it: tsx executes without typechecking, so test:agent-bench now runs tsc -p bench/agent-bench first. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ELXKXKpNy38kvVfjwJC52f --- bench/agent-bench/runner/agents.ts | 36 ++++++++--- bench/agent-bench/runner/publish.ts | 14 +++++ bench/agent-bench/runner/run.ts | 98 ++++++++++++++++++++++++++--- bench/agent-bench/runner/types.ts | 11 ++++ bench/agent-bench/tsconfig.json | 18 ++++++ package.json | 4 +- 6 files changed, 163 insertions(+), 18 deletions(-) create mode 100644 bench/agent-bench/tsconfig.json diff --git a/bench/agent-bench/runner/agents.ts b/bench/agent-bench/runner/agents.ts index a5dd1cd0..16bce390 100644 --- a/bench/agent-bench/runner/agents.ts +++ b/bench/agent-bench/runner/agents.ts @@ -8,6 +8,11 @@ import { spawn, spawnSync } from "child_process"; import * as fs from "fs"; import * as path from "path"; +import { + dockerArgv, + removeContainer, + type Containerize, +} from "../isolate/docker.js"; import type { AgentSpec } from "./types.js"; const AGENT_DIR = path.join(import.meta.dirname, "..", "agents"); @@ -37,9 +42,14 @@ export function loadAgent(id: string): AgentSpec { * degrades a competitor while flattering us (spec §10.6) — so the version that * produced a number is part of the number. */ -export function agentVersion(spec: AgentSpec): string { - const [cmd, ...args] = spec.versionCmd; - const r = spawnSync(cmd!, args, { encoding: "utf-8", timeout: 30_000 }); +export function agentVersion(spec: AgentSpec, image?: string): string { + // Isolated trials run the IMAGE's copy of the agent, so that is the copy + // whose version belongs in the record — not whatever the host has. + const argv = image + ? ["docker", "run", "--rm", image, ...spec.versionCmd] + : spec.versionCmd; + const [cmd, ...args] = argv; + const r = spawnSync(cmd!, args, { encoding: "utf-8", timeout: 120_000 }); if (r.status !== 0) return "unknown"; return (r.stdout || r.stderr).trim().split("\n")[0]!.slice(0, 80); } @@ -65,7 +75,7 @@ function render(spec: AgentSpec, prompt: string): string[] { * `empty-config/` — the only way found to stop opencode loading the operator's * personal MCP servers (see agents/opencode.json). */ -export function resolveEnv(spec: AgentSpec): NodeJS.ProcessEnv { +export function resolveEnv(spec: AgentSpec, benchDir?: string): NodeJS.ProcessEnv { const env: NodeJS.ProcessEnv = { ...process.env }; for (const [key, raw] of Object.entries(spec.env ?? {})) { if (raw === "") { @@ -73,7 +83,8 @@ export function resolveEnv(spec: AgentSpec): NodeJS.ProcessEnv { continue; } env[key] = raw - .replaceAll("{benchDir}", path.join(AGENT_DIR, "..")) + // In a container, {benchDir} is the ro mount, not the host path. + .replaceAll("{benchDir}", benchDir ?? path.join(AGENT_DIR, "..")) .replace(/\$\{(\w+)\}/g, (_, name: string) => { const value = process.env[name]; if (!value) { @@ -109,9 +120,18 @@ export function runAgent( cwd: string, artifactDir: string, timeoutMs: number, + extraEnv?: NodeJS.ProcessEnv, + containerize?: Containerize, ): Promise { - const argv = render(spec, prompt); - const env = resolveEnv(spec); + // In a container, {benchDir} in the adapter env must resolve to the ro + // mount. The env still carries real values (docker copies them from the + // spawn env for each bare `-e NAME`); only argv stays value-free. + const env = { + ...resolveEnv(spec, containerize ? "/bench" : undefined), + ...extraEnv, + }; + const agentArgv = render(spec, prompt); + const argv = containerize ? dockerArgv(containerize, agentArgv) : agentArgv; const [cmd, ...args] = argv; const out = fs.createWriteStream(path.join(artifactDir, "stdout.log")); const err = fs.createWriteStream(path.join(artifactDir, "stderr.log")); @@ -130,6 +150,8 @@ export function runAgent( const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); + // Killing the docker CLIENT does not stop the container. + if (containerize) removeContainer(containerize.name); }, timeoutMs); const done = (exitCode: number | null) => { diff --git a/bench/agent-bench/runner/publish.ts b/bench/agent-bench/runner/publish.ts index 5a2977b9..d62f2bb5 100644 --- a/bench/agent-bench/runner/publish.ts +++ b/bench/agent-bench/runner/publish.ts @@ -65,6 +65,13 @@ export interface PublishedResult { reason: string; /** Which run this row came from. See the `runs` note below. */ runId: string; + turns?: number; + inputTokens?: number; + outputTokens?: number; + cacheReadTokens?: number; + cacheWriteTokens?: number; + usd?: number | null; + auditOk?: boolean; } export interface PublishedRun { @@ -171,6 +178,13 @@ export function publish(report: Report, fresh = false): string { newFiles: t.newFiles.length, reason: t.reason, runId: report.startedAt, + turns: t.turns, + inputTokens: t.inputTokens, + outputTokens: t.outputTokens, + cacheReadTokens: t.cacheReadTokens, + cacheWriteTokens: t.cacheWriteTokens, + usd: t.usd, + auditOk: t.auditOk, }); } diff --git a/bench/agent-bench/runner/run.ts b/bench/agent-bench/runner/run.ts index 1f518fda..289da8d2 100644 --- a/bench/agent-bench/runner/run.ts +++ b/bench/agent-bench/runner/run.ts @@ -2,10 +2,12 @@ // ============================================================================= // The trial loop. agents × instances × trials, one workspace each. // -// Phase 0: no container, no grader, no metering. It answers exactly one -// question — does every adapter produce a non-empty patch — and every record it -// writes carries `isolation: "none"` so it can never be mistaken for a -// publishable number. +// Metering (spec §6.4) is on by default: a pass-through proxy so every agent +// is billed by one table; `--no-meter` restores the adapter-only loop. +// `--isolate` (spec §6.3) runs each trial in a container on an --internal +// docker network whose only exit is that proxy — build the image first with +// `docker build -t agent-bench bench/agent-bench/isolate`. Grading is a +// separate, free-to-rerun step: `pnpm bench:grade results/`. // ============================================================================= import * as fs from "fs"; @@ -15,6 +17,17 @@ import { loadInstances, readIdList } from "./instances.js"; import { publish } from "./publish.js"; import { taskPrompt } from "./prompt.js"; import { createWorkspace, extractPatch, verifyWorkspace } from "./workspace.js"; +import { + IMAGE, + NETWORK, + dockerAvailable, + ensureInternalNetwork, + forwardedEnvNames, + imageExists, +} from "../isolate/docker.js"; +import { meterEnv, upstreamFor } from "../proxy/env.js"; +import { persistTrialMeter } from "../proxy/fold.js"; +import { startProxy } from "../proxy/server.js"; import type { Report, TrialRecord } from "./types.js"; const ROOT = path.join(import.meta.dirname, ".."); @@ -32,10 +45,31 @@ const instanceIds = arg("instances") : readIdList(path.join(ROOT, "instances", "django-lite.txt")); const runId = new Date().toISOString().replace(/[:.]/g, "-"); const outDir = arg("out", path.join(ROOT, "results", runId)) as string; +const meter = !process.argv.includes("--no-meter"); +const isolate = process.argv.includes("--isolate"); async function main() { + // Isolation implies metering: without the proxy the container has no route + // to the model at all, and an unmeterable run in a locked box is just a + // slower way to produce nothing. + let gateway: string | undefined; + if (isolate) { + if (!dockerAvailable()) { + throw new Error("--isolate needs docker (daemon up, user in the docker group)"); + } + if (!imageExists(IMAGE)) { + throw new Error( + `--isolate needs the "${IMAGE}" image. Build it (online) first:\n` + + ` docker build -t ${IMAGE} bench/agent-bench/isolate`, + ); + } + gateway = ensureInternalNetwork(NETWORK); + } + const agents = agentIds.map(loadAgent); - const versions = new Map(agents.map((a) => [a.id, agentVersion(a)])); + const versions = new Map( + agents.map((a) => [a.id, agentVersion(a, isolate ? IMAGE : undefined)]), + ); const instances = await loadInstances(instanceIds); fs.mkdirSync(outDir, { recursive: true }); @@ -45,7 +79,8 @@ async function main() { console.log(` ${"".padEnd(12)} autonomy: ${a.autonomy}`); } console.log( - ` ${instances.length} instance(s) × ${trials} trial(s) × ${agents.length} agent(s)\n`, + ` ${instances.length} instance(s) × ${trials} trial(s) × ${agents.length} agent(s)` + + ` meter=${meter || isolate ? "proxy" : "off"} isolation=${isolate ? "container" : "none"}\n`, ); const records: TrialRecord[] = []; @@ -65,6 +100,7 @@ async function main() { const ws = createWorkspace(inst); let record: TrialRecord; + let proxy: Awaited> | undefined; try { if (!verifyWorkspace(ws.dir, inst.baseCommit)) { throw new Error(`checkout is not at ${inst.baseCommit}`); @@ -72,12 +108,41 @@ async function main() { const prompt = taskPrompt(inst); fs.writeFileSync(path.join(artifactDir, "prompt.txt"), prompt); + const logPath = path.join(artifactDir, "proxy.jsonl"); + proxy = + meter || isolate + ? await startProxy({ + upstream: upstreamFor(spec), + logPath, + host: gateway, + }) + : undefined; + + const extraEnv = proxy ? meterEnv(proxy.origin) : undefined; + const containerize = isolate + ? { + image: IMAGE, + network: NETWORK, + name: `bench-${runId}-${inst.instanceId}-t${trial}-${spec.id}` + .toLowerCase() + .replace(/[^a-z0-9_.-]/g, "-") + .slice(0, 63), + wsDir: ws.dir, + benchDir: ROOT, + envNames: forwardedEnvNames(spec.env, extraEnv), + uid: process.getuid?.() ?? 1000, + gid: process.getgid?.() ?? 1000, + } + : undefined; + const run = await runAgent( spec, prompt, ws.dir, artifactDir, timeoutMs, + extraEnv, + containerize, ); fs.writeFileSync( path.join(artifactDir, "argv.json"), @@ -86,6 +151,7 @@ async function main() { const patch = extractPatch(ws.dir); fs.writeFileSync(path.join(artifactDir, "patch.diff"), patch.diff); + const usage = proxy ? persistTrialMeter(artifactDir, spec.model) : undefined; record = { agent: spec.id, @@ -94,7 +160,7 @@ async function main() { autonomy: spec.autonomy, instanceId: inst.instanceId, trial, - isolation: "none", + isolation: isolate ? "container" : "none", producedPatch: patch.diff.length > 0, reason: run.timedOut ? `timed out after ${timeoutMs}ms` @@ -107,6 +173,13 @@ async function main() { patchBytes: Buffer.byteLength(patch.diff), newFiles: patch.newFiles, artifactDir: path.relative(ROOT, artifactDir), + turns: usage?.turns, + inputTokens: usage?.inputTokens, + outputTokens: usage?.outputTokens, + cacheReadTokens: usage?.cacheReadTokens, + cacheWriteTokens: usage?.cacheWriteTokens, + usd: usage ? (usage.usd ?? null) : undefined, + auditOk: usage?.auditOk, }; } catch (err) { // One dead trial must not cost the rest of the matrix. @@ -117,7 +190,7 @@ async function main() { autonomy: spec.autonomy, instanceId: inst.instanceId, trial, - isolation: "none", + isolation: isolate ? "container" : "none", producedPatch: false, reason: `harness error: ${(err as Error).message}`.slice(0, 200), exitCode: null, @@ -128,14 +201,19 @@ async function main() { artifactDir: path.relative(ROOT, artifactDir), }; } finally { + await proxy?.close(); ws.cleanup(); } records.push(record); + const tokens = + record.inputTokens !== undefined + ? ` ${(record.inputTokens + (record.outputTokens ?? 0)).toLocaleString()}tok` + : ""; console.log( `${record.producedPatch ? "patch" : "EMPTY"} ` + `${String(record.patchBytes).padStart(6)}B ` + - `${(record.durationMs / 1000).toFixed(0)}s ${record.reason}`, + `${(record.durationMs / 1000).toFixed(0)}s${tokens} ${record.reason}`, ); } } @@ -144,7 +222,7 @@ async function main() { const report: Report = { startedAt: runId, finishedAt: new Date().toISOString(), - isolation: "none", + isolation: isolate ? "container" : "none", graded: false, trials: records, }; diff --git a/bench/agent-bench/runner/types.ts b/bench/agent-bench/runner/types.ts index 57a562fc..735a5768 100644 --- a/bench/agent-bench/runner/types.ts +++ b/bench/agent-bench/runner/types.ts @@ -60,7 +60,18 @@ export interface TrialRecord { patchBytes: number; /** Files the agent created that are not part of a fix — scratch-file noise. */ newFiles: string[]; + /** Per-trial artifact dump, relative to bench/agent-bench/. */ artifactDir: string; + /** Round-trips seen by the recording proxy. Absent when `--no-meter`. */ + turns?: number; + inputTokens?: number; + outputTokens?: number; + cacheReadTokens?: number; + cacheWriteTokens?: number; + /** MiniMax-M3 USD off the proxy; null when the model is unpriced. */ + usd?: number | null; + /** False when the proxy saw a request that was not the model endpoint. */ + auditOk?: boolean; } export interface Report { diff --git a/bench/agent-bench/tsconfig.json b/bench/agent-bench/tsconfig.json new file mode 100644 index 00000000..852455fc --- /dev/null +++ b/bench/agent-bench/tsconfig.json @@ -0,0 +1,18 @@ +{ + // tsx executes without typechecking, which is how a field dropped from + // TrialRecord while run.ts still assigned it went unnoticed. `pnpm + // test:agent-bench` runs `tsc --noEmit` over this config first. + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"], + // agent-bench is outside the pnpm workspace on purpose; borrow the + // @types/node that core already pins rather than growing a package.json. + "typeRoots": ["../../apps/core/node_modules/@types"] + }, + "include": ["runner/**/*.ts", "proxy/**/*.ts", "isolate/**/*.ts"] +} diff --git a/package.json b/package.json index 2808f190..cb229836 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,9 @@ "release": "pnpm build && pnpm publish -r --access public", "gen:catalogue": "node scripts/gen-provider-catalogue.mjs", "gen:catalogue:check": "node scripts/gen-provider-catalogue.mjs --check", - "bench:agents": "tsx bench/agent-bench/runner/run.ts" + "bench:agents": "tsx bench/agent-bench/runner/run.ts", + "bench:image": "docker build -t agent-bench bench/agent-bench/isolate", + "test:agent-bench": "tsc -p bench/agent-bench/tsconfig.json && tsx --test bench/agent-bench/**/*.test.ts" }, "devDependencies": { "@changesets/cli": "^2.31.0", From 5209b6a1471e9893a6732a9b519035d1ba3e611f Mon Sep 17 00:00:00 2001 From: Ayan De Date: Sun, 6 Sep 2026 04:22:28 +0530 Subject: [PATCH 06/10] =?UTF-8?q?feat(bench):=20official=20SWE-bench=20gra?= =?UTF-8?q?der=20=E2=80=94=20pnpm=20bench:grade?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verdict is the harness's, in Docker, on patch.diff — grade.ts only shuttles patches in and verdicts out. One harness invocation per (agent, trial): the harness keys on instance_id, so trials must not share a predictions file. An empty patch is resolved:false without spending a container; a harness error leaves null, which the page counts against rather than dropping. Verdicts merge into report.json (graded: true) and --publish re-publishes the matchup, flipping the page's headline from "Produced a patch" to "Resolved". Free to re-run: grading never touches a model. Needs docker + pip install swebench (SWEBENCH_PYTHON overrides). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ELXKXKpNy38kvVfjwJC52f --- bench/agent-bench/runner/grade.test.ts | 88 +++++++++++++++++++++++++ bench/agent-bench/runner/grade.ts | Bin 0 -> 7070 bytes bench/agent-bench/runner/publish.ts | 12 ++-- bench/agent-bench/runner/types.ts | 7 ++ package.json | 1 + 5 files changed, 102 insertions(+), 6 deletions(-) create mode 100644 bench/agent-bench/runner/grade.test.ts create mode 100644 bench/agent-bench/runner/grade.ts diff --git a/bench/agent-bench/runner/grade.test.ts b/bench/agent-bench/runner/grade.test.ts new file mode 100644 index 00000000..35102f23 --- /dev/null +++ b/bench/agent-bench/runner/grade.test.ts @@ -0,0 +1,88 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { applyVerdicts, parseHarnessReport, predictionsFor } from "./grade.js"; +import type { Report, TrialRecord } from "./types.js"; + +function trial(over: Partial): TrialRecord { + return { + agent: "freecode", + agentVersion: "0.30.0", + model: "minimax/MiniMax-M3", + autonomy: "danger", + instanceId: "django__django-1", + trial: 1, + isolation: "none", + producedPatch: true, + reason: "ok", + exitCode: 0, + timedOut: false, + durationMs: 1, + patchBytes: 10, + newFiles: [], + artifactDir: "results/run/django__django-1/trial-1/freecode", + ...over, + }; +} + +function report(trials: TrialRecord[]): Report { + return { startedAt: "run", finishedAt: "run", isolation: "none", graded: false, trials }; +} + +test("predictionsFor reads each trial's patch.diff and skips empty patches", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "grade-")); + const bench = path.join(root, "bench", "agent-bench"); + const resultsDir = path.join(bench, "results", "run"); + const artDir = path.join(resultsDir, "django__django-1", "trial-1", "freecode"); + fs.mkdirSync(artDir, { recursive: true }); + fs.writeFileSync(path.join(artDir, "patch.diff"), "diff --git a b\n"); + const rep = report([ + trial({}), + trial({ instanceId: "django__django-2", producedPatch: false, artifactDir: "nowhere" }), + trial({ agent: "opencode", artifactDir: "nowhere" }), + ]); + const preds = predictionsFor(rep, resultsDir, "freecode", 1); + assert.deepEqual(preds, [ + { + instance_id: "django__django-1", + model_name_or_path: "freecode", + model_patch: "diff --git a b\n", + }, + ]); + fs.rmSync(root, { recursive: true, force: true }); +}); + +test("parseHarnessReport: resolved, unresolved and error ids are graded; absent is not", () => { + const v = parseHarnessReport({ + resolved_ids: ["a"], + unresolved_ids: ["b"], + error_ids: ["c"], + }); + assert.deepEqual([...v.resolved], ["a"]); + assert.deepEqual([...v.graded].sort(), ["a", "b", "c"]); + const empty = parseHarnessReport({ something: 1 }); + assert.equal(empty.resolved.size, 0); +}); + +test("applyVerdicts: empty patch fails without the harness; unlisted stays null", () => { + const rep = report([ + trial({ instanceId: "a" }), + trial({ instanceId: "b" }), + trial({ instanceId: "c", producedPatch: false }), + trial({ instanceId: "d" }), + trial({ instanceId: "a", agent: "opencode" }), + ]); + applyVerdicts(rep, "freecode", 1, { + resolved: new Set(["a"]), + graded: new Set(["a", "b"]), + }); + const by = (id: string, agent = "freecode") => + rep.trials.find((t) => t.instanceId === id && t.agent === agent)!; + assert.equal(by("a").resolved, true); + assert.equal(by("b").resolved, false); + assert.equal(by("c").resolved, false); // empty patch: no harness needed + assert.equal(by("d").resolved, null); // harness errored / never ran + assert.equal(by("a", "opencode").resolved, undefined); // other slice untouched +}); diff --git a/bench/agent-bench/runner/grade.ts b/bench/agent-bench/runner/grade.ts new file mode 100644 index 0000000000000000000000000000000000000000..1e0769f29acac0f3c3cc4e4c269d515b94f9d9dc GIT binary patch literal 7070 zcmb_h?T*{V70vH`ifh71NJpaL^aJhMyN1))iHp`Yyo(0ycDv$|qZPB%kl_p^tBQdB zEBd$SD-?Z?K1!aX=iZs2DD5Uefh_E`M9$o~pXZ)Co=oWEr~EmZOz3>BsGiNryv!;( z`|8E`TIqaFH*Hp^c0{W-YZ_Igbk?Yx{`;R_roT8Qi#?ng2c#U zG17gWWH(A%XZ)m9rmL)(JZ(FDN*CktB6~kx)h!G%^!k#YG~IPon)#&{`vtqp5Zkq; zSY(aJw(x1f>a)Wj>mx(0YUsIPzooANKP#I#tuHv2Ev2?4}FkWMw zv6;@*RauP4)MQuq%V**3EJVX(3)QcrMQ1GBWZ~beRg_WFDh?mca^6qro%*ILVY!c- zxZFo}YF0`-HbqSGTbM`jv?&`WzpAKz51U$Q>QY_(HZ~u7y<(t=ph-7T# zQt2g>#MgO-*xDqe&uTj-)+?VYWKDsP%xYFsukF0lH=JavwzO7hj^j^U1V`n9V?rA; zO}5f!Yn{`UW^KKoD4&;Aan-bSj?9jpgr$!Ns?YF87R?UdAaUpZ9e>>I+mO$j{(yAi zpCBc6>+HHw(C3{b%ZSc7xbIY6x5Z#1FEVc15lR68{`~p*^RpM{bV5#?qy zNnh7jzbmc6Ic~j=%n7nV3``N4^e}GUehX=cL#=;?jYsOeI13S0?JUa`z3nrJHb>N_ z9Fp^x4AN5HJi+5dU8w3xBPp-y_KLl7_pQ|X?(z2MX!7V0y+M%-v$d^P)Brw06^sbE zBBgXtccvr*alch14%t?Y3i(}FjgkO-w`&uFtv7(5-Cc|oIa zm`HTn;1k|<8;T8jxKmM8C#RAC&)YUzr=^j9kpTI{?PWsG$baNxyb%|dNndPBNLq3p zJ8@EZKvHzhrBqquo<|8C)5Rs580>Hng;rXv*bOmsjHBQgj$*_K6E--{43=M{Dbmk?WCU(??_E-A~(Lp8j67=+I0q?n6Y75YOG1S)qo7oH$y7 zoFmSqwc}o#XK+-X%*V*2%BU%=O7sKptjyeUdmfUMKw8B5mR5CF70`vLUAAU{yjaMI zYwZl)4?h7U7?8o|J=YQ2ne1dczJo>C66H44T~$r*PiL^zYW)YtP0jtt{$m(Uy?>DU zy{q>#{JylOt>FK%E(=M2wk&qJW0{ya$Z&XW5?KO(o>3ft|3MG@=(z#-`Jn)RB7C>h zo#GU8J0I9|zY}0v$S@?HM9ZxP-hvL|un_xxT?!fo6?Uy5N%p1KHXM0Zh|BRG6>+G@ zo|YUkzCqC-Q9kj3bRLhJsy%V-?n>?bF4aDs9C9@nUrMDIfUARE4Aty z6Eg&bG;tYH$hd@<^&B*g+%t5ti>TDInp@g6vQbE2O&tk>7432NN7- zw>L#Mu$>`R?30jkcFq)vG3*6*pU_PGaqL6x8R)n9JiEqR7IVe+BciWsOfw80epo=a zis)lWOHR$`l?;VcWNLvD?wXNI6OfFW2Phk%ZxS_c_f1=O%`@MB1?#pwjyCH(w>Z+u zP8sLy2G8C@_Z;}3S^q!wPakfa=q=@$W)~btPU*gpM~YR}s&7v%xp|q=A=xYmUY5LpCZb)-WXsopHo7=Bu7|j|gMyS8i$z4E1i}A$j-Q zE2?-O%jXOEji8Pf?C#)a)Vz%QDbBo7%dF~Ta_uP;Qnkoz2G0oV$5ua`1dfFT16nU5 z0T%HiF>0`kPbdq#6!QizVBeJ=%klOMh@o%dozb4fUAi3`%2)7B@FON?Ta@)N5=hS# zhRcZYn@FO-6$Hl|q|agDOuY+8y~k8l(8JAc4+cnXY#UK|J9S{dQd!Ic3QHL|YfQ6>BDDEZ)fYbYJ)z((=?Tu&CD2ay~-XsoV(kV#%+p{;X z(}sC!d^ia3VkD(w@1Cvu3AHaG(f?gH91(Wz*=0*8kr5U3TX(zD_XRl0apQrXoX#(y z0P1JwEu;Y79l1Hgtq~_h;-o+&WrjYXAFymEeqf(t&b*l$rV|#z^Zi=z3RH&E03KZ2 z7G)~|3VTOFbaGtCVXnjd9y&g4p>ThQ%+d@L%wYrl_0~zFhSc5oi9GR~X8Qqulog0+ zB6NDdi{!$}$qwDg8Fk@iP{({w*x>wxa%`&7#?c4PlHdSG6i(>1%;2$n#TM~*SR=5) z9_lYE%uBRBX90j^5VYaRdorYyAp_P)xUT_jJpVCL_>d>taVU|oh2?y2vz>x!H5IN( z;>pEdo{#^OjsN<|_}5qC%gGHwJKi^@$3Ir~he~iK9~7{^%ac^A7;d-^Bh~^joTL)( zUb+Wb7qmurjO3!5eahK7Zj!_e*3q9eW}%Ns{&jL09z|4IDSd;xu2W_g@Xukh8sfP} z??F@A^KRg!bRw$bo|Zeo-~j)aFyILm>;vETLm{{aU+wztk+{ne_Rc=L_emW_+Zc10 zR0mqdomV_=v4s6$BP7!>fkuJ08Zj2VJ=`^=PCCMTNK0kqozOYy9LbS`ja{h1!Ay)t z^yyx(92fL>CL;vqPnA~n4Q_B9%1P()pM6Cn`vB|?5I*Zd?`{jT_GWqU_>!-w992pe z%4TWTR=7wk%vTr^6mpZjItcxPkAiVz>d>zdN0)co!5XE~6K&KKlO#x&Pk72SZTM^!G3R zveR9;3 Date: Sun, 6 Sep 2026 04:22:44 +0530 Subject: [PATCH 07/10] =?UTF-8?q?feat(bench):=20evidence=20bundle=20?= =?UTF-8?q?=E2=80=94=20pnpm=20bench:bundle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One reproducible tar.gz per run (sorted, owner/mtime pinned — same evidence, same bytes) plus a sha256 sidecar: report, prompts, argv, patches, agent stdout/stderr, proxy logs, usage/audit folds, grading output. The proxy log carries no headers or bodies, so the bundle is publishable as-is; the checksum is the claim anyone can re-derive. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ELXKXKpNy38kvVfjwJC52f --- bench/agent-bench/runner/bundle.test.ts | 39 +++++++++++++++ bench/agent-bench/runner/bundle.ts | 63 +++++++++++++++++++++++++ package.json | 1 + 3 files changed, 103 insertions(+) create mode 100644 bench/agent-bench/runner/bundle.test.ts create mode 100644 bench/agent-bench/runner/bundle.ts diff --git a/bench/agent-bench/runner/bundle.test.ts b/bench/agent-bench/runner/bundle.test.ts new file mode 100644 index 00000000..510c4d95 --- /dev/null +++ b/bench/agent-bench/runner/bundle.test.ts @@ -0,0 +1,39 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import * as crypto from "crypto"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { bundleRun } from "./bundle.js"; + +test("bundleRun tars the run dir and writes a matching, reproducible sha256", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "bundle-")); + const run = path.join(root, "2026-09-06T00-00-00"); + fs.mkdirSync(path.join(run, "django__django-1", "trial-1", "freecode"), { + recursive: true, + }); + fs.writeFileSync(path.join(run, "report.json"), "{}"); + fs.writeFileSync( + path.join(run, "django__django-1", "trial-1", "freecode", "patch.diff"), + "diff\n", + ); + + const a = bundleRun(run); + assert.equal(fs.existsSync(a.tarball), true); + const recomputed = crypto + .createHash("sha256") + .update(fs.readFileSync(a.tarball)) + .digest("hex"); + assert.equal(a.sha256, recomputed); + assert.equal( + fs.readFileSync(`${a.tarball}.sha256`, "utf-8").startsWith(a.sha256), + true, + ); + + // Same content, same bytes — the published checksum must be reproducible. + const b = bundleRun(run); + assert.equal(b.sha256, a.sha256); + + assert.throws(() => bundleRun(root), /no report\.json/); + fs.rmSync(root, { recursive: true, force: true }); +}); diff --git a/bench/agent-bench/runner/bundle.ts b/bench/agent-bench/runner/bundle.ts new file mode 100644 index 00000000..9c0809b8 --- /dev/null +++ b/bench/agent-bench/runner/bundle.ts @@ -0,0 +1,63 @@ +// ============================================================================= +// The evidence bundle (spec §8/§10: published, not managed). One tar.gz per +// run — report.json, and per trial the prompt, exact argv, patch, agent +// stdout/stderr, proxy log, folded usage and isolation audit, plus any +// grading output. The proxy log carries paths and token counts only (no +// headers, no bodies), so the bundle is publishable as-is. +// +// pnpm bench:bundle bench/agent-bench/results/ +// +// Writes -evidence.tar.gz + .sha256 next to the run dir. The checksum +// is the claim: anyone holding the tarball can verify it is the one the +// numbers came from. +// ============================================================================= + +import { spawnSync } from "child_process"; +import * as crypto from "crypto"; +import * as fs from "fs"; +import * as path from "path"; + +export interface Bundle { + tarball: string; + sha256: string; +} + +export function bundleRun(resultsDir: string): Bundle { + const dir = path.resolve(resultsDir); + if (!fs.existsSync(path.join(dir, "report.json"))) { + throw new Error(`${dir} has no report.json — not a finished run`); + } + const name = path.basename(dir); + const tarball = path.join(path.dirname(dir), `${name}-evidence.tar.gz`); + // --sort=name + owner/mtime pinning: the same evidence tars to the same + // bytes, so the published sha256 is reproducible, not an accident of umask. + const r = spawnSync( + "tar", + [ + "--sort=name", "--owner=0", "--group=0", "--numeric-owner", + "--mtime=@0", + "-czf", tarball, + "-C", path.dirname(dir), + name, + ], + { stdio: "inherit" }, + ); + if (r.status !== 0) throw new Error(`tar exited ${r.status}`); + + const sha256 = crypto + .createHash("sha256") + .update(fs.readFileSync(tarball)) + .digest("hex"); + fs.writeFileSync(`${tarball}.sha256`, `${sha256} ${path.basename(tarball)}\n`); + return { tarball, sha256 }; +} + +if (process.argv[1] && import.meta.url.endsWith(path.basename(process.argv[1]))) { + const dir = process.argv[2]; + if (!dir) { + console.error("usage: bundle.ts "); + process.exit(1); + } + const b = bundleRun(dir); + console.log(`${b.tarball}\nsha256 ${b.sha256}`); +} diff --git a/package.json b/package.json index ef777882..cde66faf 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "bench:agents": "tsx bench/agent-bench/runner/run.ts", "bench:grade": "tsx bench/agent-bench/runner/grade.ts", "bench:image": "docker build -t agent-bench bench/agent-bench/isolate", + "bench:bundle": "tsx bench/agent-bench/runner/bundle.ts", "test:agent-bench": "tsc -p bench/agent-bench/tsconfig.json && tsx --test bench/agent-bench/**/*.test.ts" }, "devDependencies": { From 360579b0c41cba37d644f190ccd87adacf328e0b Mon Sep 17 00:00:00 2001 From: Ayan De Date: Sun, 6 Sep 2026 04:22:44 +0530 Subject: [PATCH 08/10] feat(web): token and cost columns on /benchmark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Token and cost bars off the one meter, with the traps §7 names handled in the rendering: an unmetered agent shows "unmetered", never $0; worst single-trial cost sits beside the mean; and the §7.2 cost-on-solved-intersection line appears only once the matchup is graded, suppressed below 3 shared resolved instances. A caveat lists per-agent metering coverage whenever any trial went unmetered. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ELXKXKpNy38kvVfjwJC52f --- apps/web/app/components/AgentBenchmark.tsx | 67 +++++++++++++- apps/web/app/data/agent-bench.ts | 101 +++++++++++++++++++++ 2 files changed, 167 insertions(+), 1 deletion(-) diff --git a/apps/web/app/components/AgentBenchmark.tsx b/apps/web/app/components/AgentBenchmark.tsx index efd00e42..5a8e3b28 100644 --- a/apps/web/app/components/AgentBenchmark.tsx +++ b/apps/web/app/components/AgentBenchmark.tsx @@ -7,6 +7,9 @@ import { BenchBarList, type BenchBar } from "./BenchBarList"; const pct = (n: number) => `${Math.round(n * 100)}%`; const secs = (ms: number) => `${(ms / 1000).toFixed(0)}s`; +const tok = (n: number) => + n >= 1_000_000 ? `${(n / 1_000_000).toFixed(1)}M` : `${Math.round(n / 1000)}k`; +const usd = (n: number) => `$${n.toFixed(4)}`; export function AgentBenchmark({ views }: { views: BenchView[] }) { const [slug, setSlug] = useState(views[0]?.slug ?? ""); @@ -52,6 +55,38 @@ export function AgentBenchmark({ views }: { views: BenchView[] }) { highlight: a.isFreeCode, })); + // Metering (spec §6.4/§7): shown only for agents the proxy actually saw. + // An unmetered agent gets a zero-width bar labelled as such, never a zero — + // a blank is not the cheapest run in the table. + const anyMetered = view.agents.some((a) => a.meteredTrials > 0); + const tokenBars: BenchBar[] = view.agents.map((a) => ({ + label: a.id, + value: a.meanTokens ?? 0, + display: a.meanTokens !== undefined ? tok(a.meanTokens) : "unmetered", + note: + a.meteredTrials > 0 + ? `mean of ${a.meteredTrials} metered trial${a.meteredTrials === 1 ? "" : "s"}${a.meanTurns !== undefined ? ` · ~${Math.round(a.meanTurns)} turns` : ""}` + : "proxy saw no traffic", + highlight: a.isFreeCode, + })); + const costBars: BenchBar[] = view.agents.map((a) => ({ + label: a.id, + value: typeof a.meanUsd === "number" ? a.meanUsd : 0, + display: + typeof a.meanUsd === "number" + ? usd(a.meanUsd) + : a.meanUsd === null + ? "unpriced" + : "unmetered", + note: + a.worstUsd !== undefined + ? `worst single trial ${usd(a.worstUsd)}` + : a.meteredTrials > 0 + ? "no rate-card row for this model" + : "proxy saw no traffic", + highlight: a.isFreeCode, + })); + // Headline cards, freecode against the strongest rival on each axis. The // comparison is deliberately unflattering — "slower" goes in the headline in // red, because a benchmark we publish only when we win is an advertisement. @@ -329,6 +364,36 @@ export function AgentBenchmark({ views }: { views: BenchView[] }) { bars={sizeBars} /> + {anyMetered && ( + + )} + + {anyMetered && ( + + `${p.id} ${typeof p.meanUsd === "number" ? usd(p.meanUsd) : "unpriced"}${p.meanTokens !== null ? ` (${tok(p.meanTokens)} tok)` : ""}`, + ) + .join(" · ")}. Averaging over failures would make the quitter cheapest — this mean covers solved bugs only.` + : "Per-solved-bug cost (spec §7.2) appears once the matchup is graded — averaging cost over failed attempts would make the agent that gives up fastest look cheapest." + } + /> + )} +

Per instance

@@ -357,7 +422,7 @@ export function AgentBenchmark({ views }: { views: BenchView[] }) { {row.cells.map((cell) => ( { return s.length % 2 ? s[mid]! : (s[mid - 1]! + s[mid]!) / 2; }; +const mean = (xs: number[]) => + xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : undefined; + +const totalTokens = (r: RawResult) => + r.inputTokens !== undefined ? r.inputTokens + (r.outputTokens ?? 0) : undefined; + export function deriveView(raw: RawBenchmark): BenchView { const { results } = raw; @@ -125,6 +169,8 @@ export function deriveView(raw: RawBenchmark): BenchView { const successes = mine.filter((r) => raw.graded ? r.resolved === true : r.producedPatch, ).length; + const metered = mine.filter((r) => r.inputTokens !== undefined); + const usds = metered.map((r) => r.usd).filter((u): u is number => typeof u === "number"); return { id: a.id, isFreeCode: a.id === "freecode", @@ -136,6 +182,13 @@ export function deriveView(raw: RawBenchmark): BenchView { rate: mine.length ? successes / mine.length : 0, medianMs: median(mine.map((r) => r.durationMs)), medianPatchBytes: median(mine.map((r) => r.patchBytes)), + meteredTrials: metered.length, + meanTokens: mean(metered.map((r) => totalTokens(r)!)), + // Metered but unpriced (no rate-card row) is null, not absent — the + // distinction between "we didn't measure" and "we can't price". + meanUsd: metered.length ? (usds.length ? mean(usds)! : null) : undefined, + worstUsd: usds.length ? Math.max(...usds) : undefined, + meanTurns: mean(metered.map((r) => r.turns ?? 0)), }; }); @@ -158,9 +211,46 @@ export function deriveView(raw: RawBenchmark): BenchView { patchBytes: r.patchBytes, reason: r.reason, trial: r.trial, + tokens: totalTokens(r), + usd: r.usd, + auditOk: r.auditOk, })), })); + /** + * §7.2, executed: cost is compared only on instances every agent resolved + * at least once, and suppressed entirely below 3 such instances. Only a + * graded matchup can have this table at all. + */ + let cost: CostComparison | undefined; + if (raw.graded) { + const solvedByAll = sharedInstances.filter((id) => + raw.agents.every((a) => + results.some((r) => r.agent === a.id && r.instanceId === id && r.resolved === true), + ), + ); + cost = { + instances: solvedByAll, + suppressed: solvedByAll.length < 3, + perAgent: raw.agents.map((a) => { + const mine = results.filter( + (r) => + r.agent === a.id && + solvedByAll.includes(r.instanceId) && + r.resolved === true && + r.inputTokens !== undefined, + ); + const usds = mine.map((r) => r.usd).filter((u): u is number => typeof u === "number"); + return { + id: a.id, + isFreeCode: a.id === "freecode", + meanUsd: usds.length ? mean(usds)! : null, + meanTokens: mine.length ? mean(mine.map((r) => totalTokens(r)!))! : null, + }; + }), + }; + } + /** * Everything not yet true about this matchup, in the page's own words. Spec * §9 wants the caveats on the page rather than in a footnote and §10 wants @@ -184,6 +274,16 @@ export function deriveView(raw: RawBenchmark): BenchView { }, ] : []), + ...(agents.some((a) => a.trials > 0 && a.meteredTrials < a.trials) + ? [ + { + title: "Not every trial was metered", + body: `Token and cost figures cover only trials the recording proxy saw (${agents + .map((a) => `${a.id}: ${a.meteredTrials}/${a.trials}`) + .join(", ")}). An agent that ignores the proxy env vars is unmetered until the container forces its egress through the proxy — comparing its cost column would compare a measurement to a blank.`, + }, + ] + : []), ...(raw.runs.length > 1 ? [ { @@ -233,6 +333,7 @@ export function deriveView(raw: RawBenchmark): BenchView { sharedInstances, ragged, matrix, + cost, caveats, }; } From 6b4e75bb780c2e8102e0be47e520f74fc76ae252 Mon Sep 17 00:00:00 2001 From: Ayan De Date: Sun, 6 Sep 2026 04:22:44 +0530 Subject: [PATCH 09/10] docs: AGENT-BENCH.md operator reference for the full pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bench/agent-bench README collapses to a pointer at the repo-root AGENT-BENCH.md (same job as EVAL.md/TRACE.md): setup, metering and the wire conventions it normalizes, --isolate, bench:grade, bench:bundle, when to run what, and §8's honest status — every layer exists, but isolation and the grader still need their first live smoke run (docker group + pip install swebench), and a publishable number is a run that used all of them. Benchmark.md becomes the instrument picker; CLAUDE.md/AGENTS.md/EVAL.md/ README.md rows updated. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ELXKXKpNy38kvVfjwJC52f --- AGENT-BENCH.md | 452 ++++++++++++++++++++++++++++++++++++ AGENTS.md | 3 +- Benchmark.md | 17 +- CLAUDE.md | 3 +- EVAL.md | 1 + README.md | 3 +- bench/agent-bench/README.md | 163 +------------ 7 files changed, 480 insertions(+), 162 deletions(-) create mode 100644 AGENT-BENCH.md diff --git a/AGENT-BENCH.md b/AGENT-BENCH.md new file mode 100644 index 00000000..14d791ea --- /dev/null +++ b/AGENT-BENCH.md @@ -0,0 +1,452 @@ +# Agent-bench — command reference + +> Operator's guide to the **agent comparison** harness: what each command does, +> which flag to reach for, and when to run it. Design lives in +> `docs/superpowers/specs/2026-09-03-agent-comparison-benchmark.md`; this is +> the "what do I type" page. +> +> Inspired by [Superbrain's public benchmark](https://www.onesuperbrain.com/benchmarks): +> same tasks, same model, same key, one meter, official grader, published +> artifacts. We are not there yet — see §8. + +This is **not** `pnpm eval`. That measures *our* agent against its own past +(`EVAL.md`). This measures agents against each other, on tasks we did not +write, and shares no code with `apps/core/src/eval` on purpose — only +`scorers/outcome.ts`'s idea survives the trip, because it is the only scorer +that never asks what produced the diff. + +This is **not** `pnpm bench:memory`. That measures PSS and time-to-first-frame +(`Benchmark.md`). A fast TUI that fixes no bugs is a different claim. + +| Question | Command | Doc | +| --- | --- | --- | +| Did my last change make *our* agent worse? | `pnpm eval` | `EVAL.md` | +| How much RAM, how fast to first frame? | `pnpm bench:memory` | `Benchmark.md` | +| **Does it fix real bugs vs Claude Code / OpenCode, and for how much?** | **`pnpm bench:agents`** | **this file** | + +**Status (2026-09-06).** The full pipeline exists: metering (on by default), +container isolation (`--isolate`), the official SWE-bench grader +(`pnpm bench:grade`), §7.2 cost columns on `/benchmark`, and the evidence +bundle (`pnpm bench:bundle`). **A publishable number is a run with all of +them**: `--isolate --trials 3`, then grade, then bundle. A run missing any +step is labelled provisional by the page and should stay that way. Grader and +isolation need Docker (daemon up, operator in the docker group) and the +grader needs `pip install swebench` — neither has had a paid smoke run yet; +`pnpm test:agent-bench` covers everything that is free. There is still no +gate, no CI wiring, no exit-on-regression — same reasoning as `eval ab`. + +Code lives in `bench/agent-bench/`, sibling to `bench/jcode-bench/`, outside +the pnpm workspace on purpose. + +--- + +## 1. One-time setup + +```bash +pnpm exec tsx bench/agent-bench/runner/fetch.ts +export MINIMAX_API_KEY=$(node -p "require(process.env.HOME+'/.freecode/config.json').providers.minimax.apiKey") +``` + +Every shipped adapter runs **MiniMax-M3 on that one key** — freecode natively, +the others through MiniMax's Anthropic-compatible endpoint. That is the "same +model, one bill" property (spec §5) and it is the only reason a cost column +would mean anything. No adapter file contains a credential; `${MINIMAX_API_KEY}` +is expanded at spawn time and an **unset variable is a hard error**, because an +agent that quietly fell back to its own key would be billed somewhere else. + +`fetch.ts` is needed **once, ever**. It pulls the django subset of SWE-bench +Lite into `bench/agent-bench/.cache/instances.jsonl` (114 rows). datasets-server +500s while its index warms; the fetch backs off five times and then stops, +leaving any existing cache untouched. A run whose instances are already cached +does not touch the Hub at all. + +The cache stores four fields per instance. `patch`, `test_patch` and +`hints_text` — the gold fix and the maintainer discussion that usually contains +it — are **dropped before anything touches disk**. Not to protect the agent +under test (it cannot see this repo): to keep an answer key out of a repository +agents work in every day. + +The django mirror (~250 MB) is cloned on first use into +`bench/agent-bench/.cache/repos/` and hardlinked per trial, so a run does not +measure GitHub's mood. + +`.cache/` and `results/` are git-ignored. + +--- + +## 2. Run a comparison — `pnpm bench:agents` + +```bash +# Smoke: one instance, one trial, default agents (freecode, claude-code) +pnpm bench:agents --instances django__django-10914 --trials 1 + +# The shape Phase 2 wants: N=3, publish the spread +pnpm bench:agents --agents freecode,claude-code --trials 3 + +# Skip the recording proxy (adapter debugging only) +pnpm bench:agents --no-meter --instances django__django-10914 +``` + +Every invocation is a **real paid agent turn × agents × instances × trials**. +Read this file before you spend. + +| Flag | Default | Does what | +| --- | --- | --- | +| `--agents` | `freecode,claude-code` | comma-separated adapter ids from `bench/agent-bench/agents/` | +| `--instances` | every id in `instances/django-lite.txt` (10 django bugs) | comma-separated SWE-bench instance ids | +| `--trials` | `1` | trials per (agent, instance). Phase 2 onward: **3**, and publish the spread. Never merge into best-of | +| `--timeout` | `900000` (15 min) | per-trial wall-clock cap, ms. Timeout SIGKILLs the agent | +| `--out` | `bench/agent-bench/results/` | artifact root | +| `--fresh` | off | passed through to publish: discard the matchup JSON and start over | +| `--no-meter` | off | skip the recording proxy | +| `--isolate` | off | one container per trial on an `--internal` docker network (§6.3); implies metering. Needs the image: `pnpm bench:image` (online, once) | + +**Exit code** is non-zero if **any trial produced an empty patch**. In Phase 0 +that is the entire verdict: a silently empty patch is a broken adapter, and a +broken adapter that reports as a lost benchmark is the worst failure this +harness has. There is **no** gate against a baseline and **no** CI job. Do not +wire one. + +The matrix is `instances × trials × agents`, nested in that order. One dead +trial does not abort the rest. + +### Task set + +`instances/django-lite.txt` is the **first ten django instance ids in lexical +order**. Chosen mechanically, and said out loud so nobody has to take it on +faith that they were not cherry-picked. Ten instances from one repo is a +**demo, not a leaderboard** (spec §10.1). SWE-bench Lite is 300 across 11 +repos. + +The prompt is one string, identical for every agent (`runner/prompt.ts`). If it +ever differs per agent, the benchmark stops comparing harnesses and starts +comparing prompts we wrote for them. + +--- + +## 3. Metering — one meter, not four self-reports + +On by default. Each trial starts a **pass-through HTTP proxy** +(`bench/agent-bench/proxy/`) and points the agent at it: + +| Agent | Env the runner injects | Upstream | +| --- | --- | --- | +| Claude Code | `ANTHROPIC_BASE_URL=http://127.0.0.1:` | adapter's `ANTHROPIC_BASE_URL` (MiniMax `/anthropic`) | +| freecode | `MINIMAX_BASE_URL=http://127.0.0.1:/v1` | same MiniMax origin; `baseURLFor()` in `catalogue.ts` honours `$_BASE_URL` | +| OpenCode | those two vars, which it **does not honour** | unmetered without `--isolate`; in a container the internal network leaves it no other route | + +The proxy: no cache, no retries, no body rewrite. Tokens are parsed off +Anthropic Messages JSON/SSE and OpenAI Chat Completions. Anthropic +`message_delta` only carries `output_tokens`; input and cache reads stay from +`message_start`. Secrets are not written to the log (no headers, no bodies). + +**Conventions differ between the two wire shapes and the meter normalizes +them**: OpenAI's `prompt_tokens` already includes `cached_tokens`, but +Anthropic's `input_tokens` EXCLUDES `cache_read/creation_input_tokens` — +they are separate additive fields. `usage.ts` folds both into one inclusive +`inputTokens`, which is what makes `price.ts`'s cache discount arithmetic +correct for either shape. If a shim turns out to report inclusive +`input_tokens` (verify on the first metered run: fresh input should be +`input_tokens`, small, beside large cache fields), cost is overstated by the +cache amount, never silently understated. + +**USD** uses a committed rate card in `proxy/price.ts`, vintage +`2026-09 MiniMax standard ≤512k`: + +| Model | Input / M | Output / M | Cache read / M | +| --- | ---: | ---: | ---: | +| MiniMax-M3 | $0.30 | $1.20 | $0.06 | + +Cache reads are a **discount off the inclusive input count**, matching +`providers/pricing.ts`. Adding them on top would report a cache win as a cost +increase. An unknown model prices as **`undefined` / `null`, never as zero**. + +A request whose path is not `/messages` or `/chat/completions` is a **leak**. +An **empty** proxy log is unmetered (`auditOk: false`), not a cheap clean run — +that is how an agent that talked *around* the proxy is supposed to look. + +Without a container, the proxy only sees traffic **pointed at it**. A clean +audit is not proof that nothing else left the machine. `isolation` stays +`"none"` until Docker lands. + +Pinning a **different shared model** is a different experiment, which is fine +if every adapter's `"model"` and the proxy upstream still match. Tokens will +record for any Anthropic- or OpenAI-shaped endpoint; USD stays null until you +add a row to `proxy/price.ts`. Do not mix bills (Gemini on one side, MiniMax on +the other) and call it a cost comparison. + +### Isolation — `--isolate` + +One container per trial (`bench/agent-bench/isolate/`), on a docker network +created with `--internal`: docker programs no route out, so the only exit is +the recording proxy, which binds the network's gateway IP on the host. The +proxy log stops being an honor system and becomes the egress audit. Known +residue, stated rather than hidden: other host services on the gateway IP +remain reachable from the container; the internet does not. + +```bash +pnpm bench:image # build agent-bench (online, once) +pnpm bench:agents --isolate --trials 3 --agents freecode,claude-code +``` + +The image (`isolate/Dockerfile`) bakes pinned versions of every agent — +freecode from a **released** binary (build arg `FREECODE_VERSION`, default +v0.30.0), the others from npm — because at trial time there is no network to +install anything with. `agentVersion` is read from the image, `$HOME` is a +tmpfs-style throwaway inside the container (no memory between trials, §6.3), +the workspace mounts rw at `/workspace`, the bench dir ro at `/bench` +(`{benchDir}` resolves there), and env values ride bare `-e NAME` flags so +`argv.json` never contains a secret. A timed-out trial gets `docker rm -f`, +not just a dead client. + +--- + +## 3b. Grading — `pnpm bench:grade` + +```bash +pip install swebench # once; or SWEBENCH_PYTHON=/path/to/venv/python3 +pnpm bench:grade bench/agent-bench/results/ [--max-workers N] [--publish] +``` + +The verdict is the **official SWE-bench harness's, in Docker, on +`patch.diff`** — `runner/grade.ts` only shuttles patches in and verdicts out. +One harness invocation per (agent, trial), because the harness keys on +`instance_id` and two trials of one instance must not share a predictions +file. An empty patch is `resolved: false` without spending a container; a +harness error leaves `null`, which the page counts as not-resolved rather +than quietly dropping. Verdicts land in `report.json` (`resolved` per trial, +`graded: true`) and `--publish` re-publishes the matchup — at which point the +page's headline flips from "Produced a patch" to "Resolved" and the §7.2 +cost-on-intersection line appears. Grading is free to re-run: it never +touches a model. + +## 3c. Evidence bundle — `pnpm bench:bundle` + +```bash +pnpm bench:bundle bench/agent-bench/results/ +``` + +Writes `-evidence.tar.gz` + `.sha256` next to the run dir: report, +prompts, argv, patches, agent stdout/stderr, proxy logs, usage and audit +folds, grading output. The tar is byte-reproducible (sorted, owner/mtime +pinned), so the published checksum is a claim anyone can re-derive. Publish +the tarball wherever the numbers go; the proxy log carries no headers or +bodies, so nothing in it needs redacting. + +--- + +## 4. Artifacts and `/benchmark` + +### Per-trial dump (git-ignored) + +``` +bench/agent-bench/results///trial-// + prompt.txt the exact task text — identical for every agent + argv.json the exact command, after {prompt}/{model} substitution + patch.diff git diff of the workspace, staged (so new files count) + stdout.log stderr.log + proxy.jsonl every request the proxy saw (paths, usage; no secrets) + usage.json folded tokens / USD / turns for this trial + audit.json isolation audit: non-model paths are leaks +bench/agent-bench/results//report.json +``` + +A new timestamped folder every run. That **is** the historical archive. Old +patches and proxy logs sit there until you delete them. + +Scratch files the agent left behind (`notes.md`, …) stay in `patch.diff` and +are listed in `newFiles` rather than filtered — a fix accompanied by six +scratch files is a fact about the agent. + +### What the page reads (committed) + +At the end of every run, `runner/publish.ts` writes a slim JSON into: + +``` +apps/web/app/data/benchmarks/.json +``` + +Examples: `freecode-vs-claude-code.json`, `freecode-vs-opencode.json`. One file +per **agent set**. The page (`apps/web/app/benchmark/page.tsx`) scans that +directory at **build time** — a new pairing appears by existing; there is no +import list. Production visitors never hit the filesystem. Locally, +`cd apps/web && pnpm dev` → `http://localhost:3000/benchmark` usually re-reads +on refresh. + +Dropped from the page JSON: transcripts, patches, argv, prompts. Kept: +numbers, version, pinned model, autonomy flag, isolation, graded, resolved +verdicts, and (when metered) turns / tokens / usd / auditOk. The page renders +token and cost bars for any metered matchup, marks unmetered agents as +"unmetered" rather than $0, and shows the §7.2 cost-on-solved-intersection +line only once graded — suppressed below 3 shared resolved instances. + +Re-point the page at an older run without paying to re-run it: + +```bash +pnpm exec tsx bench/agent-bench/runner/publish.ts bench/agent-bench/results/ [--fresh] +``` + +### History: latest cell, list of runs + +The page is **latest row wins**, not a time series. Each cell is keyed by +`(agent, instance, trial)`. Re-running the same matchup overwrites that cell. + +The file also keeps a **`runs` list** (newest first). If there is more than +one, the page says the table was **stitched** and that you should not treat the +decimals as a head-to-head — the same agent on the same bug has varied +several-fold between runs. + +`--fresh` discards the matchup file. Use it when the **meaning** of a number +changes: a new model, the grader landing, the container landing. + +A run that adds an agent writes a **different file**. Do not stitch +freecode-vs-opencode rows next to freecode-vs-claude-code rows measured an hour +apart and call it a three-way. + +While `graded` is false the page labels the headline **"Produced a patch"**, +says everyone scores 100% as soon as they edit anything, and shows a banner +that this is a pipeline check. Those flags come from the JSON, so they flip +on their own when Phase 1 lands. + +--- + +## 5. Adding an agent + +One JSON file in `bench/agent-bench/agents/`. Required: `id`, `versionCmd`, +`run` (a `{prompt}` / `{model}` template), `model`, `autonomy`. Optional: +`env`, `notes`. + +**`autonomy` is the experiment, not documentation.** freecode in `build` mode +denies every headless write (`permission/prompt.ts` — the same reason the eval +runner has to answer permission prompts). Running freecode in `build` against a +competitor's full-auto measures permission defaults, not agents. Every agent +runs at its **own maximum**; the flag that got it there is recorded and printed. + +`${VAR}` in `env` is substituted from the process environment and is a hard +error when unset. `""` means *unset this variable* (how a pre-existing +`ANTHROPIC_API_KEY` is kept from redirecting Claude Code to Anthropic). +`{benchDir}` is `bench/agent-bench/`. + +Flags verified 2026-09-03 — **the adapter file is the source of truth:** + +| Agent | Version | Full autonomy | Shipped? | +| --- | --- | --- | --- | +| freecode | local | `run "

" --model

--agent danger --max-turns 40` | yes | +| claude | 2.1.251 | `-p "

" --dangerously-skip-permissions --model ` | yes | +| opencode | 1.18.25 | `run --pure --auto --model

"

"` + `XDG_CONFIG_HOME` | yes | +| codex | 0.151.0 | `exec --dangerously-bypass-approvals-and-sandbox --ephemeral --ignore-user-config` | Phase 3 | + +OpenCode's `XDG_CONFIG_HOME` points at `empty-config/` (deliberately empty). +Without it, opencode loads `~/.config/opencode/opencode.json` and every MCP +server in it. Measured: 19 tools with the operator's config, 10 without. +Neither `--pure` nor `OPENCODE_CONFIG` suppresses MCP. + +Claude Code's `CLAUDE_CODE_AUTO_COMPACT_WINDOW=1048576` is a **fairness +correction**, not a tuning knob: MiniMax's `/anthropic` shim reports a 200K +window for M3 instead of 1M, so Claude would auto-compact at ~167K while +freecode reads models.dev's true 1048576. **Delete that line and freecode wins +on a handicap.** + +--- + +## 6. Free, no model — unit tests + +```bash +pnpm test:agent-bench +``` + +Typechecks the whole bench tree (`tsconfig.json` — tsx executes without +checking, which once let a field vanish from `TrialRecord` unnoticed), then +runs every `*.test.ts`: proxy parse/merge (both wire conventions), rate card, +pass-through (no retry on 500), leak audit, env overlays, docker argv +construction (secrets never in argv), grader prediction/verdict folding, and +bundle reproducibility. Catches a broken meter without spending a cent. Run +it before you ever pay for a matrix. + +`apps/core` also covers `MINIMAX_BASE_URL` in `catalogue.test.ts`. + +--- + +## 7. When to run what + +| Trigger | Command | Cost | +| --- | --- | --- | +| Changed the proxy, rate card, runner, grader, or isolation | `pnpm test:agent-bench` (typechecks, then tests) | free | +| Is this adapter still producing a patch? | `pnpm bench:agents --instances django__django-10914 --trials 1` | 1 turn × agents | +| Isolation smoke (first `--isolate` ever) | `pnpm bench:image`, then `--isolate --instances django__django-10914 --trials 1` | 1 turn × agents | +| **The publishable shape** | `--isolate --trials 3`, then `bench:grade --publish`, then `bench:bundle` | 3 × instances × agents paid turns + grader containers | +| Grade or re-grade a finished run | `pnpm bench:grade results/ --publish` | free (no model) | +| Re-show an old run on `/benchmark` | `tsx bench/agent-bench/runner/publish.ts results/` | free | +| Meaning of a number changed | same, with `--fresh` | free | + +**Never in normal CI.** This spends real money in (eventually) Docker and is a +release-cadence or on-demand job. The moment it exits non-zero somebody wires +it in and starts reverting on a competitor's noise. + +--- + +## 8. Parity with the prior art, and what still separates a run from a result + +| Superbrain shows | Here | +| --- | --- | +| Pass/fail grid from the official Docker grader | Built — `pnpm bench:grade`; page flips to "Resolved" on `graded: true` | +| Token comparison ("64% fewer tokens") | Built — meter + `/benchmark` token/cost bars, one rate card | +| Per-bug cost on the **intersection** of solved instances (§7.2) | Built — appears once graded; suppressed when `\|I\| < 3` | +| Network isolation + audited transcripts | Built — `--isolate` (internal network) + per-trial proxy audit | +| Downloadable evidence bundle | Built — `pnpm bench:bundle`, reproducible tar + sha256 | + +What still separates any given run from a publishable result is **running the +whole ritual**: every layer above exists, but a number is publishable only +when its run used `--isolate`, was graded, kept `auditOk`, and shipped its +bundle — and the isolation/grader layers have not yet had their first live +smoke (they need Docker group membership and `pip install swebench` on the +operator's machine). Until that first smoke run passes, treat them as +untested code, not as a proven pipeline. Codex remains Phase 3 (adapter +listed, never run). + +When more than two agents are in the table, intersection is **pairwise against +freecode**, and the table says so. A four-way intersection shrinks to nothing. + +--- + +## 9. Layout + +``` +bench/agent-bench/ + README.md # points here + tsconfig.json # test:agent-bench typechecks before it tests + agents/*.json # one adapter per agent + instances/django-lite.txt # the ten ids + runner/ # trial loop, fetch, publish, workspace, grade, bundle + proxy/ # recording meter (spec §6.4) + isolate/ # Dockerfile + container/network plumbing (§6.3) + empty-config/ # opencode: no MCP, no personal config + results// # git-ignored + .cache/ # git-ignored + +apps/web/app/benchmark/page.tsx +apps/web/app/data/benchmarks/.json +``` + +--- + +## Environment + +```bash +MINIMAX_API_KEY=... # required for every shipped adapter +MINIMAX_BASE_URL=... # injected by the runner; do not set by hand unless debugging +ANTHROPIC_BASE_URL=... # same; Claude Code. An operator export of ANTHROPIC_API_KEY + # is cleared by the claude adapter so the run cannot silently + # bill Anthropic +``` + +--- + +## Adjacent + +```bash +pnpm eval / EVAL.md # our agent vs its own past — different instrument +pnpm bench:memory # RAM / TTF — Benchmark.md +pnpm test:agent-bench # this harness, no API +freecode trace # where a *freecode* session's time went — TRACE.md +``` diff --git a/AGENTS.md b/AGENTS.md index b5f91122..c6822c19 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,8 @@ This codebase follows `docs/superpowers/specs/2026-05-25-architecture-v4.md` (su | **Trace (commands)** | **`TRACE.md`** — `freecode trace` flags, how to read the waterfall | | Eval harness | `specs/2026-08-23-eval-harness.md` (Phases 0–5, built) + `specs/2026-08-29-eval-case-registry.md` | | **Eval (commands)** | **`EVAL.md`** — which command, which flag, when to run it | -| Agent comparison | `specs/2026-09-03-agent-comparison-benchmark.md` — freecode vs Claude Code/Codex/OpenCode. **Design only, nothing built.** Deliberately outside `eval/`: only `scorers/outcome.ts` is agent-agnostic | +| **Agent comparison (commands)** | **`AGENT-BENCH.md`** — `pnpm bench:agents` (+ `bench:grade`, `bench:bundle`, `--isolate`) vs Claude Code / OpenCode; metering, grading, isolation, `/benchmark`. Spec `2026-09-03-agent-comparison-benchmark.md`. **Not `pnpm eval`.** | +| Agent comparison (design) | `specs/2026-09-03-agent-comparison-benchmark.md` — harness-vs-harness on SWE-bench Lite. Deliberately outside `eval/`. Runtime RAM is `Benchmark.md` | | Hooks | `apps/core/src/hooks/hooks-system.md` | ## Implemented Subsystems diff --git a/Benchmark.md b/Benchmark.md index 9bdac543..9d2be509 100644 --- a/Benchmark.md +++ b/Benchmark.md @@ -1,6 +1,21 @@ # Benchmarking FreeCode -> Runtime performance harness comparing **freecode** against other AI coding +> Two different instruments. Pick the question, then the command. + +| Question | Command | Operator page | +| --- | --- | --- | +| How much RAM, how fast to first frame? | `pnpm bench:memory` | **this file** | +| Does it fix real bugs vs other agents, and for how much? | `pnpm bench:agents` | **[`AGENT-BENCH.md`](AGENT-BENCH.md)** — same shape as `EVAL.md` | +| Did my last change make *our* agent worse? | `pnpm eval` | [`EVAL.md`](EVAL.md) | + +The rest of this file is the **runtime** harness (PSS, time-to-visible). It +does not measure whether anyone fixed a bug. + +--- + +# Runtime performance + +> Comparing **freecode** against other AI coding > agents (Claude Code, Codex CLI, OpenCode, pi, GitHub Copilot, Cursor Agent, > Antigravity). Mirrors the methodology used by > [jcode's](https://github.com/1jehuang/jcode) `scripts/bench_memory_cli.py`. diff --git a/CLAUDE.md b/CLAUDE.md index 3d2342ce..5c93d007 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,7 +21,8 @@ This codebase follows `docs/superpowers/specs/2026-05-25-architecture-v4.md` (su | **Trace (commands)** | **`TRACE.md`** — `freecode trace` flags, how to read the waterfall | | Eval harness | `specs/2026-08-23-eval-harness.md` (Phases 0–5, built) + `specs/2026-08-29-eval-case-registry.md` | | **Eval (commands)** | **`EVAL.md`** — which command, which flag, when to run it | -| Agent comparison | `specs/2026-09-03-agent-comparison-benchmark.md` — freecode vs Claude Code/Codex/OpenCode. **Design only, nothing built.** Deliberately outside `eval/`: only `scorers/outcome.ts` is agent-agnostic | +| **Agent comparison (commands)** | **`AGENT-BENCH.md`** — `pnpm bench:agents` (+ `bench:grade`, `bench:bundle`, `--isolate`) vs Claude Code / OpenCode; metering, grading, isolation, `/benchmark`. Spec `2026-09-03-agent-comparison-benchmark.md`. **Not `pnpm eval`.** | +| Agent comparison (design) | `specs/2026-09-03-agent-comparison-benchmark.md` — harness-vs-harness on SWE-bench Lite. Deliberately outside `eval/`. Runtime RAM is `Benchmark.md` | | Hooks | `apps/core/src/hooks/hooks-system.md` | ## Implemented Subsystems diff --git a/EVAL.md b/EVAL.md index 856c3f3a..c0415dff 100644 --- a/EVAL.md +++ b/EVAL.md @@ -162,6 +162,7 @@ for a real suite run. ```bash freecode trace [id] [--follow|--slow N|--tools|--json|--list|--otlp] # where a turn's time went pnpm bench:recall # memory retrieval benchmark +pnpm bench:agents # vs other agents — AGENT-BENCH.md ``` --- diff --git a/README.md b/README.md index 4d0bf853..1e233266 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,8 @@ Full docs: **[freecode.website](https://freecode.website)** providers, memory, compaction, permissions, eval In-repo references: [`CLAUDE.md`](CLAUDE.md) (contributor guide), -[`EVAL.md`](EVAL.md), [`TRACE.md`](TRACE.md), and the design specs under +[`EVAL.md`](EVAL.md), [`TRACE.md`](TRACE.md), [`AGENT-BENCH.md`](AGENT-BENCH.md), +[`Benchmark.md`](Benchmark.md) (runtime RAM / TTF), and the design specs under [`docs/superpowers/specs/`](docs/superpowers/specs/). ## License diff --git a/bench/agent-bench/README.md b/bench/agent-bench/README.md index cab0a642..8332e25a 100644 --- a/bench/agent-bench/README.md +++ b/bench/agent-bench/README.md @@ -1,160 +1,7 @@ -# agent-bench — command reference +# agent-bench -> freecode against Claude Code, Codex and OpenCode, on SWE-bench Lite. -> Design lives in `docs/superpowers/specs/2026-09-03-agent-comparison-benchmark.md`; -> this is the "what do I type" page. +Operator reference is **[`AGENT-BENCH.md`](../../AGENT-BENCH.md)** at the repo +root — same job as `EVAL.md` / `TRACE.md`: what to type, which flag, when to +run it. -**Status: Phase 0.** No container, no grader, no metering. It answers one -question — *does every adapter produce a non-empty patch* — and every record it -writes carries `isolation: "none"` so a Phase 0 result cannot be mistaken for a -publishable one. **Nothing here produces a number worth showing anyone yet.** - -This is not `pnpm eval`. That measures *our* agent against its own past -(`EVAL.md`). This measures agents against each other, and shares no code with it -on purpose — only `scorers/outcome.ts`'s idea survives the trip, because it is -the only scorer that never asks what produced the diff (spec §4). - ---- - -## 1. One-time setup - -```bash -tsx bench/agent-bench/runner/fetch.ts # 114 django rows -> .cache/instances.jsonl -export MINIMAX_API_KEY=$(node -p "require(process.env.HOME+'/.freecode/config.json').providers.minimax.apiKey") -``` - -Every agent runs **MiniMax-M3 on that one key** — freecode natively, the others -through MiniMax's Anthropic-compatible endpoint. That is the "same model, one -bill" property (spec §5) and it is the only reason a cost column would mean -anything. No adapter file contains a credential; `${MINIMAX_API_KEY}` is -expanded at spawn time and an unset variable is a hard error, because an agent -that quietly fell back to its own key would be billed somewhere else. - -`fetch.ts` is needed **once, ever**. datasets-server 500s and 502s while its -index warms (`"the dataset index is loading"`), and the Hub has outages of its -own; the fetch backs off five times and then stops, leaving any existing cache -untouched. A run whose instances are already cached does not touch the network -at all, so an outage is only ever a problem for instances you do not have yet. - -The cache stores four fields per instance. `patch`, `test_patch` and -`hints_text` — the gold fix and the maintainer discussion that usually contains -it — are dropped before anything touches disk. Not to protect the agent under -test, which cannot see this repo: to keep an answer key out of a repository that -agents work in every day. - -The django mirror (~250MB) is cloned on first use into `.cache/repos/` and -hardlinked per trial, so a run does not measure GitHub's mood. - -## 2. Run - -```bash -pnpm bench:agents --instances django__django-10914 --trials 1 -pnpm bench:agents --agents freecode,claude-code --trials 3 -``` - -| Flag | Default | Does what | -| --- | --- | --- | -| `--agents` | `freecode,claude-code` | comma-separated adapter ids from `agents/` | -| `--instances` | every id in `instances/django-lite.txt` | comma-separated instance ids | -| `--trials` | `1` | trials per (agent, instance). Phase 2 onward: 3, and publish the spread | -| `--timeout` | `900000` | per-trial wall-clock cap, ms | -| `--out` | `results/` | artifact root | - -Exit code is **non-zero if any trial produced an empty patch**. In Phase 0 that -is the entire verdict: a silently empty patch is a broken adapter, and a broken -adapter that reports as a lost benchmark is the worst failure this harness has. - -## 3. Artifacts - -``` -results///trial-// - prompt.txt the exact task text — identical for every agent - argv.json the exact command, after {prompt}/{model} substitution - patch.diff git diff of the workspace, staged (so new files count) - stdout.log stderr.log -results//report.json -``` - -`results/` and `.cache/` are git-ignored. Transcripts get reviewed before they -become public. - -Every run also writes **`apps/web/app/data/benchmarks/.json`** — one -file per agent set, e.g. `freecode-vs-opencode.json` — and that is what the -`/benchmark` page reads, one tab per file. A finished run is already on the -page, with no extra step. It carries the numbers and the disclosures (version, model, autonomy, -isolation, graded) and none of the transcripts, because `results/` does not -exist on a deploy. Re-point the page at an older run without paying to re-run -it: - -```bash -tsx bench/agent-bench/runner/publish.ts results/ [--fresh] -cd apps/web && pnpm dev # http://localhost:3000/benchmark -``` - -**A matchup is the unit of comparison, which is why it is the unit of storage.** -Runs of the same agent set merge into one file, keyed by (agent, instance, -trial) with the newest run winning; a run with a different agent set writes a -different file rather than quietly widening an existing matchup with rows nobody -measured side by side. Adding a pairing therefore adds a tab — `page.tsx` reads -the directory at build time, so there is no import list to update. `--fresh` -discards a matchup's history, which is the honest move whenever the meaning of -its numbers changes: a new model, the grader landing, the container landing. - -The page refuses to flatter the data: while `graded` is false it labels the -headline bar "Produced a patch", says in the footnote that everyone scores 100% -as soon as they edit anything, and shows a banner saying it is a pipeline check -rather than a result. Those come from `graded` and `isolation` in the JSON, so -they disappear on their own when Phase 1 lands — nobody has to remember. - -## 4. Adding an agent - -One file in `agents/`. Required: `id`, `versionCmd`, `run` (a `{prompt}` / -`{model}` template), `model`, `autonomy`. - -**`autonomy` is not documentation, it is the experiment.** freecode in `build` -mode denies every headless write (`permission/prompt.ts` — the same reason -`apps/core/src/eval/runner.ts:157` has to answer permission prompts), so running -freecode in `build` against a competitor's full-auto measures permission -defaults, not agents. Every agent runs at its own maximum, the flag that got it -there is recorded, and it is printed in the results table. - -Flags verified on this machine, 2026-09-03 — **the adapter is the source of -truth, this table just says where the two shipped ones came from and what the -two unshipped ones will look like:** - -| Agent | Version | Full autonomy | Shipped? | -| --- | --- | --- | --- | -| freecode | local | `run "

" --model

--agent danger --max-turns 40` | yes | -| claude | 2.1.251 | `-p "

" --dangerously-skip-permissions --model ` | yes | -| codex | 0.151.0 | `exec --dangerously-bypass-approvals-and-sandbox --ephemeral --ignore-user-config` | Phase 3 | -| opencode | 1.18.25 | `run --pure --auto --model

"

"` + `XDG_CONFIG_HOME` | yes | - -Codex is the only one left, and only because nothing has verified which model it -and freecode can both be pinned to (spec §12.1) — not because the invocation is -unknown. Its `--ephemeral --ignore-user-config` is worth noting: that is exactly -the "no config, no history carried between trials" property §6.3 wants, and the -other three need a container to get it. opencode needed `XDG_CONFIG_HOME` for a -weaker version of the same thing — see `empty-config/README.md`. - -## 5. Known gaps — what Phase 0 does not do - -1. **No isolation.** No container, so: the network is open (an agent *could* - look the fix up), `$HOME` is mounted (freecode writes sessions and memory to - `~/.freecode` and carries them between trials), and the agent's own config - applies. Every record says `isolation: "none"` for this reason. Phase 1. -2. **No grading.** "Produced a patch" is not "fixed the bug". The official - SWE-bench grader needs Docker, and on this machine the daemon is running but - the user is not in the `docker` group. Phase 1. -3. **No metering.** Cost and token columns need the recording proxy (§6.4); - four vendors' self-reports are four rounding policies. Phase 1. -4. **Model parity depends on one env var staying right.** Both agents are pinned - to `MiniMax-M3`, but MiniMax's `/anthropic` shim reports a 200K context - window instead of M3's real 1048576, so Claude Code auto-compacts at ~167K - unless `CLAUDE_CODE_AUTO_COMPACT_WINDOW` says otherwise. The adapter sets it. - **Delete that line and freecode wins on a handicap** — this class of bug is - invisible in the results and only findable by reading the adapters, which is - why they are committed and short. -5. **Scratch files land in the patch.** `extractPatch` stages everything, so an - agent that leaves `notes.md` behind ships it in the diff. They are listed in - `newFiles` rather than filtered — a fix accompanied by six scratch files is a - fact about the agent worth seeing. +Design: `docs/superpowers/specs/2026-09-03-agent-comparison-benchmark.md`. From b4ef7724349e24f8cd55099e5aec936c72771ae8 Mon Sep 17 00:00:00 2001 From: Ayan De Date: Sun, 6 Sep 2026 04:37:03 +0530 Subject: [PATCH 10/10] fix(web): an unmetered trial is not a $0 run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First live smoke: the installed freecode release binary predates the baseURLFor hook, talked around the proxy, and the page counted its zeroed usage as a metered $0.0000 trial. Metered now means the proxy saw model calls (turns > 0) — in the summary, the matrix cells, and the §7.2 intersection alike. Also folds in the smoke-run rows: the meter's inclusive normalization verified against MiniMax's shim (t2 cache-read == t1 total input), and the leak audit caught Claude Code's HEAD /api/hello probe. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ELXKXKpNy38kvVfjwJC52f --- apps/web/app/data/agent-bench.ts | 10 ++-- .../benchmarks/freecode-vs-claude-code.json | 46 +++++++++++++++---- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/apps/web/app/data/agent-bench.ts b/apps/web/app/data/agent-bench.ts index 87b1492d..8a000b21 100644 --- a/apps/web/app/data/agent-bench.ts +++ b/apps/web/app/data/agent-bench.ts @@ -169,7 +169,9 @@ export function deriveView(raw: RawBenchmark): BenchView { const successes = mine.filter((r) => raw.graded ? r.resolved === true : r.producedPatch, ).length; - const metered = mine.filter((r) => r.inputTokens !== undefined); + // Metered means the proxy saw model calls — a trial where the agent + // talked around the proxy still writes zeros, and zeros are not a $0 run. + const metered = mine.filter((r) => (r.turns ?? 0) > 0); const usds = metered.map((r) => r.usd).filter((u): u is number => typeof u === "number"); return { id: a.id, @@ -211,8 +213,8 @@ export function deriveView(raw: RawBenchmark): BenchView { patchBytes: r.patchBytes, reason: r.reason, trial: r.trial, - tokens: totalTokens(r), - usd: r.usd, + tokens: (r.turns ?? 0) > 0 ? totalTokens(r) : undefined, + usd: (r.turns ?? 0) > 0 ? r.usd : undefined, auditOk: r.auditOk, })), })); @@ -238,7 +240,7 @@ export function deriveView(raw: RawBenchmark): BenchView { r.agent === a.id && solvedByAll.includes(r.instanceId) && r.resolved === true && - r.inputTokens !== undefined, + (r.turns ?? 0) > 0, ); const usds = mine.map((r) => r.usd).filter((u): u is number => typeof u === "number"); return { diff --git a/apps/web/app/data/benchmarks/freecode-vs-claude-code.json b/apps/web/app/data/benchmarks/freecode-vs-claude-code.json index 7cff947c..df27ea23 100644 --- a/apps/web/app/data/benchmarks/freecode-vs-claude-code.json +++ b/apps/web/app/data/benchmarks/freecode-vs-claude-code.json @@ -1,8 +1,24 @@ { "slug": "freecode-vs-claude-code", - "generatedAt": "2026-09-03T19:18:04.474Z", - "runId": "2026-09-03T18-56-04-591Z", + "generatedAt": "2026-09-05T23:06:41.774Z", + "runId": "2026-09-05T23-06-02-611Z", "runs": [ + { + "runId": "2026-09-05T23-06-02-611Z", + "generatedAt": "2026-09-05T23:06:41.774Z", + "agents": [ + "freecode", + "claude-code" + ] + }, + { + "runId": "2026-09-05T23-02-42-713Z", + "generatedAt": "2026-09-05T23:03:35.231Z", + "agents": [ + "freecode", + "claude-code" + ] + }, { "runId": "2026-09-03T18-56-04-591Z", "generatedAt": "2026-09-03T19:18:04.474Z", @@ -43,7 +59,7 @@ "agents": [ { "id": "freecode", - "version": "0.28.0", + "version": "0.30.0", "model": "minimax/MiniMax-M3", "autonomy": "--agent danger: no permission prompts" }, @@ -61,11 +77,18 @@ "trial": 1, "producedPatch": true, "resolved": null, - "durationMs": 29849, + "durationMs": 10799, "patchBytes": 625, "newFiles": 0, "reason": "ok", - "runId": "2026-09-03T16-51-56-681Z" + "runId": "2026-09-05T23-06-02-611Z", + "turns": 10, + "inputTokens": 140560, + "outputTokens": 1304, + "cacheReadTokens": 124250, + "cacheWriteTokens": 0, + "usd": 0.013912800000000001, + "auditOk": true }, { "agent": "claude-code", @@ -73,11 +96,18 @@ "trial": 1, "producedPatch": true, "resolved": null, - "durationMs": 9729, - "patchBytes": 625, + "durationMs": 27158, + "patchBytes": 1099, "newFiles": 0, "reason": "ok", - "runId": "2026-09-03T16-51-56-681Z" + "runId": "2026-09-05T23-06-02-611Z", + "turns": 14, + "inputTokens": 427431, + "outputTokens": 2005, + "cacheReadTokens": 394362, + "cacheWriteTokens": 0, + "usd": 0.03598841999999999, + "auditOk": false }, { "agent": "freecode",