diff --git a/.gitignore b/.gitignore index 852f6bc..d2ee4b7 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,10 @@ node_modules .pnp .pnp.js +# Content-addressable package store. Default is ~/.local/share/pnpm/store; +# if pnpm is pointed at the repo (store-dir / PNPM_STORE_DIR), it lands here +# and must never be committed. +.pnpm-store/ # Local env files .env diff --git a/AGENT-BENCH.md b/AGENT-BENCH.md new file mode 100644 index 0000000..14d791e --- /dev/null +++ b/AGENT-BENCH.md @@ -0,0 +1,452 @@ +# Agent-bench — command reference + +> Operator's guide to the **agent comparison** harness: what each command does, +> which flag to reach for, and when to run it. Design lives in +> `docs/superpowers/specs/2026-09-03-agent-comparison-benchmark.md`; this is +> the "what do I type" page. +> +> Inspired by [Superbrain's public benchmark](https://www.onesuperbrain.com/benchmarks): +> same tasks, same model, same key, one meter, official grader, published +> artifacts. We are not there yet — see §8. + +This is **not** `pnpm eval`. That measures *our* agent against its own past +(`EVAL.md`). This measures agents against each other, on tasks we did not +write, and shares no code with `apps/core/src/eval` on purpose — only +`scorers/outcome.ts`'s idea survives the trip, because it is the only scorer +that never asks what produced the diff. + +This is **not** `pnpm bench:memory`. That measures PSS and time-to-first-frame +(`Benchmark.md`). A fast TUI that fixes no bugs is a different claim. + +| Question | Command | Doc | +| --- | --- | --- | +| Did my last change make *our* agent worse? | `pnpm eval` | `EVAL.md` | +| How much RAM, how fast to first frame? | `pnpm bench:memory` | `Benchmark.md` | +| **Does it fix real bugs vs Claude Code / OpenCode, and for how much?** | **`pnpm bench:agents`** | **this file** | + +**Status (2026-09-06).** The full pipeline exists: metering (on by default), +container isolation (`--isolate`), the official SWE-bench grader +(`pnpm bench:grade`), §7.2 cost columns on `/benchmark`, and the evidence +bundle (`pnpm bench:bundle`). **A publishable number is a run with all of +them**: `--isolate --trials 3`, then grade, then bundle. A run missing any +step is labelled provisional by the page and should stay that way. Grader and +isolation need Docker (daemon up, operator in the docker group) and the +grader needs `pip install swebench` — neither has had a paid smoke run yet; +`pnpm test:agent-bench` covers everything that is free. There is still no +gate, no CI wiring, no exit-on-regression — same reasoning as `eval ab`. + +Code lives in `bench/agent-bench/`, sibling to `bench/jcode-bench/`, outside +the pnpm workspace on purpose. + +--- + +## 1. One-time setup + +```bash +pnpm exec tsx bench/agent-bench/runner/fetch.ts +export MINIMAX_API_KEY=$(node -p "require(process.env.HOME+'/.freecode/config.json').providers.minimax.apiKey") +``` + +Every shipped adapter runs **MiniMax-M3 on that one key** — freecode natively, +the others through MiniMax's Anthropic-compatible endpoint. That is the "same +model, one bill" property (spec §5) and it is the only reason a cost column +would mean anything. No adapter file contains a credential; `${MINIMAX_API_KEY}` +is expanded at spawn time and an **unset variable is a hard error**, because an +agent that quietly fell back to its own key would be billed somewhere else. + +`fetch.ts` is needed **once, ever**. It pulls the django subset of SWE-bench +Lite into `bench/agent-bench/.cache/instances.jsonl` (114 rows). datasets-server +500s while its index warms; the fetch backs off five times and then stops, +leaving any existing cache untouched. A run whose instances are already cached +does not touch the Hub at all. + +The cache stores four fields per instance. `patch`, `test_patch` and +`hints_text` — the gold fix and the maintainer discussion that usually contains +it — are **dropped before anything touches disk**. Not to protect the agent +under test (it cannot see this repo): to keep an answer key out of a repository +agents work in every day. + +The django mirror (~250 MB) is cloned on first use into +`bench/agent-bench/.cache/repos/` and hardlinked per trial, so a run does not +measure GitHub's mood. + +`.cache/` and `results/` are git-ignored. + +--- + +## 2. Run a comparison — `pnpm bench:agents` + +```bash +# Smoke: one instance, one trial, default agents (freecode, claude-code) +pnpm bench:agents --instances django__django-10914 --trials 1 + +# The shape Phase 2 wants: N=3, publish the spread +pnpm bench:agents --agents freecode,claude-code --trials 3 + +# Skip the recording proxy (adapter debugging only) +pnpm bench:agents --no-meter --instances django__django-10914 +``` + +Every invocation is a **real paid agent turn × agents × instances × trials**. +Read this file before you spend. + +| Flag | Default | Does what | +| --- | --- | --- | +| `--agents` | `freecode,claude-code` | comma-separated adapter ids from `bench/agent-bench/agents/` | +| `--instances` | every id in `instances/django-lite.txt` (10 django bugs) | comma-separated SWE-bench instance ids | +| `--trials` | `1` | trials per (agent, instance). Phase 2 onward: **3**, and publish the spread. Never merge into best-of | +| `--timeout` | `900000` (15 min) | per-trial wall-clock cap, ms. Timeout SIGKILLs the agent | +| `--out` | `bench/agent-bench/results/` | artifact root | +| `--fresh` | off | passed through to publish: discard the matchup JSON and start over | +| `--no-meter` | off | skip the recording proxy | +| `--isolate` | off | one container per trial on an `--internal` docker network (§6.3); implies metering. Needs the image: `pnpm bench:image` (online, once) | + +**Exit code** is non-zero if **any trial produced an empty patch**. In Phase 0 +that is the entire verdict: a silently empty patch is a broken adapter, and a +broken adapter that reports as a lost benchmark is the worst failure this +harness has. There is **no** gate against a baseline and **no** CI job. Do not +wire one. + +The matrix is `instances × trials × agents`, nested in that order. One dead +trial does not abort the rest. + +### Task set + +`instances/django-lite.txt` is the **first ten django instance ids in lexical +order**. Chosen mechanically, and said out loud so nobody has to take it on +faith that they were not cherry-picked. Ten instances from one repo is a +**demo, not a leaderboard** (spec §10.1). SWE-bench Lite is 300 across 11 +repos. + +The prompt is one string, identical for every agent (`runner/prompt.ts`). If it +ever differs per agent, the benchmark stops comparing harnesses and starts +comparing prompts we wrote for them. + +--- + +## 3. Metering — one meter, not four self-reports + +On by default. Each trial starts a **pass-through HTTP proxy** +(`bench/agent-bench/proxy/`) and points the agent at it: + +| Agent | Env the runner injects | Upstream | +| --- | --- | --- | +| Claude Code | `ANTHROPIC_BASE_URL=http://127.0.0.1:` | adapter's `ANTHROPIC_BASE_URL` (MiniMax `/anthropic`) | +| freecode | `MINIMAX_BASE_URL=http://127.0.0.1:/v1` | same MiniMax origin; `baseURLFor()` in `catalogue.ts` honours `$_BASE_URL` | +| OpenCode | those two vars, which it **does not honour** | unmetered without `--isolate`; in a container the internal network leaves it no other route | + +The proxy: no cache, no retries, no body rewrite. Tokens are parsed off +Anthropic Messages JSON/SSE and OpenAI Chat Completions. Anthropic +`message_delta` only carries `output_tokens`; input and cache reads stay from +`message_start`. Secrets are not written to the log (no headers, no bodies). + +**Conventions differ between the two wire shapes and the meter normalizes +them**: OpenAI's `prompt_tokens` already includes `cached_tokens`, but +Anthropic's `input_tokens` EXCLUDES `cache_read/creation_input_tokens` — +they are separate additive fields. `usage.ts` folds both into one inclusive +`inputTokens`, which is what makes `price.ts`'s cache discount arithmetic +correct for either shape. If a shim turns out to report inclusive +`input_tokens` (verify on the first metered run: fresh input should be +`input_tokens`, small, beside large cache fields), cost is overstated by the +cache amount, never silently understated. + +**USD** uses a committed rate card in `proxy/price.ts`, vintage +`2026-09 MiniMax standard ≤512k`: + +| Model | Input / M | Output / M | Cache read / M | +| --- | ---: | ---: | ---: | +| MiniMax-M3 | $0.30 | $1.20 | $0.06 | + +Cache reads are a **discount off the inclusive input count**, matching +`providers/pricing.ts`. Adding them on top would report a cache win as a cost +increase. An unknown model prices as **`undefined` / `null`, never as zero**. + +A request whose path is not `/messages` or `/chat/completions` is a **leak**. +An **empty** proxy log is unmetered (`auditOk: false`), not a cheap clean run — +that is how an agent that talked *around* the proxy is supposed to look. + +Without a container, the proxy only sees traffic **pointed at it**. A clean +audit is not proof that nothing else left the machine. `isolation` stays +`"none"` until Docker lands. + +Pinning a **different shared model** is a different experiment, which is fine +if every adapter's `"model"` and the proxy upstream still match. Tokens will +record for any Anthropic- or OpenAI-shaped endpoint; USD stays null until you +add a row to `proxy/price.ts`. Do not mix bills (Gemini on one side, MiniMax on +the other) and call it a cost comparison. + +### Isolation — `--isolate` + +One container per trial (`bench/agent-bench/isolate/`), on a docker network +created with `--internal`: docker programs no route out, so the only exit is +the recording proxy, which binds the network's gateway IP on the host. The +proxy log stops being an honor system and becomes the egress audit. Known +residue, stated rather than hidden: other host services on the gateway IP +remain reachable from the container; the internet does not. + +```bash +pnpm bench:image # build agent-bench (online, once) +pnpm bench:agents --isolate --trials 3 --agents freecode,claude-code +``` + +The image (`isolate/Dockerfile`) bakes pinned versions of every agent — +freecode from a **released** binary (build arg `FREECODE_VERSION`, default +v0.30.0), the others from npm — because at trial time there is no network to +install anything with. `agentVersion` is read from the image, `$HOME` is a +tmpfs-style throwaway inside the container (no memory between trials, §6.3), +the workspace mounts rw at `/workspace`, the bench dir ro at `/bench` +(`{benchDir}` resolves there), and env values ride bare `-e NAME` flags so +`argv.json` never contains a secret. A timed-out trial gets `docker rm -f`, +not just a dead client. + +--- + +## 3b. Grading — `pnpm bench:grade` + +```bash +pip install swebench # once; or SWEBENCH_PYTHON=/path/to/venv/python3 +pnpm bench:grade bench/agent-bench/results/ [--max-workers N] [--publish] +``` + +The verdict is the **official SWE-bench harness's, in Docker, on +`patch.diff`** — `runner/grade.ts` only shuttles patches in and verdicts out. +One harness invocation per (agent, trial), because the harness keys on +`instance_id` and two trials of one instance must not share a predictions +file. An empty patch is `resolved: false` without spending a container; a +harness error leaves `null`, which the page counts as not-resolved rather +than quietly dropping. Verdicts land in `report.json` (`resolved` per trial, +`graded: true`) and `--publish` re-publishes the matchup — at which point the +page's headline flips from "Produced a patch" to "Resolved" and the §7.2 +cost-on-intersection line appears. Grading is free to re-run: it never +touches a model. + +## 3c. Evidence bundle — `pnpm bench:bundle` + +```bash +pnpm bench:bundle bench/agent-bench/results/ +``` + +Writes `-evidence.tar.gz` + `.sha256` next to the run dir: report, +prompts, argv, patches, agent stdout/stderr, proxy logs, usage and audit +folds, grading output. The tar is byte-reproducible (sorted, owner/mtime +pinned), so the published checksum is a claim anyone can re-derive. Publish +the tarball wherever the numbers go; the proxy log carries no headers or +bodies, so nothing in it needs redacting. + +--- + +## 4. Artifacts and `/benchmark` + +### Per-trial dump (git-ignored) + +``` +bench/agent-bench/results///trial-// + prompt.txt the exact task text — identical for every agent + argv.json the exact command, after {prompt}/{model} substitution + patch.diff git diff of the workspace, staged (so new files count) + stdout.log stderr.log + proxy.jsonl every request the proxy saw (paths, usage; no secrets) + usage.json folded tokens / USD / turns for this trial + audit.json isolation audit: non-model paths are leaks +bench/agent-bench/results//report.json +``` + +A new timestamped folder every run. That **is** the historical archive. Old +patches and proxy logs sit there until you delete them. + +Scratch files the agent left behind (`notes.md`, …) stay in `patch.diff` and +are listed in `newFiles` rather than filtered — a fix accompanied by six +scratch files is a fact about the agent. + +### What the page reads (committed) + +At the end of every run, `runner/publish.ts` writes a slim JSON into: + +``` +apps/web/app/data/benchmarks/.json +``` + +Examples: `freecode-vs-claude-code.json`, `freecode-vs-opencode.json`. One file +per **agent set**. The page (`apps/web/app/benchmark/page.tsx`) scans that +directory at **build time** — a new pairing appears by existing; there is no +import list. Production visitors never hit the filesystem. Locally, +`cd apps/web && pnpm dev` → `http://localhost:3000/benchmark` usually re-reads +on refresh. + +Dropped from the page JSON: transcripts, patches, argv, prompts. Kept: +numbers, version, pinned model, autonomy flag, isolation, graded, resolved +verdicts, and (when metered) turns / tokens / usd / auditOk. The page renders +token and cost bars for any metered matchup, marks unmetered agents as +"unmetered" rather than $0, and shows the §7.2 cost-on-solved-intersection +line only once graded — suppressed below 3 shared resolved instances. + +Re-point the page at an older run without paying to re-run it: + +```bash +pnpm exec tsx bench/agent-bench/runner/publish.ts bench/agent-bench/results/ [--fresh] +``` + +### History: latest cell, list of runs + +The page is **latest row wins**, not a time series. Each cell is keyed by +`(agent, instance, trial)`. Re-running the same matchup overwrites that cell. + +The file also keeps a **`runs` list** (newest first). If there is more than +one, the page says the table was **stitched** and that you should not treat the +decimals as a head-to-head — the same agent on the same bug has varied +several-fold between runs. + +`--fresh` discards the matchup file. Use it when the **meaning** of a number +changes: a new model, the grader landing, the container landing. + +A run that adds an agent writes a **different file**. Do not stitch +freecode-vs-opencode rows next to freecode-vs-claude-code rows measured an hour +apart and call it a three-way. + +While `graded` is false the page labels the headline **"Produced a patch"**, +says everyone scores 100% as soon as they edit anything, and shows a banner +that this is a pipeline check. Those flags come from the JSON, so they flip +on their own when Phase 1 lands. + +--- + +## 5. Adding an agent + +One JSON file in `bench/agent-bench/agents/`. Required: `id`, `versionCmd`, +`run` (a `{prompt}` / `{model}` template), `model`, `autonomy`. Optional: +`env`, `notes`. + +**`autonomy` is the experiment, not documentation.** freecode in `build` mode +denies every headless write (`permission/prompt.ts` — the same reason the eval +runner has to answer permission prompts). Running freecode in `build` against a +competitor's full-auto measures permission defaults, not agents. Every agent +runs at its **own maximum**; the flag that got it there is recorded and printed. + +`${VAR}` in `env` is substituted from the process environment and is a hard +error when unset. `""` means *unset this variable* (how a pre-existing +`ANTHROPIC_API_KEY` is kept from redirecting Claude Code to Anthropic). +`{benchDir}` is `bench/agent-bench/`. + +Flags verified 2026-09-03 — **the adapter file is the source of truth:** + +| Agent | Version | Full autonomy | Shipped? | +| --- | --- | --- | --- | +| freecode | local | `run "

" --model

--agent danger --max-turns 40` | yes | +| claude | 2.1.251 | `-p "

" --dangerously-skip-permissions --model ` | yes | +| opencode | 1.18.25 | `run --pure --auto --model

"

"` + `XDG_CONFIG_HOME` | yes | +| codex | 0.151.0 | `exec --dangerously-bypass-approvals-and-sandbox --ephemeral --ignore-user-config` | Phase 3 | + +OpenCode's `XDG_CONFIG_HOME` points at `empty-config/` (deliberately empty). +Without it, opencode loads `~/.config/opencode/opencode.json` and every MCP +server in it. Measured: 19 tools with the operator's config, 10 without. +Neither `--pure` nor `OPENCODE_CONFIG` suppresses MCP. + +Claude Code's `CLAUDE_CODE_AUTO_COMPACT_WINDOW=1048576` is a **fairness +correction**, not a tuning knob: MiniMax's `/anthropic` shim reports a 200K +window for M3 instead of 1M, so Claude would auto-compact at ~167K while +freecode reads models.dev's true 1048576. **Delete that line and freecode wins +on a handicap.** + +--- + +## 6. Free, no model — unit tests + +```bash +pnpm test:agent-bench +``` + +Typechecks the whole bench tree (`tsconfig.json` — tsx executes without +checking, which once let a field vanish from `TrialRecord` unnoticed), then +runs every `*.test.ts`: proxy parse/merge (both wire conventions), rate card, +pass-through (no retry on 500), leak audit, env overlays, docker argv +construction (secrets never in argv), grader prediction/verdict folding, and +bundle reproducibility. Catches a broken meter without spending a cent. Run +it before you ever pay for a matrix. + +`apps/core` also covers `MINIMAX_BASE_URL` in `catalogue.test.ts`. + +--- + +## 7. When to run what + +| Trigger | Command | Cost | +| --- | --- | --- | +| Changed the proxy, rate card, runner, grader, or isolation | `pnpm test:agent-bench` (typechecks, then tests) | free | +| Is this adapter still producing a patch? | `pnpm bench:agents --instances django__django-10914 --trials 1` | 1 turn × agents | +| Isolation smoke (first `--isolate` ever) | `pnpm bench:image`, then `--isolate --instances django__django-10914 --trials 1` | 1 turn × agents | +| **The publishable shape** | `--isolate --trials 3`, then `bench:grade --publish`, then `bench:bundle` | 3 × instances × agents paid turns + grader containers | +| Grade or re-grade a finished run | `pnpm bench:grade results/ --publish` | free (no model) | +| Re-show an old run on `/benchmark` | `tsx bench/agent-bench/runner/publish.ts results/` | free | +| Meaning of a number changed | same, with `--fresh` | free | + +**Never in normal CI.** This spends real money in (eventually) Docker and is a +release-cadence or on-demand job. The moment it exits non-zero somebody wires +it in and starts reverting on a competitor's noise. + +--- + +## 8. Parity with the prior art, and what still separates a run from a result + +| Superbrain shows | Here | +| --- | --- | +| Pass/fail grid from the official Docker grader | Built — `pnpm bench:grade`; page flips to "Resolved" on `graded: true` | +| Token comparison ("64% fewer tokens") | Built — meter + `/benchmark` token/cost bars, one rate card | +| Per-bug cost on the **intersection** of solved instances (§7.2) | Built — appears once graded; suppressed when `\|I\| < 3` | +| Network isolation + audited transcripts | Built — `--isolate` (internal network) + per-trial proxy audit | +| Downloadable evidence bundle | Built — `pnpm bench:bundle`, reproducible tar + sha256 | + +What still separates any given run from a publishable result is **running the +whole ritual**: every layer above exists, but a number is publishable only +when its run used `--isolate`, was graded, kept `auditOk`, and shipped its +bundle — and the isolation/grader layers have not yet had their first live +smoke (they need Docker group membership and `pip install swebench` on the +operator's machine). Until that first smoke run passes, treat them as +untested code, not as a proven pipeline. Codex remains Phase 3 (adapter +listed, never run). + +When more than two agents are in the table, intersection is **pairwise against +freecode**, and the table says so. A four-way intersection shrinks to nothing. + +--- + +## 9. Layout + +``` +bench/agent-bench/ + README.md # points here + tsconfig.json # test:agent-bench typechecks before it tests + agents/*.json # one adapter per agent + instances/django-lite.txt # the ten ids + runner/ # trial loop, fetch, publish, workspace, grade, bundle + proxy/ # recording meter (spec §6.4) + isolate/ # Dockerfile + container/network plumbing (§6.3) + empty-config/ # opencode: no MCP, no personal config + results// # git-ignored + .cache/ # git-ignored + +apps/web/app/benchmark/page.tsx +apps/web/app/data/benchmarks/.json +``` + +--- + +## Environment + +```bash +MINIMAX_API_KEY=... # required for every shipped adapter +MINIMAX_BASE_URL=... # injected by the runner; do not set by hand unless debugging +ANTHROPIC_BASE_URL=... # same; Claude Code. An operator export of ANTHROPIC_API_KEY + # is cleared by the claude adapter so the run cannot silently + # bill Anthropic +``` + +--- + +## Adjacent + +```bash +pnpm eval / EVAL.md # our agent vs its own past — different instrument +pnpm bench:memory # RAM / TTF — Benchmark.md +pnpm test:agent-bench # this harness, no API +freecode trace # where a *freecode* session's time went — TRACE.md +``` diff --git a/AGENTS.md b/AGENTS.md index b5f9112..c6822c1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,8 @@ This codebase follows `docs/superpowers/specs/2026-05-25-architecture-v4.md` (su | **Trace (commands)** | **`TRACE.md`** — `freecode trace` flags, how to read the waterfall | | Eval harness | `specs/2026-08-23-eval-harness.md` (Phases 0–5, built) + `specs/2026-08-29-eval-case-registry.md` | | **Eval (commands)** | **`EVAL.md`** — which command, which flag, when to run it | -| Agent comparison | `specs/2026-09-03-agent-comparison-benchmark.md` — freecode vs Claude Code/Codex/OpenCode. **Design only, nothing built.** Deliberately outside `eval/`: only `scorers/outcome.ts` is agent-agnostic | +| **Agent comparison (commands)** | **`AGENT-BENCH.md`** — `pnpm bench:agents` (+ `bench:grade`, `bench:bundle`, `--isolate`) vs Claude Code / OpenCode; metering, grading, isolation, `/benchmark`. Spec `2026-09-03-agent-comparison-benchmark.md`. **Not `pnpm eval`.** | +| Agent comparison (design) | `specs/2026-09-03-agent-comparison-benchmark.md` — harness-vs-harness on SWE-bench Lite. Deliberately outside `eval/`. Runtime RAM is `Benchmark.md` | | Hooks | `apps/core/src/hooks/hooks-system.md` | ## Implemented Subsystems diff --git a/Benchmark.md b/Benchmark.md index 9bdac54..9d2be50 100644 --- a/Benchmark.md +++ b/Benchmark.md @@ -1,6 +1,21 @@ # Benchmarking FreeCode -> Runtime performance harness comparing **freecode** against other AI coding +> Two different instruments. Pick the question, then the command. + +| Question | Command | Operator page | +| --- | --- | --- | +| How much RAM, how fast to first frame? | `pnpm bench:memory` | **this file** | +| Does it fix real bugs vs other agents, and for how much? | `pnpm bench:agents` | **[`AGENT-BENCH.md`](AGENT-BENCH.md)** — same shape as `EVAL.md` | +| Did my last change make *our* agent worse? | `pnpm eval` | [`EVAL.md`](EVAL.md) | + +The rest of this file is the **runtime** harness (PSS, time-to-visible). It +does not measure whether anyone fixed a bug. + +--- + +# Runtime performance + +> Comparing **freecode** against other AI coding > agents (Claude Code, Codex CLI, OpenCode, pi, GitHub Copilot, Cursor Agent, > Antigravity). Mirrors the methodology used by > [jcode's](https://github.com/1jehuang/jcode) `scripts/bench_memory_cli.py`. diff --git a/CLAUDE.md b/CLAUDE.md index 3d2342c..5c93d00 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,7 +21,8 @@ This codebase follows `docs/superpowers/specs/2026-05-25-architecture-v4.md` (su | **Trace (commands)** | **`TRACE.md`** — `freecode trace` flags, how to read the waterfall | | Eval harness | `specs/2026-08-23-eval-harness.md` (Phases 0–5, built) + `specs/2026-08-29-eval-case-registry.md` | | **Eval (commands)** | **`EVAL.md`** — which command, which flag, when to run it | -| Agent comparison | `specs/2026-09-03-agent-comparison-benchmark.md` — freecode vs Claude Code/Codex/OpenCode. **Design only, nothing built.** Deliberately outside `eval/`: only `scorers/outcome.ts` is agent-agnostic | +| **Agent comparison (commands)** | **`AGENT-BENCH.md`** — `pnpm bench:agents` (+ `bench:grade`, `bench:bundle`, `--isolate`) vs Claude Code / OpenCode; metering, grading, isolation, `/benchmark`. Spec `2026-09-03-agent-comparison-benchmark.md`. **Not `pnpm eval`.** | +| Agent comparison (design) | `specs/2026-09-03-agent-comparison-benchmark.md` — harness-vs-harness on SWE-bench Lite. Deliberately outside `eval/`. Runtime RAM is `Benchmark.md` | | Hooks | `apps/core/src/hooks/hooks-system.md` | ## Implemented Subsystems diff --git a/EVAL.md b/EVAL.md index 856c3f3..c0415df 100644 --- a/EVAL.md +++ b/EVAL.md @@ -162,6 +162,7 @@ for a real suite run. ```bash freecode trace [id] [--follow|--slow N|--tools|--json|--list|--otlp] # where a turn's time went pnpm bench:recall # memory retrieval benchmark +pnpm bench:agents # vs other agents — AGENT-BENCH.md ``` --- diff --git a/README.md b/README.md index 4d0bf85..1e23326 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,8 @@ Full docs: **[freecode.website](https://freecode.website)** providers, memory, compaction, permissions, eval In-repo references: [`CLAUDE.md`](CLAUDE.md) (contributor guide), -[`EVAL.md`](EVAL.md), [`TRACE.md`](TRACE.md), and the design specs under +[`EVAL.md`](EVAL.md), [`TRACE.md`](TRACE.md), [`AGENT-BENCH.md`](AGENT-BENCH.md), +[`Benchmark.md`](Benchmark.md) (runtime RAM / TTF), and the design specs under [`docs/superpowers/specs/`](docs/superpowers/specs/). ## License diff --git a/apps/core/src/providers/catalogue.test.ts b/apps/core/src/providers/catalogue.test.ts index 7a28a64..35c3c5d 100644 --- a/apps/core/src/providers/catalogue.test.ts +++ b/apps/core/src/providers/catalogue.test.ts @@ -3,6 +3,7 @@ import assert from "node:assert/strict"; import { resolveCatalogue, envKeysFor, + baseURLFor, FEATURED_PROVIDER_IDS, } from "./catalogue.js"; import { CATALOGUE_SNAPSHOT } from "./catalogue-snapshot.js"; @@ -86,6 +87,20 @@ test("an id the catalogue does not carry yields no keys rather than guessing", ( assert.deepEqual(envKeysFor("not-a-real-provider"), []); }); +test("MINIMAX_BASE_URL overrides the catalogue endpoint", () => { + const minimax = resolveCatalogue().find((e) => e.id === "minimax"); + assert.ok(minimax); + assert.equal(baseURLFor(minimax), minimax.baseURL); + const prev = process.env.MINIMAX_BASE_URL; + process.env.MINIMAX_BASE_URL = "http://127.0.0.1:9/v1"; + try { + assert.equal(baseURLFor(minimax), "http://127.0.0.1:9/v1"); + } finally { + if (prev === undefined) delete process.env.MINIMAX_BASE_URL; + else process.env.MINIMAX_BASE_URL = prev; + } +}); + test("resolution is memoized, and the memo can be dropped", async () => { const { invalidateCatalogue } = await import("./catalogue.js"); const first = resolveCatalogue(); diff --git a/apps/core/src/providers/catalogue.ts b/apps/core/src/providers/catalogue.ts index eb0f906..ca4583e 100644 --- a/apps/core/src/providers/catalogue.ts +++ b/apps/core/src/providers/catalogue.ts @@ -211,3 +211,20 @@ export function envKeysFor(id: string): string[] { } return envKeyIndex.get(id) ?? []; } + +/** + * Catalogue `baseURL`, or `$_BASE_URL` when set. + * + * The recording proxy in `bench/agent-bench` points every agent at one meter + * (spec §6.4). Claude Code already honours `ANTHROPIC_BASE_URL`; freecode + * previously ignored it because the SDK is constructed with the catalogue + * URL. `MINIMAX_BASE_URL` (and the same pattern for any other id) is the + * equivalent hook — a bench-only override, not a documented user setting. + */ +export function baseURLFor(entry: { + id: string; + baseURL?: string; +}): string | undefined { + const envName = `${entry.id.replace(/-/g, "_").toUpperCase()}_BASE_URL`; + return process.env[envName] || entry.baseURL; +} diff --git a/apps/core/src/providers/generic-provider.ts b/apps/core/src/providers/generic-provider.ts index bbbbce3..0280b24 100644 --- a/apps/core/src/providers/generic-provider.ts +++ b/apps/core/src/providers/generic-provider.ts @@ -28,7 +28,10 @@ import { normalizeAiSdkStream } from "./streaming.js"; import { mapUsage } from "./provider-shared.js"; import { applyEffort } from "./effort.js"; import { loadSdkFactory } from "./sdk-factories.js"; -import type { ProviderCatalogueEntry } from "./catalogue.js"; +import { + baseURLFor, + type ProviderCatalogueEntry, +} from "./catalogue.js"; /** * Which branch of request-shaping an SDK package needs. @@ -230,7 +233,7 @@ export function createGenericProvider(entry: ProviderCatalogueEntry): AIProvider apiKey: oauth ? "oauth-subscription" : getApiKey(entry.id, entry.envKeys), - baseURL: entry.baseURL, + baseURL: baseURLFor(entry), fetch: oauth ? createAnthropicOAuthFetch(createTimeoutFetch()) : createTimeoutFetch(), diff --git a/apps/web/app/components/AgentBenchmark.tsx b/apps/web/app/components/AgentBenchmark.tsx index efd00e4..5a8e3b2 100644 --- a/apps/web/app/components/AgentBenchmark.tsx +++ b/apps/web/app/components/AgentBenchmark.tsx @@ -7,6 +7,9 @@ import { BenchBarList, type BenchBar } from "./BenchBarList"; const pct = (n: number) => `${Math.round(n * 100)}%`; const secs = (ms: number) => `${(ms / 1000).toFixed(0)}s`; +const tok = (n: number) => + n >= 1_000_000 ? `${(n / 1_000_000).toFixed(1)}M` : `${Math.round(n / 1000)}k`; +const usd = (n: number) => `$${n.toFixed(4)}`; export function AgentBenchmark({ views }: { views: BenchView[] }) { const [slug, setSlug] = useState(views[0]?.slug ?? ""); @@ -52,6 +55,38 @@ export function AgentBenchmark({ views }: { views: BenchView[] }) { highlight: a.isFreeCode, })); + // Metering (spec §6.4/§7): shown only for agents the proxy actually saw. + // An unmetered agent gets a zero-width bar labelled as such, never a zero — + // a blank is not the cheapest run in the table. + const anyMetered = view.agents.some((a) => a.meteredTrials > 0); + const tokenBars: BenchBar[] = view.agents.map((a) => ({ + label: a.id, + value: a.meanTokens ?? 0, + display: a.meanTokens !== undefined ? tok(a.meanTokens) : "unmetered", + note: + a.meteredTrials > 0 + ? `mean of ${a.meteredTrials} metered trial${a.meteredTrials === 1 ? "" : "s"}${a.meanTurns !== undefined ? ` · ~${Math.round(a.meanTurns)} turns` : ""}` + : "proxy saw no traffic", + highlight: a.isFreeCode, + })); + const costBars: BenchBar[] = view.agents.map((a) => ({ + label: a.id, + value: typeof a.meanUsd === "number" ? a.meanUsd : 0, + display: + typeof a.meanUsd === "number" + ? usd(a.meanUsd) + : a.meanUsd === null + ? "unpriced" + : "unmetered", + note: + a.worstUsd !== undefined + ? `worst single trial ${usd(a.worstUsd)}` + : a.meteredTrials > 0 + ? "no rate-card row for this model" + : "proxy saw no traffic", + highlight: a.isFreeCode, + })); + // Headline cards, freecode against the strongest rival on each axis. The // comparison is deliberately unflattering — "slower" goes in the headline in // red, because a benchmark we publish only when we win is an advertisement. @@ -329,6 +364,36 @@ export function AgentBenchmark({ views }: { views: BenchView[] }) { bars={sizeBars} /> + {anyMetered && ( + + )} + + {anyMetered && ( + + `${p.id} ${typeof p.meanUsd === "number" ? usd(p.meanUsd) : "unpriced"}${p.meanTokens !== null ? ` (${tok(p.meanTokens)} tok)` : ""}`, + ) + .join(" · ")}. Averaging over failures would make the quitter cheapest — this mean covers solved bugs only.` + : "Per-solved-bug cost (spec §7.2) appears once the matchup is graded — averaging cost over failed attempts would make the agent that gives up fastest look cheapest." + } + /> + )} +

Per instance

@@ -357,7 +422,7 @@ export function AgentBenchmark({ views }: { views: BenchView[] }) { {row.cells.map((cell) => ( { return s.length % 2 ? s[mid]! : (s[mid - 1]! + s[mid]!) / 2; }; +const mean = (xs: number[]) => + xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : undefined; + +const totalTokens = (r: RawResult) => + r.inputTokens !== undefined ? r.inputTokens + (r.outputTokens ?? 0) : undefined; + export function deriveView(raw: RawBenchmark): BenchView { const { results } = raw; @@ -125,6 +169,10 @@ export function deriveView(raw: RawBenchmark): BenchView { const successes = mine.filter((r) => raw.graded ? r.resolved === true : r.producedPatch, ).length; + // Metered means the proxy saw model calls — a trial where the agent + // talked around the proxy still writes zeros, and zeros are not a $0 run. + const metered = mine.filter((r) => (r.turns ?? 0) > 0); + const usds = metered.map((r) => r.usd).filter((u): u is number => typeof u === "number"); return { id: a.id, isFreeCode: a.id === "freecode", @@ -136,6 +184,13 @@ export function deriveView(raw: RawBenchmark): BenchView { rate: mine.length ? successes / mine.length : 0, medianMs: median(mine.map((r) => r.durationMs)), medianPatchBytes: median(mine.map((r) => r.patchBytes)), + meteredTrials: metered.length, + meanTokens: mean(metered.map((r) => totalTokens(r)!)), + // Metered but unpriced (no rate-card row) is null, not absent — the + // distinction between "we didn't measure" and "we can't price". + meanUsd: metered.length ? (usds.length ? mean(usds)! : null) : undefined, + worstUsd: usds.length ? Math.max(...usds) : undefined, + meanTurns: mean(metered.map((r) => r.turns ?? 0)), }; }); @@ -158,9 +213,46 @@ export function deriveView(raw: RawBenchmark): BenchView { patchBytes: r.patchBytes, reason: r.reason, trial: r.trial, + tokens: (r.turns ?? 0) > 0 ? totalTokens(r) : undefined, + usd: (r.turns ?? 0) > 0 ? r.usd : undefined, + auditOk: r.auditOk, })), })); + /** + * §7.2, executed: cost is compared only on instances every agent resolved + * at least once, and suppressed entirely below 3 such instances. Only a + * graded matchup can have this table at all. + */ + let cost: CostComparison | undefined; + if (raw.graded) { + const solvedByAll = sharedInstances.filter((id) => + raw.agents.every((a) => + results.some((r) => r.agent === a.id && r.instanceId === id && r.resolved === true), + ), + ); + cost = { + instances: solvedByAll, + suppressed: solvedByAll.length < 3, + perAgent: raw.agents.map((a) => { + const mine = results.filter( + (r) => + r.agent === a.id && + solvedByAll.includes(r.instanceId) && + r.resolved === true && + (r.turns ?? 0) > 0, + ); + const usds = mine.map((r) => r.usd).filter((u): u is number => typeof u === "number"); + return { + id: a.id, + isFreeCode: a.id === "freecode", + meanUsd: usds.length ? mean(usds)! : null, + meanTokens: mine.length ? mean(mine.map((r) => totalTokens(r)!))! : null, + }; + }), + }; + } + /** * Everything not yet true about this matchup, in the page's own words. Spec * §9 wants the caveats on the page rather than in a footnote and §10 wants @@ -184,6 +276,16 @@ export function deriveView(raw: RawBenchmark): BenchView { }, ] : []), + ...(agents.some((a) => a.trials > 0 && a.meteredTrials < a.trials) + ? [ + { + title: "Not every trial was metered", + body: `Token and cost figures cover only trials the recording proxy saw (${agents + .map((a) => `${a.id}: ${a.meteredTrials}/${a.trials}`) + .join(", ")}). An agent that ignores the proxy env vars is unmetered until the container forces its egress through the proxy — comparing its cost column would compare a measurement to a blank.`, + }, + ] + : []), ...(raw.runs.length > 1 ? [ { @@ -233,6 +335,7 @@ export function deriveView(raw: RawBenchmark): BenchView { sharedInstances, ragged, matrix, + cost, caveats, }; } diff --git a/apps/web/app/data/benchmarks/freecode-vs-claude-code.json b/apps/web/app/data/benchmarks/freecode-vs-claude-code.json index 7cff947..df27ea2 100644 --- a/apps/web/app/data/benchmarks/freecode-vs-claude-code.json +++ b/apps/web/app/data/benchmarks/freecode-vs-claude-code.json @@ -1,8 +1,24 @@ { "slug": "freecode-vs-claude-code", - "generatedAt": "2026-09-03T19:18:04.474Z", - "runId": "2026-09-03T18-56-04-591Z", + "generatedAt": "2026-09-05T23:06:41.774Z", + "runId": "2026-09-05T23-06-02-611Z", "runs": [ + { + "runId": "2026-09-05T23-06-02-611Z", + "generatedAt": "2026-09-05T23:06:41.774Z", + "agents": [ + "freecode", + "claude-code" + ] + }, + { + "runId": "2026-09-05T23-02-42-713Z", + "generatedAt": "2026-09-05T23:03:35.231Z", + "agents": [ + "freecode", + "claude-code" + ] + }, { "runId": "2026-09-03T18-56-04-591Z", "generatedAt": "2026-09-03T19:18:04.474Z", @@ -43,7 +59,7 @@ "agents": [ { "id": "freecode", - "version": "0.28.0", + "version": "0.30.0", "model": "minimax/MiniMax-M3", "autonomy": "--agent danger: no permission prompts" }, @@ -61,11 +77,18 @@ "trial": 1, "producedPatch": true, "resolved": null, - "durationMs": 29849, + "durationMs": 10799, "patchBytes": 625, "newFiles": 0, "reason": "ok", - "runId": "2026-09-03T16-51-56-681Z" + "runId": "2026-09-05T23-06-02-611Z", + "turns": 10, + "inputTokens": 140560, + "outputTokens": 1304, + "cacheReadTokens": 124250, + "cacheWriteTokens": 0, + "usd": 0.013912800000000001, + "auditOk": true }, { "agent": "claude-code", @@ -73,11 +96,18 @@ "trial": 1, "producedPatch": true, "resolved": null, - "durationMs": 9729, - "patchBytes": 625, + "durationMs": 27158, + "patchBytes": 1099, "newFiles": 0, "reason": "ok", - "runId": "2026-09-03T16-51-56-681Z" + "runId": "2026-09-05T23-06-02-611Z", + "turns": 14, + "inputTokens": 427431, + "outputTokens": 2005, + "cacheReadTokens": 394362, + "cacheWriteTokens": 0, + "usd": 0.03598841999999999, + "auditOk": false }, { "agent": "freecode", diff --git a/bench/agent-bench/README.md b/bench/agent-bench/README.md index cab0a64..8332e25 100644 --- a/bench/agent-bench/README.md +++ b/bench/agent-bench/README.md @@ -1,160 +1,7 @@ -# agent-bench — command reference +# agent-bench -> freecode against Claude Code, Codex and OpenCode, on SWE-bench Lite. -> Design lives in `docs/superpowers/specs/2026-09-03-agent-comparison-benchmark.md`; -> this is the "what do I type" page. +Operator reference is **[`AGENT-BENCH.md`](../../AGENT-BENCH.md)** at the repo +root — same job as `EVAL.md` / `TRACE.md`: what to type, which flag, when to +run it. -**Status: Phase 0.** No container, no grader, no metering. It answers one -question — *does every adapter produce a non-empty patch* — and every record it -writes carries `isolation: "none"` so a Phase 0 result cannot be mistaken for a -publishable one. **Nothing here produces a number worth showing anyone yet.** - -This is not `pnpm eval`. That measures *our* agent against its own past -(`EVAL.md`). This measures agents against each other, and shares no code with it -on purpose — only `scorers/outcome.ts`'s idea survives the trip, because it is -the only scorer that never asks what produced the diff (spec §4). - ---- - -## 1. One-time setup - -```bash -tsx bench/agent-bench/runner/fetch.ts # 114 django rows -> .cache/instances.jsonl -export MINIMAX_API_KEY=$(node -p "require(process.env.HOME+'/.freecode/config.json').providers.minimax.apiKey") -``` - -Every agent runs **MiniMax-M3 on that one key** — freecode natively, the others -through MiniMax's Anthropic-compatible endpoint. That is the "same model, one -bill" property (spec §5) and it is the only reason a cost column would mean -anything. No adapter file contains a credential; `${MINIMAX_API_KEY}` is -expanded at spawn time and an unset variable is a hard error, because an agent -that quietly fell back to its own key would be billed somewhere else. - -`fetch.ts` is needed **once, ever**. datasets-server 500s and 502s while its -index warms (`"the dataset index is loading"`), and the Hub has outages of its -own; the fetch backs off five times and then stops, leaving any existing cache -untouched. A run whose instances are already cached does not touch the network -at all, so an outage is only ever a problem for instances you do not have yet. - -The cache stores four fields per instance. `patch`, `test_patch` and -`hints_text` — the gold fix and the maintainer discussion that usually contains -it — are dropped before anything touches disk. Not to protect the agent under -test, which cannot see this repo: to keep an answer key out of a repository that -agents work in every day. - -The django mirror (~250MB) is cloned on first use into `.cache/repos/` and -hardlinked per trial, so a run does not measure GitHub's mood. - -## 2. Run - -```bash -pnpm bench:agents --instances django__django-10914 --trials 1 -pnpm bench:agents --agents freecode,claude-code --trials 3 -``` - -| Flag | Default | Does what | -| --- | --- | --- | -| `--agents` | `freecode,claude-code` | comma-separated adapter ids from `agents/` | -| `--instances` | every id in `instances/django-lite.txt` | comma-separated instance ids | -| `--trials` | `1` | trials per (agent, instance). Phase 2 onward: 3, and publish the spread | -| `--timeout` | `900000` | per-trial wall-clock cap, ms | -| `--out` | `results/` | artifact root | - -Exit code is **non-zero if any trial produced an empty patch**. In Phase 0 that -is the entire verdict: a silently empty patch is a broken adapter, and a broken -adapter that reports as a lost benchmark is the worst failure this harness has. - -## 3. Artifacts - -``` -results///trial-// - prompt.txt the exact task text — identical for every agent - argv.json the exact command, after {prompt}/{model} substitution - patch.diff git diff of the workspace, staged (so new files count) - stdout.log stderr.log -results//report.json -``` - -`results/` and `.cache/` are git-ignored. Transcripts get reviewed before they -become public. - -Every run also writes **`apps/web/app/data/benchmarks/.json`** — one -file per agent set, e.g. `freecode-vs-opencode.json` — and that is what the -`/benchmark` page reads, one tab per file. A finished run is already on the -page, with no extra step. It carries the numbers and the disclosures (version, model, autonomy, -isolation, graded) and none of the transcripts, because `results/` does not -exist on a deploy. Re-point the page at an older run without paying to re-run -it: - -```bash -tsx bench/agent-bench/runner/publish.ts results/ [--fresh] -cd apps/web && pnpm dev # http://localhost:3000/benchmark -``` - -**A matchup is the unit of comparison, which is why it is the unit of storage.** -Runs of the same agent set merge into one file, keyed by (agent, instance, -trial) with the newest run winning; a run with a different agent set writes a -different file rather than quietly widening an existing matchup with rows nobody -measured side by side. Adding a pairing therefore adds a tab — `page.tsx` reads -the directory at build time, so there is no import list to update. `--fresh` -discards a matchup's history, which is the honest move whenever the meaning of -its numbers changes: a new model, the grader landing, the container landing. - -The page refuses to flatter the data: while `graded` is false it labels the -headline bar "Produced a patch", says in the footnote that everyone scores 100% -as soon as they edit anything, and shows a banner saying it is a pipeline check -rather than a result. Those come from `graded` and `isolation` in the JSON, so -they disappear on their own when Phase 1 lands — nobody has to remember. - -## 4. Adding an agent - -One file in `agents/`. Required: `id`, `versionCmd`, `run` (a `{prompt}` / -`{model}` template), `model`, `autonomy`. - -**`autonomy` is not documentation, it is the experiment.** freecode in `build` -mode denies every headless write (`permission/prompt.ts` — the same reason -`apps/core/src/eval/runner.ts:157` has to answer permission prompts), so running -freecode in `build` against a competitor's full-auto measures permission -defaults, not agents. Every agent runs at its own maximum, the flag that got it -there is recorded, and it is printed in the results table. - -Flags verified on this machine, 2026-09-03 — **the adapter is the source of -truth, this table just says where the two shipped ones came from and what the -two unshipped ones will look like:** - -| Agent | Version | Full autonomy | Shipped? | -| --- | --- | --- | --- | -| freecode | local | `run "

" --model

--agent danger --max-turns 40` | yes | -| claude | 2.1.251 | `-p "

" --dangerously-skip-permissions --model ` | yes | -| codex | 0.151.0 | `exec --dangerously-bypass-approvals-and-sandbox --ephemeral --ignore-user-config` | Phase 3 | -| opencode | 1.18.25 | `run --pure --auto --model

"

"` + `XDG_CONFIG_HOME` | yes | - -Codex is the only one left, and only because nothing has verified which model it -and freecode can both be pinned to (spec §12.1) — not because the invocation is -unknown. Its `--ephemeral --ignore-user-config` is worth noting: that is exactly -the "no config, no history carried between trials" property §6.3 wants, and the -other three need a container to get it. opencode needed `XDG_CONFIG_HOME` for a -weaker version of the same thing — see `empty-config/README.md`. - -## 5. Known gaps — what Phase 0 does not do - -1. **No isolation.** No container, so: the network is open (an agent *could* - look the fix up), `$HOME` is mounted (freecode writes sessions and memory to - `~/.freecode` and carries them between trials), and the agent's own config - applies. Every record says `isolation: "none"` for this reason. Phase 1. -2. **No grading.** "Produced a patch" is not "fixed the bug". The official - SWE-bench grader needs Docker, and on this machine the daemon is running but - the user is not in the `docker` group. Phase 1. -3. **No metering.** Cost and token columns need the recording proxy (§6.4); - four vendors' self-reports are four rounding policies. Phase 1. -4. **Model parity depends on one env var staying right.** Both agents are pinned - to `MiniMax-M3`, but MiniMax's `/anthropic` shim reports a 200K context - window instead of M3's real 1048576, so Claude Code auto-compacts at ~167K - unless `CLAUDE_CODE_AUTO_COMPACT_WINDOW` says otherwise. The adapter sets it. - **Delete that line and freecode wins on a handicap** — this class of bug is - invisible in the results and only findable by reading the adapters, which is - why they are committed and short. -5. **Scratch files land in the patch.** `extractPatch` stages everything, so an - agent that leaves `notes.md` behind ships it in the diff. They are listed in - `newFiles` rather than filtered — a fix accompanied by six scratch files is a - fact about the agent worth seeing. +Design: `docs/superpowers/specs/2026-09-03-agent-comparison-benchmark.md`. diff --git a/bench/agent-bench/isolate/Dockerfile b/bench/agent-bench/isolate/Dockerfile new file mode 100644 index 0000000..d514ac6 --- /dev/null +++ b/bench/agent-bench/isolate/Dockerfile @@ -0,0 +1,35 @@ +# One image, every agent. Built ONLINE (`pnpm bench:image`); trials then run +# it on an --internal network with no route out, so everything an agent needs +# must already be in here. Versions are build args so a run can pin exactly +# what it benchmarked — they default to the versions verified in +# AGENT-BENCH.md §5 and should move in lockstep with the adapter files. + +FROM node:22-bookworm-slim + +ARG CLAUDE_CODE_VERSION=2.1.251 +ARG OPENCODE_VERSION=1.18.25 +# A released freecode binary — the self-contained bun build, never the SEA one. +ARG FREECODE_VERSION=v0.30.0 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +RUN npm install -g \ + @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} \ + opencode-ai@${OPENCODE_VERSION} + +RUN curl -fsSL -o /tmp/freecode.tar.gz \ + https://github.com/ayan-de/freecode/releases/download/${FREECODE_VERSION}/freecode-linux-x86_64.tar.gz \ + && mkdir -p /opt/freecode \ + && tar -xzf /tmp/freecode.tar.gz -C /opt/freecode \ + && ln -s /opt/freecode/freecode /usr/local/bin/freecode \ + && rm /tmp/freecode.tar.gz + +# Trials run --user : of the operator, so HOME must be writable by +# anyone. Agent CLIs write their config/state here and it dies with the +# container — which is the point: no memory carried between trials (§6.3). +RUN mkdir -p /tmp/agent-home && chmod 777 /tmp/agent-home +ENV HOME=/tmp/agent-home + +WORKDIR /workspace diff --git a/bench/agent-bench/isolate/docker.test.ts b/bench/agent-bench/isolate/docker.test.ts new file mode 100644 index 0000000..2825943 --- /dev/null +++ b/bench/agent-bench/isolate/docker.test.ts @@ -0,0 +1,33 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { dockerArgv, forwardedEnvNames, type Containerize } from "./docker.js"; + +const c: Containerize = { + image: "agent-bench", + network: "agent-bench-internal", + name: "trial-x", + wsDir: "/tmp/ws", + benchDir: "/repo/bench/agent-bench", + envNames: ["ANTHROPIC_BASE_URL", "MINIMAX_API_KEY"], + uid: 1000, + gid: 1000, +}; + +test("dockerArgv: env rides as bare -e NAME — no secret ever lands in argv", () => { + const argv = dockerArgv(c, ["freecode", "run", "fix it"]); + assert.equal(argv.includes("MINIMAX_API_KEY"), true); + assert.equal(argv.some((a) => a.includes("MINIMAX_API_KEY=")), false); + assert.deepEqual(argv.slice(-3), ["freecode", "run", "fix it"]); + assert.equal(argv[argv.indexOf("--network") + 1], "agent-bench-internal"); + assert.equal(argv[argv.indexOf("--user") + 1], "1000:1000"); + assert.equal(argv.includes("/tmp/ws:/workspace"), true); + assert.equal(argv.includes("/repo/bench/agent-bench:/bench:ro"), true); +}); + +test("forwardedEnvNames: adapter names + meter names; \"\" (unset) is not forwarded", () => { + const names = forwardedEnvNames( + { XDG_CONFIG_HOME: "{benchDir}/empty-config", ANTHROPIC_API_KEY: "" }, + { MINIMAX_BASE_URL: "http://x", ANTHROPIC_BASE_URL: "http://x" }, + ); + assert.deepEqual(names, ["ANTHROPIC_BASE_URL", "MINIMAX_BASE_URL", "XDG_CONFIG_HOME"]); +}); diff --git a/bench/agent-bench/isolate/docker.ts b/bench/agent-bench/isolate/docker.ts new file mode 100644 index 0000000..581328b --- /dev/null +++ b/bench/agent-bench/isolate/docker.ts @@ -0,0 +1,102 @@ +// ============================================================================= +// Container isolation (spec §6.3). One container per trial on an --internal +// docker network: no route to the internet, so the only way out is the +// recording proxy listening on the network's gateway (the host side of the +// bridge). That makes the proxy log a real egress audit instead of an honor +// system — with the stated residue that OTHER host services on the gateway +// IP remain reachable; the network namespace blocks the internet, the proxy +// log still audits what was actually sent. +// +// Secrets never enter argv: env vars ride as bare `-e NAME` flags, which +// docker resolves from the spawning process's environment. argv.json stays +// publishable. +// ============================================================================= + +import { spawnSync } from "child_process"; + +export const IMAGE = "agent-bench"; +export const NETWORK = "agent-bench-internal"; +/** Where the workspace and bench dir land inside the container. */ +export const WORKSPACE = "/workspace"; +export const BENCH_MOUNT = "/bench"; + +export interface Containerize { + image: string; + network: string; + /** Unique per trial, so a timeout can `docker rm -f` exactly this one. */ + name: string; + /** Host workspace dir, mounted rw at /workspace. */ + wsDir: string; + /** Host bench/agent-bench dir, mounted ro at /bench (empty-config lives there). */ + benchDir: string; + /** Env NAMES to forward. Values come from the spawn env, never argv. */ + envNames: string[]; + uid: number; + gid: number; +} + +/** `docker run` argv around an agent's own argv. Pure — unit-testable. */ +export function dockerArgv(c: Containerize, argv: string[]): string[] { + return [ + "docker", "run", "--rm", "--init", + "--name", c.name, + "--network", c.network, + "--user", `${c.uid}:${c.gid}`, + "-v", `${c.wsDir}:${WORKSPACE}`, + "-v", `${c.benchDir}:${BENCH_MOUNT}:ro`, + "-w", WORKSPACE, + // Writable HOME for CLIs that insist on one; --user means /root is not it. + "-e", "HOME=/tmp/agent-home", + ...c.envNames.flatMap((n) => ["-e", n]), + c.image, + ...argv, + ]; +} + +/** The env names worth forwarding: the adapter's own, plus the meter's. */ +export function forwardedEnvNames( + adapterEnv: Record | undefined, + meterEnv: NodeJS.ProcessEnv | undefined, +): string[] { + const names = new Set(); + for (const [k, v] of Object.entries(adapterEnv ?? {})) { + if (v !== "") names.add(k); // "" means unset — simply do not forward it + } + for (const k of Object.keys(meterEnv ?? {})) names.add(k); + return [...names].sort(); +} + +function docker(args: string[]): { ok: boolean; out: string } { + const r = spawnSync("docker", args, { encoding: "utf-8" }); + return { ok: r.status === 0, out: (r.stdout ?? "").trim() }; +} + +export function dockerAvailable(): boolean { + return docker(["info", "--format", "{{.ServerVersion}}"]).ok; +} + +export function imageExists(image: string): boolean { + return docker(["image", "inspect", image, "--format", "ok"]).ok; +} + +/** + * Create the internal network if missing; return the gateway IP the proxy + * should bind. `--internal` is the isolation: docker programs no masquerade + * for it, so nothing routes past the bridge. + */ +export function ensureInternalNetwork(name: string): string { + const fmt = ["--format", "{{(index .IPAM.Config 0).Gateway}}"]; + let r = docker(["network", "inspect", name, ...fmt]); + if (!r.ok) { + const created = docker(["network", "create", "--internal", name]); + if (!created.ok) throw new Error(`docker network create ${name} failed`); + r = docker(["network", "inspect", name, ...fmt]); + } + if (!r.ok || !r.out) throw new Error(`no gateway on docker network ${name}`); + return r.out; +} + +/** Best-effort teardown after a timeout — the client dying does not stop a container. */ +export function removeContainer(name: string): void { + spawnSync("docker", ["rm", "-f", name], { stdio: "ignore" }); +} diff --git a/bench/agent-bench/proxy/audit.test.ts b/bench/agent-bench/proxy/audit.test.ts new file mode 100644 index 0000000..1c7ea74 --- /dev/null +++ b/bench/agent-bench/proxy/audit.test.ts @@ -0,0 +1,28 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { destUrl } from "./forward.js"; +import { audit, isModelEndpoint } from "./audit.js"; + +test("keeps the upstream path prefix when joining /v1/messages", () => { + assert.equal( + destUrl("https://api.minimax.io/anthropic", "/v1/messages").href, + "https://api.minimax.io/anthropic/v1/messages", + ); +}); + +test("messages and chat/completions are the model; a GitHub path is not", () => { + assert.equal(isModelEndpoint("/v1/messages"), true); + assert.equal(isModelEndpoint("/messages"), true); + assert.equal(isModelEndpoint("/v1/chat/completions"), true); + assert.equal(isModelEndpoint("/repos/django/django"), false); +}); + +test("audit fails closed on any non-model path", () => { + const result = audit([ + { modelEndpoint: true, method: "POST", path: "/v1/messages", host: "api.minimax.io", status: 200 }, + { modelEndpoint: false, method: "GET", path: "/repos/django/django", host: "github.com", status: 200 }, + ]); + assert.equal(result.ok, false); + assert.equal(result.modelCalls, 1); + assert.equal(result.leaks[0]?.path, "/repos/django/django"); +}); diff --git a/bench/agent-bench/proxy/audit.ts b/bench/agent-bench/proxy/audit.ts new file mode 100644 index 0000000..9c89bb4 --- /dev/null +++ b/bench/agent-bench/proxy/audit.ts @@ -0,0 +1,34 @@ +export function isModelEndpoint(pathname: string): boolean { + return /\/(v\d+\/)?(messages|chat\/completions)\/?$/.test(pathname); +} + +export interface IsolationAudit { + ok: boolean; + modelCalls: number; + leaks: { method: string; path: string; host: string; status: number }[]; +} + +/** The proxy log is the list of everything that left. A non-model path is a leak. */ +export function audit( + calls: { + modelEndpoint: boolean; + method: string; + path: string; + host: string; + status: number; + }[], +): IsolationAudit { + const leaks = calls + .filter((c) => !c.modelEndpoint) + .map((c) => ({ + method: c.method, + path: c.path, + host: c.host, + status: c.status, + })); + return { + ok: leaks.length === 0, + modelCalls: calls.filter((c) => c.modelEndpoint).length, + leaks, + }; +} diff --git a/bench/agent-bench/proxy/env.test.ts b/bench/agent-bench/proxy/env.test.ts new file mode 100644 index 0000000..6871e19 --- /dev/null +++ b/bench/agent-bench/proxy/env.test.ts @@ -0,0 +1,19 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { DEFAULT_UPSTREAM, meterEnv, upstreamFor } from "./env.js"; + +test("Claude gets the origin, freecode gets the origin plus /v1", () => { + assert.deepEqual(meterEnv("http://127.0.0.1:9"), { + ANTHROPIC_BASE_URL: "http://127.0.0.1:9", + MINIMAX_BASE_URL: "http://127.0.0.1:9/v1", + }); +}); + +test("upstream is the adapter's Anthropic URL when it is a literal", () => { + assert.equal( + upstreamFor({ env: { ANTHROPIC_BASE_URL: "https://api.minimax.io/anthropic" } }), + "https://api.minimax.io/anthropic", + ); + assert.equal(upstreamFor({ env: { ANTHROPIC_BASE_URL: "${MINIMAX_API_KEY}" } }), DEFAULT_UPSTREAM); + assert.equal(upstreamFor({}), DEFAULT_UPSTREAM); +}); diff --git a/bench/agent-bench/proxy/env.ts b/bench/agent-bench/proxy/env.ts new file mode 100644 index 0000000..4b08fb9 --- /dev/null +++ b/bench/agent-bench/proxy/env.ts @@ -0,0 +1,22 @@ +export const DEFAULT_UPSTREAM = "https://api.minimax.io/anthropic"; + +/** + * Point every current adapter at the recording proxy. + * + * Claude Code reads `ANTHROPIC_BASE_URL` (no `/v1` suffix — it adds it). + * Freecode's MiniMax SDK is constructed with the catalogue URL + * (`.../anthropic/v1`), so `MINIMAX_BASE_URL` carries that suffix. + */ +export function meterEnv(origin: string): Record { + const base = origin.replace(/\/$/, ""); + return { + ANTHROPIC_BASE_URL: base, + MINIMAX_BASE_URL: `${base}/v1`, + }; +} + +export function upstreamFor(spec: { env?: Record }): string { + const raw = spec.env?.ANTHROPIC_BASE_URL; + if (raw && raw.startsWith("http") && !raw.includes("${")) return raw; + return DEFAULT_UPSTREAM; +} diff --git a/bench/agent-bench/proxy/fold.test.ts b/bench/agent-bench/proxy/fold.test.ts new file mode 100644 index 0000000..ade498f --- /dev/null +++ b/bench/agent-bench/proxy/fold.test.ts @@ -0,0 +1,64 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { foldTrial, readLog } from "./fold.js"; +import type { LoggedCall } from "./server.js"; + +function call(partial: Partial & Pick): LoggedCall { + return { + ts: "t", + method: "POST", + path: "/v1/messages", + host: "api.minimax.io", + status: 200, + durationMs: 1, + requestBytes: 0, + responseBytes: 0, + ...partial, + }; +} + +test("folds two model calls into turns, tokens, and MiniMax-M3 USD", () => { + const calls: LoggedCall[] = [ + call({ + modelEndpoint: true, + model: "MiniMax-M3", + usage: { + inputTokens: 1_000_000, + outputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + }), + call({ + modelEndpoint: true, + usage: { + inputTokens: 0, + outputTokens: 1_000_000, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + }), + ]; + const folded = foldTrial(calls, "MiniMax-M3"); + assert.equal(folded.turns, 2); + assert.equal(folded.inputTokens, 1_000_000); + assert.equal(folded.outputTokens, 1_000_000); + assert.equal(folded.usd, 1.5); + assert.equal(folded.leaks, 0); +}); + +test("a leak does not count as a turn and flags the fold", () => { + const folded = foldTrial( + [call({ modelEndpoint: false, path: "/repos/x" })], + "MiniMax-M3", + ); + assert.equal(folded.turns, 0); + assert.equal(folded.leaks, 1); + assert.equal(folded.auditOk, false); +}); + +test("an empty log is unmetered, not a cheap clean run", () => { + const folded = foldTrial([], "MiniMax-M3"); + assert.equal(folded.turns, 0); + assert.equal(folded.auditOk, false); +}); diff --git a/bench/agent-bench/proxy/fold.ts b/bench/agent-bench/proxy/fold.ts new file mode 100644 index 0000000..cc3e6b8 --- /dev/null +++ b/bench/agent-bench/proxy/fold.ts @@ -0,0 +1,73 @@ +import * as fs from "fs"; +import * as path from "path"; +import { audit } from "./audit.js"; +import { priceUsd } from "./price.js"; +import type { LoggedCall } from "./server.js"; +import type { Usage } from "./usage.js"; + +export interface TrialMeter { + turns: number; + inputTokens: number; + outputTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + usd: number | undefined; + leaks: number; + auditOk: boolean; +} + +const ZERO: Usage = { + inputTokens: 0, + outputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, +}; + +export function readLog(file: string): LoggedCall[] { + if (!fs.existsSync(file)) return []; + return fs + .readFileSync(file, "utf-8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as LoggedCall); +} + +export function foldTrial(calls: LoggedCall[], model: string): TrialMeter { + const modelCalls = calls.filter((c) => c.modelEndpoint); + const usage = modelCalls.reduce( + (acc, c) => ({ + inputTokens: acc.inputTokens + (c.usage?.inputTokens ?? 0), + outputTokens: acc.outputTokens + (c.usage?.outputTokens ?? 0), + cacheReadTokens: acc.cacheReadTokens + (c.usage?.cacheReadTokens ?? 0), + cacheWriteTokens: acc.cacheWriteTokens + (c.usage?.cacheWriteTokens ?? 0), + }), + { ...ZERO }, + ); + const isolation = audit(calls); + return { + turns: modelCalls.length, + ...usage, + usd: priceUsd(model, usage), + leaks: isolation.leaks.length, + // An empty log is not a clean audit: the agent never went through the + // meter (wrong base URL, or it talked around us). + auditOk: isolation.ok && modelCalls.length > 0, + }; +} + +/** Fold the jsonl into usage.json + audit.json next to it. */ +export function persistTrialMeter(artifactDir: string, model: string): TrialMeter { + const logPath = path.join(artifactDir, "proxy.jsonl"); + const calls = readLog(logPath); + const usage = foldTrial(calls, model); + fs.writeFileSync( + path.join(artifactDir, "usage.json"), + JSON.stringify(usage, null, 2) + "\n", + ); + fs.writeFileSync( + path.join(artifactDir, "audit.json"), + JSON.stringify(audit(calls), null, 2) + "\n", + ); + return usage; +} diff --git a/bench/agent-bench/proxy/forward.ts b/bench/agent-bench/proxy/forward.ts new file mode 100644 index 0000000..c8d3007 --- /dev/null +++ b/bench/agent-bench/proxy/forward.ts @@ -0,0 +1,35 @@ +import type { IncomingMessage } from "http"; + +const HOP = new Set([ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailers", + "transfer-encoding", + "upgrade", + "host", +]); + +export function isHopByHop(name: string): boolean { + return HOP.has(name.toLowerCase()); +} + +/** Join the incoming path onto the upstream prefix without collapsing it. */ +export function destUrl(upstream: string, reqUrl: string): URL { + const prefix = new URL(upstream); + const incoming = new URL(reqUrl, "http://proxy.local"); + const base = prefix.pathname.replace(/\/$/, ""); + return new URL(base + incoming.pathname + incoming.search, prefix.origin); +} + +export function outboundHeaders(req: IncomingMessage, dest: URL): Headers { + const headers = new Headers(); + for (const [name, value] of Object.entries(req.headers)) { + if (!value || isHopByHop(name)) continue; + headers.set(name, Array.isArray(value) ? value.join(", ") : value); + } + headers.set("host", dest.host); + return headers; +} diff --git a/bench/agent-bench/proxy/price.test.ts b/bench/agent-bench/proxy/price.test.ts new file mode 100644 index 0000000..3138eb9 --- /dev/null +++ b/bench/agent-bench/proxy/price.test.ts @@ -0,0 +1,49 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { priceUsd } from "./price.js"; + +test("an unknown model prices as undefined, never as zero", () => { + assert.equal( + priceUsd("mystery-model", { + inputTokens: 1_000_000, + outputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }), + undefined, + ); +}); + +test("MiniMax-M3: 1M plain input is $0.30, 1M output is $1.20", () => { + assert.equal( + priceUsd("MiniMax-M3", { + inputTokens: 1_000_000, + outputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }), + 0.3, + ); + assert.equal( + priceUsd("minimax/MiniMax-M3", { + inputTokens: 0, + outputTokens: 1_000_000, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }), + 1.2, + ); +}); + +test("a cache read is a discount off inclusive input, not an addend", () => { + // 1M input of which 800k is a cache read: 200k × $0.30 + 800k × $0.06 = $0.108 + assert.equal( + priceUsd("MiniMax-M3", { + inputTokens: 1_000_000, + outputTokens: 0, + cacheReadTokens: 800_000, + cacheWriteTokens: 0, + }), + 0.108, + ); +}); diff --git a/bench/agent-bench/proxy/price.ts b/bench/agent-bench/proxy/price.ts new file mode 100644 index 0000000..0cd40e0 --- /dev/null +++ b/bench/agent-bench/proxy/price.ts @@ -0,0 +1,45 @@ +// One rate card for every agent. Spec §6.4 / §7: comparing four vendors' +// self-reports is comparing four rounding policies. Cache reads are a +// discount off the inclusive input count, matching providers/pricing.ts — +// adding them on top would report a cache win as a cost increase. +// +// This copy lives here on purpose: agent-bench shares no code with +// apps/core/src/eval, and importing pricing.ts would pull the models.dev +// catalogue (and whatever ~/.freecode/pricing.json the operator has) into +// a published number. A committed table with a vintage is the honest stamp. + +import type { Usage } from "./usage.js"; + +const MILLION = 1_000_000; + +/** USD per million tokens. Vintage: MiniMax pay-as-you-go, standard ≤512k, 2026-09. */ +export interface Rate { + input: number; + output: number; + cacheRead?: number; + cacheWrite?: number; +} + +export const RATES_AS_OF = "2026-09 MiniMax standard ≤512k"; + +const RATES: Record = { + "minimax-m3": { input: 0.3, output: 1.2, cacheRead: 0.06 }, +}; + +function key(model: string): string { + const bare = model.includes("/") ? model.slice(model.indexOf("/") + 1) : model; + return bare.toLowerCase(); +} + +export function priceUsd(model: string, usage: Usage): number | undefined { + const rate = RATES[key(model)]; + if (!rate) return undefined; + const cacheRead = usage.cacheReadTokens; + const cacheWrite = usage.cacheWriteTokens; + const plainInput = Math.max(0, usage.inputTokens - cacheRead - cacheWrite); + let usd = (plainInput / MILLION) * rate.input; + usd += (cacheRead / MILLION) * (rate.cacheRead ?? rate.input); + usd += (cacheWrite / MILLION) * (rate.cacheWrite ?? rate.input); + usd += (usage.outputTokens / MILLION) * rate.output; + return usd; +} diff --git a/bench/agent-bench/proxy/server.test.ts b/bench/agent-bench/proxy/server.test.ts new file mode 100644 index 0000000..ab676c4 --- /dev/null +++ b/bench/agent-bench/proxy/server.test.ts @@ -0,0 +1,154 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "fs"; +import * as http from "http"; +import * as os from "os"; +import * as path from "path"; +import { startProxy, type LoggedCall } from "./server.js"; + +function tmpLog(): string { + return path.join( + fs.mkdtempSync(path.join(os.tmpdir(), "agent-bench-proxy-")), + "proxy.jsonl", + ); +} + +function listen( + handler: http.RequestListener, +): Promise<{ origin: string; close: () => Promise }> { + const server = http.createServer(handler); + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const addr = server.address(); + if (!addr || typeof addr === "string") throw new Error("no port"); + resolve({ + origin: `http://127.0.0.1:${addr.port}`, + close: () => + new Promise((r, j) => server.close((e) => (e ? j(e) : r()))), + }); + }); + }); +} + +function readLog(file: string): LoggedCall[] { + return fs + .readFileSync(file, "utf-8") + .trim() + .split("\n") + .filter(Boolean) + .map((l) => JSON.parse(l) as LoggedCall); +} + +test("forwards the body unchanged and records Anthropic usage from the reply", async () => { + const logPath = tmpLog(); + let hits = 0; + const upstream = await listen((req, res) => { + hits++; + const chunks: Buffer[] = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => { + assert.equal(Buffer.concat(chunks).toString(), '{"model":"MiniMax-M3"}'); + assert.equal(req.headers["x-api-key"], "secret"); + res.setHeader("content-type", "application/json"); + res.end( + JSON.stringify({ + usage: { input_tokens: 12, output_tokens: 3, cache_read_input_tokens: 8 }, + }), + ); + }); + }); + const proxy = await startProxy({ upstream: upstream.origin, logPath }); + try { + const res = await fetch(`${proxy.origin}/v1/messages`, { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "secret" }, + body: '{"model":"MiniMax-M3"}', + }); + assert.equal(res.status, 200); + assert.equal(hits, 1); + const body = await res.json(); + assert.equal((body as { usage: { input_tokens: number } }).usage.input_tokens, 12); + + const [row] = readLog(logPath); + assert.equal(row.method, "POST"); + assert.equal(row.path, "/v1/messages"); + assert.equal(row.status, 200); + assert.equal(row.model, "MiniMax-M3"); + // Wire input_tokens (12) is exclusive; the log normalizes to inclusive. + assert.deepEqual(row.usage, { + inputTokens: 20, + outputTokens: 3, + cacheReadTokens: 8, + cacheWriteTokens: 0, + }); + assert.equal(row.modelEndpoint, true); + assert.equal(JSON.stringify(row).includes("secret"), false); + } finally { + await proxy.close(); + await upstream.close(); + } +}); + +test("does not retry a 500 — a retry would be an optimization of one side", async () => { + const logPath = tmpLog(); + let hits = 0; + const upstream = await listen((_req, res) => { + hits++; + res.statusCode = 500; + res.end("nope"); + }); + const proxy = await startProxy({ upstream: upstream.origin, logPath }); + try { + const res = await fetch(`${proxy.origin}/v1/messages`, { method: "POST" }); + assert.equal(res.status, 500); + assert.equal(hits, 1); + assert.equal(await res.text(), "nope"); + } finally { + await proxy.close(); + await upstream.close(); + } +}); + +test("SSE streams through and usage is taken from the last event", async () => { + const logPath = tmpLog(); + const sse = [ + "event: message_start", + 'data: {"type":"message_start","message":{"usage":{"input_tokens":4,"output_tokens":1}}}', + "", + "event: message_delta", + 'data: {"type":"message_delta","usage":{"output_tokens":9}}', + "", + ].join("\n"); + const upstream = await listen((_req, res) => { + res.setHeader("content-type", "text/event-stream"); + res.end(sse); + }); + const proxy = await startProxy({ upstream: upstream.origin, logPath }); + try { + const res = await fetch(`${proxy.origin}/v1/messages`, { method: "POST" }); + assert.equal(await res.text(), sse); + const [row] = readLog(logPath); + assert.equal(row.usage?.outputTokens, 9); + assert.equal(row.usage?.inputTokens, 4); + } finally { + await proxy.close(); + await upstream.close(); + } +}); + +test("a request that is not the model endpoint is recorded as a leak", async () => { + const logPath = tmpLog(); + const upstream = await listen((_req, res) => { + res.end("ok"); + }); + const proxy = await startProxy({ upstream: upstream.origin, logPath }); + try { + await fetch(`${proxy.origin}/repos/django/django`); + const [row] = readLog(logPath); + assert.equal(row.path, "/repos/django/django"); + assert.equal(row.modelEndpoint, false); + } finally { + await proxy.close(); + await upstream.close(); + } +}); diff --git a/bench/agent-bench/proxy/server.ts b/bench/agent-bench/proxy/server.ts new file mode 100644 index 0000000..bf1c599 --- /dev/null +++ b/bench/agent-bench/proxy/server.ts @@ -0,0 +1,151 @@ +// Pass-through recording proxy. Spec §6.4: no caching, no retries, no +// rewriting. Tokens come off the wire; the log is also the isolation audit. + +import * as fs from "fs"; +import * as http from "http"; +import * as path from "path"; +import { isModelEndpoint } from "./audit.js"; +import { destUrl, isHopByHop, outboundHeaders } from "./forward.js"; +import { parseUsage, usageFromSse, type Usage } from "./usage.js"; + +export interface LoggedCall { + ts: string; + method: string; + path: string; + host: string; + status: number; + durationMs: number; + requestBytes: number; + responseBytes: number; + model?: string; + usage?: Usage; + /** False when the path is not a model API — a leak, or an attempt at one. */ + modelEndpoint: boolean; +} + +export interface ProxyOptions { + /** Origin + path prefix the agent thinks it is talking to, e.g. MiniMax /anthropic. */ + upstream: string; + logPath: string; + /** + * Bind address. Default loopback; isolated trials bind the internal docker + * network's gateway instead, the one address an agent container can reach. + */ + host?: string; +} + +export interface RecordingProxy { + origin: string; + port: number; + close(): Promise; +} + +function readBody(req: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => resolve(Buffer.concat(chunks))); + req.on("error", reject); + }); +} + +function modelFrom(body: Buffer): string | undefined { + try { + const parsed = JSON.parse(body.toString("utf-8")); + return typeof parsed?.model === "string" ? parsed.model : undefined; + } catch { + return undefined; + } +} + +function usageFrom(body: Buffer, contentType: string | undefined): Usage | undefined { + const text = body.toString("utf-8"); + if (contentType?.includes("event-stream") || text.startsWith("event:")) { + return usageFromSse(text); + } + try { + return parseUsage(JSON.parse(text)); + } catch { + return usageFromSse(text); + } +} + +function appendLog(file: string, row: LoggedCall): void { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.appendFileSync(file, JSON.stringify(row) + "\n"); +} + +export async function startProxy(opts: ProxyOptions): Promise { + const server = http.createServer(async (req, res) => { + const started = Date.now(); + const reqUrl = req.url ?? "/"; + const dest = destUrl(opts.upstream, reqUrl); + const method = req.method ?? "GET"; + let status = 502; + let responseBytes = 0; + let requestBytes = 0; + let model: string | undefined; + let usage: Usage | undefined; + try { + const body = await readBody(req); + requestBytes = body.length; + model = modelFrom(body); + const init: RequestInit = { + method, + headers: outboundHeaders(req, dest), + }; + if (method !== "GET" && method !== "HEAD") init.body = new Uint8Array(body); + const up = await fetch(dest, init); + status = up.status; + for (const [name, value] of up.headers) { + if (isHopByHop(name)) continue; + res.setHeader(name, value); + } + res.statusCode = status; + const chunks: Buffer[] = []; + if (up.body) { + for await (const chunk of up.body) { + const buf = Buffer.from(chunk); + chunks.push(buf); + responseBytes += buf.length; + res.write(buf); + } + } + res.end(); + usage = usageFrom(Buffer.concat(chunks), up.headers.get("content-type") ?? undefined); + } catch (err) { + if (!res.headersSent) { + res.statusCode = 502; + res.end(String((err as Error).message)); + } else { + res.end(); + } + } finally { + const pathname = new URL(reqUrl, "http://proxy.local").pathname; + appendLog(opts.logPath, { + ts: new Date().toISOString(), + method, + path: pathname, + host: dest.host, + status, + durationMs: Date.now() - started, + requestBytes, + responseBytes, + model, + usage, + modelEndpoint: isModelEndpoint(pathname), + }); + } + }); + + const host = opts.host ?? "127.0.0.1"; + await new Promise((resolve) => server.listen(0, host, resolve)); + const addr = server.address(); + if (!addr || typeof addr === "string") throw new Error("proxy: no port"); + return { + origin: `http://${host}:${addr.port}`, + port: addr.port, + close: () => + new Promise((resolve, reject) => server.close((e) => (e ? reject(e) : resolve()))), + }; +} diff --git a/bench/agent-bench/proxy/usage.test.ts b/bench/agent-bench/proxy/usage.test.ts new file mode 100644 index 0000000..e70f553 --- /dev/null +++ b/bench/agent-bench/proxy/usage.test.ts @@ -0,0 +1,71 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { parseUsage, usageFromSse } from "./usage.js"; + +test("Anthropic wire is exclusive; Usage.inputTokens comes back inclusive", () => { + // Real Anthropic convention: input_tokens does NOT include the cache + // fields. 15 fresh + 80 read + 5 written = 100 inclusive. + assert.deepEqual( + parseUsage({ + usage: { + input_tokens: 15, + output_tokens: 20, + cache_read_input_tokens: 80, + cache_creation_input_tokens: 5, + }, + }), + { + inputTokens: 100, + outputTokens: 20, + cacheReadTokens: 80, + cacheWriteTokens: 5, + }, + ); +}); + +test("Anthropic with no cache fields: input passes through untouched", () => { + assert.deepEqual( + parseUsage({ usage: { input_tokens: 40, output_tokens: 3 } }), + { inputTokens: 40, outputTokens: 3, cacheReadTokens: 0, cacheWriteTokens: 0 }, + ); +}); + +test("reads OpenAI Chat Completions usage, cached_tokens as cache reads", () => { + assert.deepEqual( + parseUsage({ + usage: { + prompt_tokens: 50, + completion_tokens: 10, + prompt_tokens_details: { cached_tokens: 40 }, + }, + }), + { + inputTokens: 50, + outputTokens: 10, + cacheReadTokens: 40, + cacheWriteTokens: 0, + }, + ); +}); + +test("ignores a body with no usage", () => { + assert.equal(parseUsage({ id: "msg_1", content: [] }), undefined); +}); + +test("SSE: message_delta only has output_tokens; input and cache stay from message_start", () => { + const sse = [ + "event: message_start", + 'data: {"type":"message_start","message":{"usage":{"input_tokens":10,"output_tokens":1,"cache_read_input_tokens":4}}}', + "", + "event: message_delta", + 'data: {"type":"message_delta","usage":{"output_tokens":7}}', + "", + ].join("\n"); + // 10 fresh + 4 cache-read = 14 inclusive; the delta overlays only output. + assert.deepEqual(usageFromSse(sse), { + inputTokens: 14, + outputTokens: 7, + cacheReadTokens: 4, + cacheWriteTokens: 0, + }); +}); diff --git a/bench/agent-bench/proxy/usage.ts b/bench/agent-bench/proxy/usage.ts new file mode 100644 index 0000000..e522b3a --- /dev/null +++ b/bench/agent-bench/proxy/usage.ts @@ -0,0 +1,113 @@ +// Parse token usage off the wire. Anthropic Messages and OpenAI Chat +// Completions only — those are the two shapes MiniMax's /anthropic shim and +// a native openai-compatible agent both emit. Spec §6.4: one meter, not four +// self-reports. + +export interface Usage { + /** + * INCLUSIVE of cache reads and writes, whatever the wire said. OpenAI's + * `prompt_tokens` already is; Anthropic's `input_tokens` is exclusive + * (cache_read/cache_creation are separate additive fields), so the + * Anthropic parser adds them in. price.ts subtracts the cache fields back + * out — normalizing here is what makes that subtraction correct for both + * shapes. + */ + inputTokens: number; + outputTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; +} + +function num(v: unknown): number { + return typeof v === "number" && Number.isFinite(v) ? Math.max(0, v) : 0; +} + +function take(usage: Record, wire: string, field: keyof Usage): Partial { + return wire in usage ? { [field]: num(usage[wire]) } : {}; +} + +/** Only fields the payload actually carried — a delta must not zero the rest. */ +function fromAnthropicPartial(usage: Record): Partial { + const partial: Partial = { + ...take(usage, "input_tokens", "inputTokens"), + ...take(usage, "output_tokens", "outputTokens"), + ...take(usage, "cache_read_input_tokens", "cacheReadTokens"), + ...take(usage, "cache_creation_input_tokens", "cacheWriteTokens"), + }; + // Anthropic's input_tokens EXCLUDES cache tokens; Usage.inputTokens is + // inclusive. The cache fields ride the same payload (message_start), so + // normalizing per-payload survives the SSE overlay in usageFromSse. + if (partial.inputTokens !== undefined) { + partial.inputTokens += + (partial.cacheReadTokens ?? 0) + (partial.cacheWriteTokens ?? 0); + } + return partial; +} + +function complete(partial: Partial): Usage { + return { + inputTokens: partial.inputTokens ?? 0, + outputTokens: partial.outputTokens ?? 0, + cacheReadTokens: partial.cacheReadTokens ?? 0, + cacheWriteTokens: partial.cacheWriteTokens ?? 0, + }; +} + +function fromOpenAi(usage: Record): Usage | undefined { + if (usage.prompt_tokens === undefined && usage.completion_tokens === undefined) { + return undefined; + } + const details = usage.prompt_tokens_details; + const cached = + details && typeof details === "object" + ? num((details as Record).cached_tokens) + : 0; + return { + inputTokens: num(usage.prompt_tokens), + outputTokens: num(usage.completion_tokens), + cacheReadTokens: cached, + cacheWriteTokens: 0, + }; +} + +function usageFields(body: unknown): Partial | undefined { + if (!body || typeof body !== "object") return undefined; + const rec = body as Record; + const usage = rec.usage; + if (usage && typeof usage === "object") { + const u = usage as Record; + if ("input_tokens" in u || "output_tokens" in u) return fromAnthropicPartial(u); + const openai = fromOpenAi(u); + if (openai) return openai; + } + const nested = rec.message; + if (nested && typeof nested === "object") return usageFields(nested); + return undefined; +} + +/** Usage object nested on a JSON response (or an SSE `data:` payload). */ +export function parseUsage(body: unknown): Usage | undefined { + const fields = usageFields(body); + if (!fields || Object.keys(fields).length === 0) return undefined; + return complete(fields); +} + +/** Walk an SSE buffer; later events overlay only the fields they carry. */ +export function usageFromSse(text: string): Usage | undefined { + let acc: Partial | undefined; + for (const block of text.split(/\r?\n\r?\n/)) { + const data = block + .split(/\r?\n/) + .filter((l) => l.startsWith("data:")) + .map((l) => l.slice(5).trimStart()) + .join(""); + if (!data || data === "[DONE]") continue; + try { + const parsed = usageFields(JSON.parse(data)); + if (parsed && Object.keys(parsed).length > 0) acc = { ...acc, ...parsed }; + } catch { + // A truncated frame is not a usage event. + } + } + return acc ? complete(acc) : undefined; +} diff --git a/bench/agent-bench/runner/agents.ts b/bench/agent-bench/runner/agents.ts index a5dd1cd..16bce39 100644 --- a/bench/agent-bench/runner/agents.ts +++ b/bench/agent-bench/runner/agents.ts @@ -8,6 +8,11 @@ import { spawn, spawnSync } from "child_process"; import * as fs from "fs"; import * as path from "path"; +import { + dockerArgv, + removeContainer, + type Containerize, +} from "../isolate/docker.js"; import type { AgentSpec } from "./types.js"; const AGENT_DIR = path.join(import.meta.dirname, "..", "agents"); @@ -37,9 +42,14 @@ export function loadAgent(id: string): AgentSpec { * degrades a competitor while flattering us (spec §10.6) — so the version that * produced a number is part of the number. */ -export function agentVersion(spec: AgentSpec): string { - const [cmd, ...args] = spec.versionCmd; - const r = spawnSync(cmd!, args, { encoding: "utf-8", timeout: 30_000 }); +export function agentVersion(spec: AgentSpec, image?: string): string { + // Isolated trials run the IMAGE's copy of the agent, so that is the copy + // whose version belongs in the record — not whatever the host has. + const argv = image + ? ["docker", "run", "--rm", image, ...spec.versionCmd] + : spec.versionCmd; + const [cmd, ...args] = argv; + const r = spawnSync(cmd!, args, { encoding: "utf-8", timeout: 120_000 }); if (r.status !== 0) return "unknown"; return (r.stdout || r.stderr).trim().split("\n")[0]!.slice(0, 80); } @@ -65,7 +75,7 @@ function render(spec: AgentSpec, prompt: string): string[] { * `empty-config/` — the only way found to stop opencode loading the operator's * personal MCP servers (see agents/opencode.json). */ -export function resolveEnv(spec: AgentSpec): NodeJS.ProcessEnv { +export function resolveEnv(spec: AgentSpec, benchDir?: string): NodeJS.ProcessEnv { const env: NodeJS.ProcessEnv = { ...process.env }; for (const [key, raw] of Object.entries(spec.env ?? {})) { if (raw === "") { @@ -73,7 +83,8 @@ export function resolveEnv(spec: AgentSpec): NodeJS.ProcessEnv { continue; } env[key] = raw - .replaceAll("{benchDir}", path.join(AGENT_DIR, "..")) + // In a container, {benchDir} is the ro mount, not the host path. + .replaceAll("{benchDir}", benchDir ?? path.join(AGENT_DIR, "..")) .replace(/\$\{(\w+)\}/g, (_, name: string) => { const value = process.env[name]; if (!value) { @@ -109,9 +120,18 @@ export function runAgent( cwd: string, artifactDir: string, timeoutMs: number, + extraEnv?: NodeJS.ProcessEnv, + containerize?: Containerize, ): Promise { - const argv = render(spec, prompt); - const env = resolveEnv(spec); + // In a container, {benchDir} in the adapter env must resolve to the ro + // mount. The env still carries real values (docker copies them from the + // spawn env for each bare `-e NAME`); only argv stays value-free. + const env = { + ...resolveEnv(spec, containerize ? "/bench" : undefined), + ...extraEnv, + }; + const agentArgv = render(spec, prompt); + const argv = containerize ? dockerArgv(containerize, agentArgv) : agentArgv; const [cmd, ...args] = argv; const out = fs.createWriteStream(path.join(artifactDir, "stdout.log")); const err = fs.createWriteStream(path.join(artifactDir, "stderr.log")); @@ -130,6 +150,8 @@ export function runAgent( const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); + // Killing the docker CLIENT does not stop the container. + if (containerize) removeContainer(containerize.name); }, timeoutMs); const done = (exitCode: number | null) => { diff --git a/bench/agent-bench/runner/bundle.test.ts b/bench/agent-bench/runner/bundle.test.ts new file mode 100644 index 0000000..510c4d9 --- /dev/null +++ b/bench/agent-bench/runner/bundle.test.ts @@ -0,0 +1,39 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import * as crypto from "crypto"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { bundleRun } from "./bundle.js"; + +test("bundleRun tars the run dir and writes a matching, reproducible sha256", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "bundle-")); + const run = path.join(root, "2026-09-06T00-00-00"); + fs.mkdirSync(path.join(run, "django__django-1", "trial-1", "freecode"), { + recursive: true, + }); + fs.writeFileSync(path.join(run, "report.json"), "{}"); + fs.writeFileSync( + path.join(run, "django__django-1", "trial-1", "freecode", "patch.diff"), + "diff\n", + ); + + const a = bundleRun(run); + assert.equal(fs.existsSync(a.tarball), true); + const recomputed = crypto + .createHash("sha256") + .update(fs.readFileSync(a.tarball)) + .digest("hex"); + assert.equal(a.sha256, recomputed); + assert.equal( + fs.readFileSync(`${a.tarball}.sha256`, "utf-8").startsWith(a.sha256), + true, + ); + + // Same content, same bytes — the published checksum must be reproducible. + const b = bundleRun(run); + assert.equal(b.sha256, a.sha256); + + assert.throws(() => bundleRun(root), /no report\.json/); + fs.rmSync(root, { recursive: true, force: true }); +}); diff --git a/bench/agent-bench/runner/bundle.ts b/bench/agent-bench/runner/bundle.ts new file mode 100644 index 0000000..9c0809b --- /dev/null +++ b/bench/agent-bench/runner/bundle.ts @@ -0,0 +1,63 @@ +// ============================================================================= +// The evidence bundle (spec §8/§10: published, not managed). One tar.gz per +// run — report.json, and per trial the prompt, exact argv, patch, agent +// stdout/stderr, proxy log, folded usage and isolation audit, plus any +// grading output. The proxy log carries paths and token counts only (no +// headers, no bodies), so the bundle is publishable as-is. +// +// pnpm bench:bundle bench/agent-bench/results/ +// +// Writes -evidence.tar.gz + .sha256 next to the run dir. The checksum +// is the claim: anyone holding the tarball can verify it is the one the +// numbers came from. +// ============================================================================= + +import { spawnSync } from "child_process"; +import * as crypto from "crypto"; +import * as fs from "fs"; +import * as path from "path"; + +export interface Bundle { + tarball: string; + sha256: string; +} + +export function bundleRun(resultsDir: string): Bundle { + const dir = path.resolve(resultsDir); + if (!fs.existsSync(path.join(dir, "report.json"))) { + throw new Error(`${dir} has no report.json — not a finished run`); + } + const name = path.basename(dir); + const tarball = path.join(path.dirname(dir), `${name}-evidence.tar.gz`); + // --sort=name + owner/mtime pinning: the same evidence tars to the same + // bytes, so the published sha256 is reproducible, not an accident of umask. + const r = spawnSync( + "tar", + [ + "--sort=name", "--owner=0", "--group=0", "--numeric-owner", + "--mtime=@0", + "-czf", tarball, + "-C", path.dirname(dir), + name, + ], + { stdio: "inherit" }, + ); + if (r.status !== 0) throw new Error(`tar exited ${r.status}`); + + const sha256 = crypto + .createHash("sha256") + .update(fs.readFileSync(tarball)) + .digest("hex"); + fs.writeFileSync(`${tarball}.sha256`, `${sha256} ${path.basename(tarball)}\n`); + return { tarball, sha256 }; +} + +if (process.argv[1] && import.meta.url.endsWith(path.basename(process.argv[1]))) { + const dir = process.argv[2]; + if (!dir) { + console.error("usage: bundle.ts "); + process.exit(1); + } + const b = bundleRun(dir); + console.log(`${b.tarball}\nsha256 ${b.sha256}`); +} diff --git a/bench/agent-bench/runner/grade.test.ts b/bench/agent-bench/runner/grade.test.ts new file mode 100644 index 0000000..35102f2 --- /dev/null +++ b/bench/agent-bench/runner/grade.test.ts @@ -0,0 +1,88 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { applyVerdicts, parseHarnessReport, predictionsFor } from "./grade.js"; +import type { Report, TrialRecord } from "./types.js"; + +function trial(over: Partial): TrialRecord { + return { + agent: "freecode", + agentVersion: "0.30.0", + model: "minimax/MiniMax-M3", + autonomy: "danger", + instanceId: "django__django-1", + trial: 1, + isolation: "none", + producedPatch: true, + reason: "ok", + exitCode: 0, + timedOut: false, + durationMs: 1, + patchBytes: 10, + newFiles: [], + artifactDir: "results/run/django__django-1/trial-1/freecode", + ...over, + }; +} + +function report(trials: TrialRecord[]): Report { + return { startedAt: "run", finishedAt: "run", isolation: "none", graded: false, trials }; +} + +test("predictionsFor reads each trial's patch.diff and skips empty patches", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "grade-")); + const bench = path.join(root, "bench", "agent-bench"); + const resultsDir = path.join(bench, "results", "run"); + const artDir = path.join(resultsDir, "django__django-1", "trial-1", "freecode"); + fs.mkdirSync(artDir, { recursive: true }); + fs.writeFileSync(path.join(artDir, "patch.diff"), "diff --git a b\n"); + const rep = report([ + trial({}), + trial({ instanceId: "django__django-2", producedPatch: false, artifactDir: "nowhere" }), + trial({ agent: "opencode", artifactDir: "nowhere" }), + ]); + const preds = predictionsFor(rep, resultsDir, "freecode", 1); + assert.deepEqual(preds, [ + { + instance_id: "django__django-1", + model_name_or_path: "freecode", + model_patch: "diff --git a b\n", + }, + ]); + fs.rmSync(root, { recursive: true, force: true }); +}); + +test("parseHarnessReport: resolved, unresolved and error ids are graded; absent is not", () => { + const v = parseHarnessReport({ + resolved_ids: ["a"], + unresolved_ids: ["b"], + error_ids: ["c"], + }); + assert.deepEqual([...v.resolved], ["a"]); + assert.deepEqual([...v.graded].sort(), ["a", "b", "c"]); + const empty = parseHarnessReport({ something: 1 }); + assert.equal(empty.resolved.size, 0); +}); + +test("applyVerdicts: empty patch fails without the harness; unlisted stays null", () => { + const rep = report([ + trial({ instanceId: "a" }), + trial({ instanceId: "b" }), + trial({ instanceId: "c", producedPatch: false }), + trial({ instanceId: "d" }), + trial({ instanceId: "a", agent: "opencode" }), + ]); + applyVerdicts(rep, "freecode", 1, { + resolved: new Set(["a"]), + graded: new Set(["a", "b"]), + }); + const by = (id: string, agent = "freecode") => + rep.trials.find((t) => t.instanceId === id && t.agent === agent)!; + assert.equal(by("a").resolved, true); + assert.equal(by("b").resolved, false); + assert.equal(by("c").resolved, false); // empty patch: no harness needed + assert.equal(by("d").resolved, null); // harness errored / never ran + assert.equal(by("a", "opencode").resolved, undefined); // other slice untouched +}); diff --git a/bench/agent-bench/runner/grade.ts b/bench/agent-bench/runner/grade.ts new file mode 100644 index 0000000..1e0769f Binary files /dev/null and b/bench/agent-bench/runner/grade.ts differ diff --git a/bench/agent-bench/runner/publish.ts b/bench/agent-bench/runner/publish.ts index 5a2977b..fa4dbbc 100644 --- a/bench/agent-bench/runner/publish.ts +++ b/bench/agent-bench/runner/publish.ts @@ -65,6 +65,13 @@ export interface PublishedResult { reason: string; /** Which run this row came from. See the `runs` note below. */ runId: string; + turns?: number; + inputTokens?: number; + outputTokens?: number; + cacheReadTokens?: number; + cacheWriteTokens?: number; + usd?: number | null; + auditOk?: boolean; } export interface PublishedRun { @@ -160,17 +167,24 @@ export function publish(report: Report, fresh = false): string { instanceId: t.instanceId, trial: t.trial, producedPatch: t.producedPatch, - // Only the grader's own verdict may ever fill this in (Phase 1). - // producedPatch is not a substitute: wiring it here would let a flipped - // `graded` flag relabel "changed a file" as "fixed the bug". Until the - // grader writes a real verdict per trial, a graded run with null verdicts - // fails closed on the page — every bar reads 0%, not 100%. - resolved: null, + // Only the grader's own verdict may ever fill this in — grade.ts is the + // sole writer of `TrialRecord.resolved`. producedPatch is not a + // substitute: wiring it here would let a flipped `graded` flag relabel + // "changed a file" as "fixed the bug". An ungraded or harness-errored + // trial stays null and fails closed on the page. + resolved: t.resolved ?? null, durationMs: t.durationMs, patchBytes: t.patchBytes, newFiles: t.newFiles.length, reason: t.reason, runId: report.startedAt, + turns: t.turns, + inputTokens: t.inputTokens, + outputTokens: t.outputTokens, + cacheReadTokens: t.cacheReadTokens, + cacheWriteTokens: t.cacheWriteTokens, + usd: t.usd, + auditOk: t.auditOk, }); } diff --git a/bench/agent-bench/runner/run.ts b/bench/agent-bench/runner/run.ts index 1f518fd..289da8d 100644 --- a/bench/agent-bench/runner/run.ts +++ b/bench/agent-bench/runner/run.ts @@ -2,10 +2,12 @@ // ============================================================================= // The trial loop. agents × instances × trials, one workspace each. // -// Phase 0: no container, no grader, no metering. It answers exactly one -// question — does every adapter produce a non-empty patch — and every record it -// writes carries `isolation: "none"` so it can never be mistaken for a -// publishable number. +// Metering (spec §6.4) is on by default: a pass-through proxy so every agent +// is billed by one table; `--no-meter` restores the adapter-only loop. +// `--isolate` (spec §6.3) runs each trial in a container on an --internal +// docker network whose only exit is that proxy — build the image first with +// `docker build -t agent-bench bench/agent-bench/isolate`. Grading is a +// separate, free-to-rerun step: `pnpm bench:grade results/`. // ============================================================================= import * as fs from "fs"; @@ -15,6 +17,17 @@ import { loadInstances, readIdList } from "./instances.js"; import { publish } from "./publish.js"; import { taskPrompt } from "./prompt.js"; import { createWorkspace, extractPatch, verifyWorkspace } from "./workspace.js"; +import { + IMAGE, + NETWORK, + dockerAvailable, + ensureInternalNetwork, + forwardedEnvNames, + imageExists, +} from "../isolate/docker.js"; +import { meterEnv, upstreamFor } from "../proxy/env.js"; +import { persistTrialMeter } from "../proxy/fold.js"; +import { startProxy } from "../proxy/server.js"; import type { Report, TrialRecord } from "./types.js"; const ROOT = path.join(import.meta.dirname, ".."); @@ -32,10 +45,31 @@ const instanceIds = arg("instances") : readIdList(path.join(ROOT, "instances", "django-lite.txt")); const runId = new Date().toISOString().replace(/[:.]/g, "-"); const outDir = arg("out", path.join(ROOT, "results", runId)) as string; +const meter = !process.argv.includes("--no-meter"); +const isolate = process.argv.includes("--isolate"); async function main() { + // Isolation implies metering: without the proxy the container has no route + // to the model at all, and an unmeterable run in a locked box is just a + // slower way to produce nothing. + let gateway: string | undefined; + if (isolate) { + if (!dockerAvailable()) { + throw new Error("--isolate needs docker (daemon up, user in the docker group)"); + } + if (!imageExists(IMAGE)) { + throw new Error( + `--isolate needs the "${IMAGE}" image. Build it (online) first:\n` + + ` docker build -t ${IMAGE} bench/agent-bench/isolate`, + ); + } + gateway = ensureInternalNetwork(NETWORK); + } + const agents = agentIds.map(loadAgent); - const versions = new Map(agents.map((a) => [a.id, agentVersion(a)])); + const versions = new Map( + agents.map((a) => [a.id, agentVersion(a, isolate ? IMAGE : undefined)]), + ); const instances = await loadInstances(instanceIds); fs.mkdirSync(outDir, { recursive: true }); @@ -45,7 +79,8 @@ async function main() { console.log(` ${"".padEnd(12)} autonomy: ${a.autonomy}`); } console.log( - ` ${instances.length} instance(s) × ${trials} trial(s) × ${agents.length} agent(s)\n`, + ` ${instances.length} instance(s) × ${trials} trial(s) × ${agents.length} agent(s)` + + ` meter=${meter || isolate ? "proxy" : "off"} isolation=${isolate ? "container" : "none"}\n`, ); const records: TrialRecord[] = []; @@ -65,6 +100,7 @@ async function main() { const ws = createWorkspace(inst); let record: TrialRecord; + let proxy: Awaited> | undefined; try { if (!verifyWorkspace(ws.dir, inst.baseCommit)) { throw new Error(`checkout is not at ${inst.baseCommit}`); @@ -72,12 +108,41 @@ async function main() { const prompt = taskPrompt(inst); fs.writeFileSync(path.join(artifactDir, "prompt.txt"), prompt); + const logPath = path.join(artifactDir, "proxy.jsonl"); + proxy = + meter || isolate + ? await startProxy({ + upstream: upstreamFor(spec), + logPath, + host: gateway, + }) + : undefined; + + const extraEnv = proxy ? meterEnv(proxy.origin) : undefined; + const containerize = isolate + ? { + image: IMAGE, + network: NETWORK, + name: `bench-${runId}-${inst.instanceId}-t${trial}-${spec.id}` + .toLowerCase() + .replace(/[^a-z0-9_.-]/g, "-") + .slice(0, 63), + wsDir: ws.dir, + benchDir: ROOT, + envNames: forwardedEnvNames(spec.env, extraEnv), + uid: process.getuid?.() ?? 1000, + gid: process.getgid?.() ?? 1000, + } + : undefined; + const run = await runAgent( spec, prompt, ws.dir, artifactDir, timeoutMs, + extraEnv, + containerize, ); fs.writeFileSync( path.join(artifactDir, "argv.json"), @@ -86,6 +151,7 @@ async function main() { const patch = extractPatch(ws.dir); fs.writeFileSync(path.join(artifactDir, "patch.diff"), patch.diff); + const usage = proxy ? persistTrialMeter(artifactDir, spec.model) : undefined; record = { agent: spec.id, @@ -94,7 +160,7 @@ async function main() { autonomy: spec.autonomy, instanceId: inst.instanceId, trial, - isolation: "none", + isolation: isolate ? "container" : "none", producedPatch: patch.diff.length > 0, reason: run.timedOut ? `timed out after ${timeoutMs}ms` @@ -107,6 +173,13 @@ async function main() { patchBytes: Buffer.byteLength(patch.diff), newFiles: patch.newFiles, artifactDir: path.relative(ROOT, artifactDir), + turns: usage?.turns, + inputTokens: usage?.inputTokens, + outputTokens: usage?.outputTokens, + cacheReadTokens: usage?.cacheReadTokens, + cacheWriteTokens: usage?.cacheWriteTokens, + usd: usage ? (usage.usd ?? null) : undefined, + auditOk: usage?.auditOk, }; } catch (err) { // One dead trial must not cost the rest of the matrix. @@ -117,7 +190,7 @@ async function main() { autonomy: spec.autonomy, instanceId: inst.instanceId, trial, - isolation: "none", + isolation: isolate ? "container" : "none", producedPatch: false, reason: `harness error: ${(err as Error).message}`.slice(0, 200), exitCode: null, @@ -128,14 +201,19 @@ async function main() { artifactDir: path.relative(ROOT, artifactDir), }; } finally { + await proxy?.close(); ws.cleanup(); } records.push(record); + const tokens = + record.inputTokens !== undefined + ? ` ${(record.inputTokens + (record.outputTokens ?? 0)).toLocaleString()}tok` + : ""; console.log( `${record.producedPatch ? "patch" : "EMPTY"} ` + `${String(record.patchBytes).padStart(6)}B ` + - `${(record.durationMs / 1000).toFixed(0)}s ${record.reason}`, + `${(record.durationMs / 1000).toFixed(0)}s${tokens} ${record.reason}`, ); } } @@ -144,7 +222,7 @@ async function main() { const report: Report = { startedAt: runId, finishedAt: new Date().toISOString(), - isolation: "none", + isolation: isolate ? "container" : "none", graded: false, trials: records, }; diff --git a/bench/agent-bench/runner/types.ts b/bench/agent-bench/runner/types.ts index 57a562f..d8935c4 100644 --- a/bench/agent-bench/runner/types.ts +++ b/bench/agent-bench/runner/types.ts @@ -60,7 +60,25 @@ export interface TrialRecord { patchBytes: number; /** Files the agent created that are not part of a fix — scratch-file noise. */ newFiles: string[]; + /** Per-trial artifact dump, relative to bench/agent-bench/. */ artifactDir: string; + /** Round-trips seen by the recording proxy. Absent when `--no-meter`. */ + turns?: number; + inputTokens?: number; + outputTokens?: number; + cacheReadTokens?: number; + cacheWriteTokens?: number; + /** MiniMax-M3 USD off the proxy; null when the model is unpriced. */ + usd?: number | null; + /** False when the proxy saw a request that was not the model endpoint. */ + auditOk?: boolean; + /** + * The official SWE-bench grader's verdict for this trial's patch. Written + * only by `grade.ts` — never inferred from producedPatch. `null` after a + * grading run means the harness errored on this instance; absent means the + * run was never graded. + */ + resolved?: boolean | null; } export interface Report { diff --git a/bench/agent-bench/tsconfig.json b/bench/agent-bench/tsconfig.json new file mode 100644 index 0000000..852455f --- /dev/null +++ b/bench/agent-bench/tsconfig.json @@ -0,0 +1,18 @@ +{ + // tsx executes without typechecking, which is how a field dropped from + // TrialRecord while run.ts still assigned it went unnoticed. `pnpm + // test:agent-bench` runs `tsc --noEmit` over this config first. + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"], + // agent-bench is outside the pnpm workspace on purpose; borrow the + // @types/node that core already pins rather than growing a package.json. + "typeRoots": ["../../apps/core/node_modules/@types"] + }, + "include": ["runner/**/*.ts", "proxy/**/*.ts", "isolate/**/*.ts"] +} diff --git a/package.json b/package.json index 2808f19..cde66fa 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,11 @@ "release": "pnpm build && pnpm publish -r --access public", "gen:catalogue": "node scripts/gen-provider-catalogue.mjs", "gen:catalogue:check": "node scripts/gen-provider-catalogue.mjs --check", - "bench:agents": "tsx bench/agent-bench/runner/run.ts" + "bench:agents": "tsx bench/agent-bench/runner/run.ts", + "bench:grade": "tsx bench/agent-bench/runner/grade.ts", + "bench:image": "docker build -t agent-bench bench/agent-bench/isolate", + "bench:bundle": "tsx bench/agent-bench/runner/bundle.ts", + "test:agent-bench": "tsc -p bench/agent-bench/tsconfig.json && tsx --test bench/agent-bench/**/*.test.ts" }, "devDependencies": { "@changesets/cli": "^2.31.0",