From b8287d82af4c6cbcc37f8970f606a026e96cc16b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:04:55 +0000 Subject: [PATCH 1/5] Initial plan From 856f0b0be8e574ae5aa03e911364f9a5e5b83a6d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:19:22 +0000 Subject: [PATCH 2/5] feat: add frozen adaptive review experiment Co-authored-by: JRichlen <9574264+JRichlen@users.noreply.github.com> --- .gitignore | 2 + docs/testing.md | 22 + evals/cheap/run.sh | 13 + evals/paid/adaptive-review/README.md | 107 ++++ .../compiled/baseline.image.json | 1 + .../paid/adaptive-review/compiled/baseline.md | 11 + .../compiled/candidate.image.json | 1 + .../adaptive-review/compiled/candidate.md | 28 + evals/paid/adaptive-review/corpus.json | 135 ++++ evals/paid/adaptive-review/corpus.sha256 | 1 + evals/paid/adaptive-review/evaluate.py | 600 ++++++++++++++++++ evals/paid/adaptive-review/experiment.json | 62 ++ .../paid/adaptive-review/prompt.template.txt | 11 + .../adaptive-review/queries/baseline.json | 9 + .../adaptive-review/queries/candidate.json | 9 + .../registry/behaviors/concise-review.md | 39 ++ .../registry/views/baseline.md | 11 + .../registry/views/concise-review.md | 12 + evals/paid/adaptive-review/self-test.sh | 29 + evals/paid/adaptive-review/status.json | 8 + 20 files changed, 1111 insertions(+) create mode 100644 evals/paid/adaptive-review/README.md create mode 100644 evals/paid/adaptive-review/compiled/baseline.image.json create mode 100644 evals/paid/adaptive-review/compiled/baseline.md create mode 100644 evals/paid/adaptive-review/compiled/candidate.image.json create mode 100644 evals/paid/adaptive-review/compiled/candidate.md create mode 100644 evals/paid/adaptive-review/corpus.json create mode 100644 evals/paid/adaptive-review/corpus.sha256 create mode 100644 evals/paid/adaptive-review/evaluate.py create mode 100644 evals/paid/adaptive-review/experiment.json create mode 100644 evals/paid/adaptive-review/prompt.template.txt create mode 100644 evals/paid/adaptive-review/queries/baseline.json create mode 100644 evals/paid/adaptive-review/queries/candidate.json create mode 100644 evals/paid/adaptive-review/registry/behaviors/concise-review.md create mode 100644 evals/paid/adaptive-review/registry/views/baseline.md create mode 100644 evals/paid/adaptive-review/registry/views/concise-review.md create mode 100644 evals/paid/adaptive-review/self-test.sh create mode 100644 evals/paid/adaptive-review/status.json diff --git a/.gitignore b/.gitignore index 3910d572..7d57fdbe 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,5 @@ plugins/*/evals/promptfoo/results.matrix.json plugins/*/evals/promptfoo/promptfooconfig.regrade.*.yaml plugins/*/evals/promptfoo/replay.*.json plugins/*/evals/promptfoo/results.regrade.*.json +# Adaptive-review raw outputs, human labels, and judge evidence stay private. +/evals/paid/adaptive-review/runs/ diff --git a/docs/testing.md b/docs/testing.md index 0d2321f5..e7420e8b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -33,6 +33,7 @@ structurally cannot. | [subject-model matrix](#subject-model-matrix-manual-advisory) | `subject-matrix.yml` + `evals/paid/subject-matrix.sh` | (1 + subjects) × the pack's usual cents | manual dispatch only | no — advisory; the baseline subject decides the job, extra subjects never do | | [calibration sheet](#calibration-sheet-manual-no-model-calls) | `calibration-sheet.yml` + `evals/paid/calibration/` | free (no model calls) | manual dispatch only | no — writes a blind sheet to a `calibration/` branch for a human to label | | [grader agreement](#grader-agreement-manual-grading-only) | `grader-agreement.yml` + `evals/paid/calibration/regrade.sh` | grading spend only (no subject calls) | manual dispatch only | no — reports agreement and kappa between graders on a finished run's outputs | +| [adaptive-review experiment](#adaptive-review-experiment-offline-contract) | `evals/paid/adaptive-review/` | offline contract is free; actor/judge runs are separately authorized | contract self-test on every cheap run; inference is manual only | no — reports improvement, regression, or inconclusive; never installs | | [scale](#scale-tier) | `plugins/{redgate,agent-compiler}/evals/scale/` | free, offline, minutes | path-gated (`plugins/redgate/**`, `plugins/agent-compiler/**`) | no — evidence, not a merge gate | | [deep](#deep-tier-pier) | `plugins/

/evals/pier/` | dollars + minutes (sandboxed agents) | path-gated to the safety surface (`plugins/*/skills/**/scripts/**`, `plugins/*/evals/pier/**`) | yes — `deep tier (pier)` (aggregate) | | [example gallery](#example-gallery-refresh--pages) | `refresh-examples.yml` / `pages.yml` | real API budget per refresh | scheduled (1st + 15th, 06:00 UTC) / on `docs/**` push to main | no — review-gated PR / publish | @@ -317,6 +318,27 @@ that it is green because it did not run, never silently. `evals/paid/calibration/regrade.sh --self-test`; the cheap tier §18b fixture-tests the overlay and the replay provider. +## adaptive-review experiment (offline contract) + +- **What it proves.** `evals/paid/adaptive-review/self-test.sh` proves that the +frozen public corpus and both `agent-compiler` images have not drifted, and +that the experiment-specific scorer distinguishes improvement, regression, +and inconclusive fixtures. It also fails closed when external judge inputs +lack a separate authorization bound to their exact digest. +- **What it cannot prove.** The offline fixtures are scorer tests, not evidence +that concise review helps a person. `status.json` therefore remains +inconclusive until the pinned local actor, blind human labels, and independent +frontier judge all produce retained evidence. Detailed run artifacts stay in +the git-ignored private `runs/` directory. +- **Fires.** The contract self-test runs inside every cheap-tier invocation. +Actor and judge inference are manual only; there is no workflow, schedule, +fallback route, automatic installation, or new required check. +- **Cost.** Free for the committed self-test. Any external grading needs its own +input-bound authorization and positive spending limit. +- **Local run.** `evals/paid/adaptive-review/self-test.sh`; the experiment +procedure and evidence schemas are in +`evals/paid/adaptive-review/README.md`. + ## scale tier - **What it proves.** The same invariants the cheap tier proves once, held diff --git a/evals/cheap/run.sh b/evals/cheap/run.sh index e69fdae3..bd5907d1 100755 --- a/evals/cheap/run.sh +++ b/evals/cheap/run.sh @@ -454,6 +454,19 @@ if [ -f "evals/paid/discover-paid-packs.sh" ]; then fi fi +# --- 11c. Adaptive-review experiment contract ------------------------------- +# The experiment never calls a model here. This recompiles both treatment +# images, verifies the frozen corpus, and exercises the evidence scorer's +# improvement/regression/inconclusive and authorization branches. +if [ -f "evals/paid/adaptive-review/self-test.sh" ]; then + group "adaptive-review experiment self-test" + if bash evals/paid/adaptive-review/self-test.sh >/dev/null 2>&1; then + ok "adaptive-review frozen experiment and scorer self-test" + else + bad "adaptive-review experiment drifted or its scorer no longer fails closed" + fi +fi + # --- 12. Install-smoke coverage (every registered plugin) ------------------- # The install smoke test (ci/install-smoke.sh) proves ONE plugin installs # structurally. In CI it is fanned out over a matrix enumerated from diff --git a/evals/paid/adaptive-review/README.md b/evals/paid/adaptive-review/README.md new file mode 100644 index 00000000..eb6e3e1f --- /dev/null +++ b/evals/paid/adaptive-review/README.md @@ -0,0 +1,107 @@ +# Adaptive review v1 experiment + +This is the smallest benefit experiment for +[#117](https://github.com/JRichlen/agent-plugins/issues/117). It is **not an +installed plugin, scheduled job, model driver, or new general eval harness**. +Runtime expansion remains paused. `status.json` stays `inconclusive` until a +qualified run supplies complete evidence. + +## What is frozen + +- `corpus.json`: two development and six held-out excerpts from public project + issues and pull requests. `corpus.sha256` detects any post-registration edit. +- `queries/`: baseline and one candidate AgentQuery. +- `registry/`: one candidate behavior plus the two views. The committed + `compiled/` images and agents are generated by the existing + `agent-compiler`; `self-test.sh` recompiles and compares them byte for byte. +- `experiment.json`: the feedback source, scoped inspectable preference, + qualified actor route, thresholds, approval boundary, monitoring rule, and + rollback. + +The recurrence record is intentionally honest: #117 explicitly selected this +experiment, but no three cited sightings exist, so it makes no +`recurrence-detector` threshold claim. + +## Preregistered decision + +Only the six held-out pairs count. **Improvement** requires all of: + +- at least 4 candidate wins and at most 1 baseline win under blind human review; +- mean comprehension lift at least `+0.75` on a 1–5 scale; +- median reduction of at least 1 substantive edit before posting; +- zero required facts or actions missing from candidate outputs; +- no truncated/non-stop actor response; +- independent frontier-judge agreement with human acceptability labels of at + least `0.80` and Cohen's kappa of at least `0.60`. + +A lost candidate requirement, more baseline than candidate wins, or negative +comprehension/edit delta is a **regression**. Everything else is +**inconclusive**. Even improvement produces only +`eligible-for-scoped-human-approval`; the scorer never installs anything. + +## Run contract + +1. Run both committed agents against the identical rendered prompt for every + corpus row. The actor must be a pinned local Qwen 27B, direct 32K context, + one sequence, with no external fallback. Use the approved + `agent-request/v1` capability when it becomes available; do not duplicate + its transport or authority semantics here. +2. Retain a private `run.json` with schema `adaptive-review-run/v1`: exact git, + corpus, prompt, and image hashes; actor model digest; every raw output; + finish reason; and numeric input tokens, output tokens, and cost for every + call. Generated run directories are git-ignored because detailed lab + evidence stays private. +3. Create the human sheet: + + ```sh + python3 evaluate.py blind --run runs//run.json \ + --out runs//human-labels.json + ``` + + The sheet contains output hashes and texts but no arm names. For each output, + label comprehension (1–5), minimum substantive edits before posting, + acceptability, and missing required IDs; then choose a winner or tie. +4. Give the exact same output set and task requirements to an independent + frontier judge. If grading is external, its record must bind a separate + authorization to the exact input digest and a positive spending limit. + Preserve its labels and total usage as + `adaptive-review-judge-labels/v1`. +5. Evaluate without rerunning either model: + + ```sh + python3 evaluate.py evaluate \ + --run runs//run.json \ + --human runs//human-labels.json \ + --judge runs//judge-labels.json \ + --out runs//report.json + ``` + +The evaluator rejects contaminated hashes, missing pairs, unknown requirement +IDs, unqualified actors, same-family judges, unbound external inputs, unknown +usage, and overspend. Missing labels or judge evidence stays inconclusive. + +## Lifecycle and rollback + +The corpus captures the goal (**understand**); the compiled view is the sole +treatment (**select**); the qualified actor produces retained outputs +(**act**); blind labels record edits and comprehension (**collect feedback**); +the scorer compares paired arms (**compare**); only an explicit human may +approve a review-only install (**approve or reject**). Any approved install +must record the baseline image hash, repeat this held-out check after material +feedback or model changes (**monitor**), and restore that baseline/remove the +candidate on regression (**roll back**). + +The preference is visible in `experiment.json`, limited to issue review, +corrected by versioning this experiment, and deleted by removing the candidate. +There is no hidden profile, schedule, remote memory, learning/coding transfer, +or automatic curation. + +## Offline verification + +```sh +evals/paid/adaptive-review/self-test.sh +``` + +This recompiles both arms, verifies the corpus checksum, exercises improvement, +regression, inconclusive, and missing-authorization fixtures, and confirms the +checked-in status cannot imply an unrun benefit. diff --git a/evals/paid/adaptive-review/compiled/baseline.image.json b/evals/paid/adaptive-review/compiled/baseline.image.json new file mode 100644 index 00000000..cba2d736 --- /dev/null +++ b/evals/paid/adaptive-review/compiled/baseline.image.json @@ -0,0 +1 @@ +{"behavior":[],"capabilities":[],"compilerVersion":"0.1.0","diagnostics":[],"effectCeiling":["filesystem:read"],"effects":[],"hash":"sha256:ef278a1088b6f09cb60582b8fb3b9b565c4900ed9e17fb55b84e572a80bfe0ec","query":{"effectCeiling":["filesystem:read"],"name":"issue-review-baseline","role":"reviewer","stance":["goal-aligned"],"task":"issue-review","views":["view.issue-review-baseline"]},"registryRevision":"sha256:7079b3cc6b6ec21b2e1dc92a339dd500efb422ff250b7aff634be3e7c732bc8b","schemaVersion":"0.1","views":["view.issue-review-baseline"]} diff --git a/evals/paid/adaptive-review/compiled/baseline.md b/evals/paid/adaptive-review/compiled/baseline.md new file mode 100644 index 00000000..4713146c --- /dev/null +++ b/evals/paid/adaptive-review/compiled/baseline.md @@ -0,0 +1,11 @@ +--- +name: issue-review-baseline +description: Compiled reviewer agent for issue-review. Effects capped at: filesystem:read. +--- + + + +# issue-review-baseline + +Stance: goal-aligned. + diff --git a/evals/paid/adaptive-review/compiled/candidate.image.json b/evals/paid/adaptive-review/compiled/candidate.image.json new file mode 100644 index 00000000..d3793424 --- /dev/null +++ b/evals/paid/adaptive-review/compiled/candidate.image.json @@ -0,0 +1 @@ +{"behavior":[{"content":"Replace chronology and repeated rationale with the shortest causal context needed to understand why each retained fact or action matters.","id":"behavior.issue-review-concise#compress-context","kind":"rule","provenance":{"lines":[21,23],"module":"behavior.issue-review-concise","source":"behaviors/concise-review.md","version":"1.0.0"},"strength":"must"},{"content":"Would deleting this sentence change a decision, constraint, risk, or action? If not, remove it.","id":"behavior.issue-review-concise#deletion-test","kind":"probe","provenance":{"lines":[29,31],"module":"behavior.issue-review-concise","source":"behaviors/concise-review.md","version":"1.0.0"}},{"content":"Lead with the decision, requested outcome, or next action the issue exists to support.","id":"behavior.issue-review-concise#lead-with-goal","kind":"rule","provenance":{"lines":[9,11],"module":"behavior.issue-review-concise","source":"behaviors/concise-review.md","version":"1.0.0"},"strength":"must"},{"content":"Separate decisions, actions, gates, and deferred work so a maintainer can scan what happens next without rereading the background.","id":"behavior.issue-review-concise#make-actions-scannable","kind":"rule","provenance":{"lines":[25,27],"module":"behavior.issue-review-concise","source":"behaviors/concise-review.md","version":"1.0.0"},"strength":"must"},{"content":"Can every required fact and action in the source be located in the revision without inference?","id":"behavior.issue-review-concise#preservation-test","kind":"probe","provenance":{"lines":[33,35],"module":"behavior.issue-review-concise","source":"behaviors/concise-review.md","version":"1.0.0"}},{"content":"Preserve every requirement, constraint, unresolved risk, and explicit action that can change the decision or its execution.","id":"behavior.issue-review-concise#preserve-requirements","kind":"rule","provenance":{"lines":[13,15],"module":"behavior.issue-review-concise","source":"behaviors/concise-review.md","version":"1.0.0"},"strength":"must"},{"content":"Use only domain lenses that affect the stated goal; omit generic review categories that produce no finding.","id":"behavior.issue-review-concise#select-lenses","kind":"rule","provenance":{"lines":[17,19],"module":"behavior.issue-review-concise","source":"behaviors/concise-review.md","version":"1.0.0"},"strength":"must"},{"content":"Never call an issue concise by dropping a required fact, action, limitation, or approval boundary.","id":"behavior.issue-review-concise#shorter-by-omission","kind":"antipattern","provenance":{"lines":[37,39],"module":"behavior.issue-review-concise","source":"behaviors/concise-review.md","version":"1.0.0"}}],"capabilities":[],"compilerVersion":"0.1.0","diagnostics":[],"effectCeiling":["filesystem:read"],"effects":[],"hash":"sha256:50140878d2d6d271604ae745ff607c8e9bc710fe2ba376d7cef034016e2680f6","query":{"effectCeiling":["filesystem:read"],"name":"issue-review-concise","role":"reviewer","stance":["goal-aligned"],"task":"issue-review","views":["view.issue-review-concise"]},"registryRevision":"sha256:7079b3cc6b6ec21b2e1dc92a339dd500efb422ff250b7aff634be3e7c732bc8b","schemaVersion":"0.1","views":["view.issue-review-concise"]} diff --git a/evals/paid/adaptive-review/compiled/candidate.md b/evals/paid/adaptive-review/compiled/candidate.md new file mode 100644 index 00000000..7c6d4164 --- /dev/null +++ b/evals/paid/adaptive-review/compiled/candidate.md @@ -0,0 +1,28 @@ +--- +name: issue-review-concise +description: Compiled reviewer agent for issue-review. Effects capped at: filesystem:read. +--- + + + +# issue-review-concise + +Stance: goal-aligned. + +## Rules + +- **MUST** Replace chronology and repeated rationale with the shortest causal context needed to understand why each retained fact or action matters. +- **MUST** Lead with the decision, requested outcome, or next action the issue exists to support. +- **MUST** Separate decisions, actions, gates, and deferred work so a maintainer can scan what happens next without rereading the background. +- **MUST** Preserve every requirement, constraint, unresolved risk, and explicit action that can change the decision or its execution. +- **MUST** Use only domain lenses that affect the stated goal; omit generic review categories that produce no finding. + +## Probes to run + +- Would deleting this sentence change a decision, constraint, risk, or action? If not, remove it. +- Can every required fact and action in the source be located in the revision without inference? + +## Never + +- Never call an issue concise by dropping a required fact, action, limitation, or approval boundary. + diff --git a/evals/paid/adaptive-review/corpus.json b/evals/paid/adaptive-review/corpus.json new file mode 100644 index 00000000..25eee746 --- /dev/null +++ b/evals/paid/adaptive-review/corpus.json @@ -0,0 +1,135 @@ +{ + "schema_version": "adaptive-review-corpus/v1", + "frozen": true, + "tasks": [ + { + "id": "dev-117", + "split": "development", + "source": "https://github.com/JRichlen/agent-plugins/issues/117", + "title": "Adaptive behavior v1", + "input": "Control mechanisms have been exercised; user benefit and judge calibration remain unproven. Runtime expansion is paused. Freeze a small corpus around issues containing too much context. Compare a baseline and one concise-review candidate on held-out tasks, retain outputs, deterministic checks, blinded human labels and total usage, and calibrate an independent frontier judge against human labels. External grading requires separately authorized inputs and spending limits. Report improvement, regression or inconclusive evidence. Only a supported result advances to an approved, scoped install with demonstrated rollback.", + "required_facts": [ + {"id": "runtime-paused", "text": "Runtime expansion is paused."}, + {"id": "judge-authorization", "text": "External grading needs separate input authorization and a spending limit."} + ], + "required_actions": [ + {"id": "compare-held-out", "text": "Compare baseline and candidate on held-out tasks."}, + {"id": "classify-result", "text": "Report improvement, regression, or inconclusive evidence."}, + {"id": "gate-install", "text": "Require approval, scoped installation, and rollback before advancement."} + ] + }, + { + "id": "dev-084", + "split": "development", + "source": "https://github.com/JRichlen/agent-plugins/issues/84", + "title": "Trigger semantics and description compression", + "input": "Skill descriptions currently blur implied triggers, explicit user triggers, dependency calls, state or risk gates, always-on invariants, and offer-only behavior. Descriptions should be compact routing indexes rather than miniature instruction sets. The design must separate routing eligibility, authorization, and workflow ownership; define precedence and overrides; generate harness-specific descriptions from one canonical declaration; and measure routing accuracy and context cost. It must also cover collisions, false positives, false negatives, opt-out, confirmation gates, and migration without inflating descriptions.", + "required_facts": [ + {"id": "separate-concepts", "text": "Routing eligibility, authorization, and workflow ownership are distinct."}, + {"id": "compact-index", "text": "Descriptions should be compact routing indexes."} + ], + "required_actions": [ + {"id": "define-precedence", "text": "Define trigger precedence and overrides."}, + {"id": "measure-routing", "text": "Measure routing quality and context cost."} + ] + }, + { + "id": "holdout-101", + "split": "held-out", + "source": "https://github.com/JRichlen/agent-plugins/issues/101", + "title": "Portable agent request contract", + "input": "Define a small, versioned, portable contract that lets an external gateway identify an agent request, correlate it to durable work, and enforce coarse model and tool policy without ingesting the full prompt or hidden reasoning. Use opaque identifiers and bounded enums. Keep routing eligibility, authorization, workflow ownership, and approval state distinct. Unknown schema versions and malformed fields fail closed. The deliverable includes a schema, validator, transport mapping, capability taxonomy, threat model, counterfeit cases, and representative requests. Deploying Aperture, implementing an always-on dispatcher, granting connector access, and embedding prompt content in telemetry are explicitly out of scope.", + "required_facts": [ + {"id": "no-prompt-metadata", "text": "The contract must not carry prompts or hidden reasoning."}, + {"id": "separate-authority", "text": "Routing and model choice do not grant tool authority."}, + {"id": "non-goals", "text": "Deployment and connector access are out of scope."} + ], + "required_actions": [ + {"id": "deliver-contract", "text": "Produce the versioned schema and validator."}, + {"id": "test-counterfeits", "text": "Cover spoofing, replay, malformed input, and privacy violations."} + ] + }, + { + "id": "holdout-102", + "split": "held-out", + "source": "https://github.com/JRichlen/agent-plugins/issues/102", + "title": "Subject matrix and grader calibration", + "input": "The behavioral tier currently uses one subject model and one grader with three repeats, so its claims are narrow. Add an advisory subject-model matrix, a blind human calibration set keyed to stored output hashes, grader self-consistency and cross-family checks, then real-agent protocol runs. The offline harness can land without keys, but no new layer becomes required in the same pull request. A calibration stub that passes under a new subject is a rubric finding, not a green result. Record findings with run numbers and update testing documentation with any new job or pack.", + "required_facts": [ + {"id": "current-limit", "text": "Current evidence is one subject, one grader, and three repeats."}, + {"id": "advisory-first", "text": "New model legs are advisory first."}, + {"id": "stub-failure", "text": "A passing calibration stub invalidates the rubric for that subject."} + ], + "required_actions": [ + {"id": "blind-human", "text": "Create blind human labels tied to output hashes."}, + {"id": "measure-agreement", "text": "Measure grader agreement and self-consistency."}, + {"id": "update-docs", "text": "Update testing documentation with new jobs or packs."} + ] + }, + { + "id": "holdout-089", + "split": "held-out", + "source": "https://github.com/JRichlen/agent-plugins/issues/89", + "title": "Agentic and composition testing", + "input": "Document every existing evaluation tier before extending the test architecture. Then add decision-point probes, plan audits, trajectory runs with deterministic artifact audit, and release-cadence cross-plugin composition runs. Test decisions rather than scripts, force protocol state onto disk, include a negative on both sides of every boundary, and preserve FAULT versus verdict separation. Any pull request that adds, removes, renames, or re-scopes an eval tier, workflow job, or per-plugin pack must update the testing document in the same pull request, with a cheap drift check enforcing both directions.", + "required_facts": [ + {"id": "document-first", "text": "Document the existing architecture before extending it."}, + {"id": "fault-verdict", "text": "Transport faults remain distinct from behavioral verdicts."}, + {"id": "same-pr-docs", "text": "Testing inventory changes require same-PR documentation."} + ], + "required_actions": [ + {"id": "layer-tests", "text": "Sequence decision probes, plan audits, trajectories, and composition runs."}, + {"id": "negative-controls", "text": "Add negative controls on both sides of each boundary."}, + {"id": "drift-check", "text": "Enforce documentation drift deterministically."} + ] + }, + { + "id": "holdout-085", + "split": "held-out", + "source": "https://github.com/JRichlen/agent-plugins/issues/85", + "title": "Minimal context experiment", + "input": "Compare baseline, taxonomy, recipe-aware, and full-agent-os context on the same six scenarios. Keep total spend below $0.05 with deterministic model resolution and conservative preflight. Preserve raw responses, blind judgments, per-call token and cost ledgers, aggregate and scenario deltas, hard failures, and representative examples. Do not make the experiment required CI and do not merge the design pull request. Prefer the smallest treatment within the useful-signal band without new hard failures.", + "required_facts": [ + {"id": "four-arms", "text": "The experiment compares four named context treatments."}, + {"id": "cost-cap", "text": "The maximum spend is $0.05."}, + {"id": "not-required", "text": "The experiment must not become required CI."} + ], + "required_actions": [ + {"id": "retain-evidence", "text": "Retain raw, blinded, usage, delta, and hard-failure evidence."}, + {"id": "smallest-useful", "text": "Prefer the smallest useful treatment without new hard failures."} + ] + }, + { + "id": "holdout-107", + "split": "held-out", + "source": "https://github.com/JRichlen/agent-plugins/pull/107", + "title": "Offline agent-request core", + "input": "This draft adds an offline agent-request/v1 core with a closed schema, deterministic validation against separately supplied trusted context, exact approval and replay checks, bounded canonical HTTP and OpenAI header mapping, counterfeit fixtures, and safe role examples. The sender rejects unknown or free-form fields before encoding. The receiver rejects duplicate or noncanonical JSON, binds operational fields and dispatcher provenance to trusted context, and returns stable non-sensitive errors. It does not provide a live issuer, signatures, durable replay storage, verified Aperture integration, permissions, gateway deployment, or rollout.", + "required_facts": [ + {"id": "offline-draft", "text": "The implementation is an offline draft."}, + {"id": "trusted-context", "text": "Operational fields are validated against separate trusted context."}, + {"id": "no-live-security", "text": "Live issuance, signatures, durable replay storage, and deployment are absent."} + ], + "required_actions": [ + {"id": "reject-free-form", "text": "Reject unknown and free-form sender fields."}, + {"id": "reject-noncanonical", "text": "Reject duplicate and noncanonical receiver input."} + ] + }, + { + "id": "holdout-115", + "split": "held-out", + "source": "https://github.com/JRichlen/agent-plugins/pull/115", + "title": "Marketplace-wide agentic test framework", + "input": "The pull request proposes an offline agentic test framework, a red-team lane, a 52-entry catalog, a 75-card corpus, native adapters, approval-gated drivers, provenance controls, attempt accounting, clustered statistics, and model-pricing checks. Its own evidence says native and paid behavior remains blocked and corpus tests are framework evidence rather than empirical plugin findings. Review also found required-gate portability failures: one probe assumes installed CLIs, another cheap check assumes a host-local Promptfoo install, and several text reads rely on locale-default encoding.", + "required_facts": [ + {"id": "framework-only", "text": "The corpus proves framework mechanics, not plugin benefit."}, + {"id": "native-paid-blocked", "text": "Native and paid behavior is still blocked."}, + {"id": "portability-findings", "text": "Review found clean-checkout and locale portability problems."} + ], + "required_actions": [ + {"id": "fix-portability", "text": "Resolve portability failures before adopting the framework."}, + {"id": "do-not-claim-benefit", "text": "Do not treat framework tests as behavioral benefit."} + ] + } + ] +} diff --git a/evals/paid/adaptive-review/corpus.sha256 b/evals/paid/adaptive-review/corpus.sha256 new file mode 100644 index 00000000..b2a44ed6 --- /dev/null +++ b/evals/paid/adaptive-review/corpus.sha256 @@ -0,0 +1 @@ +af30d2ac0c43a3610787167379f89cd8944c23a1ee8808c462387696a02d94c4 corpus.json diff --git a/evals/paid/adaptive-review/evaluate.py b/evals/paid/adaptive-review/evaluate.py new file mode 100644 index 00000000..ec995860 --- /dev/null +++ b/evals/paid/adaptive-review/evaluate.py @@ -0,0 +1,600 @@ +#!/usr/bin/env python3 +"""Prepare blind labels and score the frozen adaptive-review experiment.""" + +import argparse +import hashlib +import json +import statistics +import sys +from pathlib import Path + + +HERE = Path(__file__).resolve().parent + + +class EvidenceError(ValueError): + pass + + +def read_json(path): + with open(path, encoding="utf-8") as handle: + return json.load(handle) + + +def write_json(path, value): + with open(path, "w", encoding="utf-8") as handle: + json.dump(value, handle, indent=2, sort_keys=True) + handle.write("\n") + + +def file_sha256(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def output_id(task_id, text): + return hashlib.sha256(f"{task_id}\0{text}".encode("utf-8")).hexdigest() + + +def rendered_prompt(task, template): + return template.replace("{{input}}", task["input"]) + + +def prompt_sha256(task, template): + return hashlib.sha256(rendered_prompt(task, template).encode("utf-8")).hexdigest() + + +def load_contract(): + experiment = read_json(HERE / "experiment.json") + corpus = read_json(HERE / experiment["corpus"]) + template = (HERE / experiment["prompt_template"]).read_text(encoding="utf-8") + tasks = {task["id"]: task for task in corpus["tasks"]} + if len(tasks) != len(corpus["tasks"]): + raise EvidenceError("corpus contains duplicate task ids") + held_out = [task for task in corpus["tasks"] if task["split"] == "held-out"] + if len(held_out) < experiment["thresholds"]["minimum_held_out_pairs"]: + raise EvidenceError("frozen corpus is smaller than minimum_held_out_pairs") + return experiment, corpus, template, tasks, held_out + + +def expected_image_hash(experiment, arm): + image = read_json(HERE / experiment["arms"][arm]["image"]) + return image["hash"] + + +def validate_usage(usage, where): + if not isinstance(usage, dict): + raise EvidenceError(f"{where}: missing usage") + for key in ("input_tokens", "output_tokens"): + value = usage.get(key) + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise EvidenceError(f"{where}: {key} must be a non-negative integer") + cost = usage.get("cost_usd") + if not isinstance(cost, (int, float)) or isinstance(cost, bool) or cost < 0: + raise EvidenceError(f"{where}: cost_usd must be a known non-negative number") + + +def validate_run(run): + experiment, corpus, template, tasks, held_out = load_contract() + if run.get("schema_version") != "adaptive-review-run/v1": + raise EvidenceError("run schema_version must be adaptive-review-run/v1") + if run.get("experiment_id") != experiment["experiment_id"]: + raise EvidenceError("run experiment_id does not match the frozen experiment") + + provenance = run.get("provenance", {}) + if provenance.get("corpus_sha256") != file_sha256(HERE / experiment["corpus"]): + raise EvidenceError("run corpus hash does not match the frozen corpus") + for arm in ("baseline", "candidate"): + key = f"{arm}_image_hash" + if provenance.get(key) != expected_image_hash(experiment, arm): + raise EvidenceError(f"run {key} does not match the compiled image") + commit = provenance.get("git_commit", "") + if len(commit) != 40 or any(c not in "0123456789abcdef" for c in commit): + raise EvidenceError("run git_commit must be a lowercase 40-hex commit") + + actor = run.get("actor", {}) + required = experiment["actor_qualification"] + checks = { + "provider": "local", + "model_family": required["model_family"], + "context_window": required["context_window"], + "sequences": required["sequences"], + "external_fallback": required["external_fallback"], + } + for key, expected in checks.items(): + if actor.get(key) != expected: + raise EvidenceError(f"actor {key} must be {expected!r}") + if required["model_digest_required"] and not actor.get("model_digest"): + raise EvidenceError("actor model_digest is required") + + expected_pairs = {(task_id, arm) for task_id in tasks for arm in ("baseline", "candidate")} + rows = {} + for row in run.get("outputs", []): + pair = (row.get("task_id"), row.get("arm")) + if pair in rows: + raise EvidenceError(f"duplicate output for {pair}") + if pair not in expected_pairs: + raise EvidenceError(f"unexpected task/arm pair {pair}") + text = row.get("output") + if not isinstance(text, str) or not text.strip(): + raise EvidenceError(f"{pair}: output must be non-empty text") + if row.get("output_id") != output_id(pair[0], text): + raise EvidenceError(f"{pair}: output_id does not match task and output") + if row.get("prompt_sha256") != prompt_sha256(tasks[pair[0]], template): + raise EvidenceError(f"{pair}: prompt hash does not match the shared frozen prompt") + if row.get("agent_image_hash") != expected_image_hash(experiment, pair[1]): + raise EvidenceError(f"{pair}: agent image hash does not match its arm") + validate_usage(row.get("usage"), f"{pair}") + rows[pair] = row + missing = sorted(expected_pairs - set(rows)) + if missing: + raise EvidenceError(f"run is missing {len(missing)} task/arm outputs") + return experiment, corpus, tasks, held_out, rows + + +def make_blind_sheet(run): + experiment, _, _, held_out, rows = validate_run(run) + result = { + "schema_version": "adaptive-review-human-labels/v1", + "experiment_id": experiment["experiment_id"], + "instructions": ( + "Label without identifying arms. For each output set comprehension 1-5, " + "edits_needed to the minimum substantive edits before posting, acceptable " + "true/false, and missing_required_ids. Set winner to an output id or tie." + ), + "pairs": [], + } + for task in held_out: + task_id = task["id"] + candidates = [ + { + "id": rows[(task_id, arm)]["output_id"], + "text": rows[(task_id, arm)]["output"], + "comprehension": None, + "edits_needed": None, + "acceptable": None, + "missing_required_ids": [], + } + for arm in ("baseline", "candidate") + ] + if int(hashlib.sha256(f"{experiment['experiment_id']}:{task_id}".encode()).hexdigest(), 16) % 2: + candidates.reverse() + result["pairs"].append({"task_id": task_id, "outputs": candidates, "winner": None}) + return result + + +def validate_human(sheet, experiment, tasks, held_out, rows): + if sheet.get("schema_version") != "adaptive-review-human-labels/v1": + raise EvidenceError("human labels have the wrong schema_version") + if sheet.get("experiment_id") != experiment["experiment_id"]: + raise EvidenceError("human labels refer to another experiment") + pairs = {pair.get("task_id"): pair for pair in sheet.get("pairs", [])} + labels = {} + incomplete = [] + for task in held_out: + task_id = task["id"] + pair = pairs.get(task_id) + if not pair: + incomplete.append(f"human:{task_id}") + continue + allowed_ids = { + item["id"] for group in ("required_facts", "required_actions") for item in task[group] + } + expected = {rows[(task_id, arm)]["output_id"] for arm in ("baseline", "candidate")} + found = {entry.get("id") for entry in pair.get("outputs", [])} + if found != expected: + raise EvidenceError(f"human:{task_id}: output ids do not match the run") + for entry in pair["outputs"]: + output = next(r for (tid, _), r in rows.items() if tid == task_id and r["output_id"] == entry["id"]) + if entry.get("text") != output["output"]: + raise EvidenceError(f"human:{task_id}: retained output text changed") + comprehension = entry.get("comprehension") + edits = entry.get("edits_needed") + acceptable = entry.get("acceptable") + missing = entry.get("missing_required_ids") + if ( + not isinstance(comprehension, int) + or isinstance(comprehension, bool) + or comprehension not in range(1, 6) + or not isinstance(edits, int) + or isinstance(edits, bool) + or edits < 0 + or not isinstance(acceptable, bool) + or not isinstance(missing, list) + ): + incomplete.append(f"human:{task_id}:{entry['id'][:12]}") + continue + unknown = set(missing) - allowed_ids + if unknown: + raise EvidenceError(f"human:{task_id}: unknown required ids {sorted(unknown)}") + labels[entry["id"]] = { + "comprehension": comprehension, + "edits_needed": edits, + "acceptable": acceptable, + "missing_required_ids": sorted(set(missing)), + } + if pair.get("winner") not in expected | {"tie"}: + incomplete.append(f"human:{task_id}:winner") + return pairs, labels, incomplete + + +def judge_input_sha256(rows): + payload = [ + {"output_id": row["output_id"], "output": row["output"]} + for row in sorted(rows.values(), key=lambda item: item["output_id"]) + ] + encoded = json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + +def validate_judge(judge, experiment, tasks, held_out, rows, actor): + if judge is None: + return {}, {}, ["judge:missing"], {} + if judge.get("schema_version") != "adaptive-review-judge-labels/v1": + raise EvidenceError("judge labels have the wrong schema_version") + if judge.get("experiment_id") != experiment["experiment_id"]: + raise EvidenceError("judge labels refer to another experiment") + meta = judge.get("judge", {}) + if meta.get("tier") != "frontier": + raise EvidenceError("judge tier must be frontier") + if meta.get("model_family") == actor.get("model_family"): + raise EvidenceError("judge must be from a different model family than the actor") + validate_usage(meta.get("usage"), "judge") + if meta.get("external"): + if not meta.get("authorization_id"): + raise EvidenceError("external judge requires a separate authorization_id") + if meta.get("authorized_input_sha256") != judge_input_sha256(rows): + raise EvidenceError("external judge authorization is not bound to the exact output inputs") + cap = meta.get("max_spend_usd") + if not isinstance(cap, (int, float)) or isinstance(cap, bool) or cap <= 0: + raise EvidenceError("external judge requires a positive max_spend_usd") + if meta["usage"]["cost_usd"] > cap: + raise EvidenceError("external judge cost exceeds its authorized spending limit") + + pairs = {pair.get("task_id"): pair for pair in judge.get("pairs", [])} + labels = {} + incomplete = [] + for task in held_out: + task_id = task["id"] + pair = pairs.get(task_id) + if not pair: + incomplete.append(f"judge:{task_id}") + continue + allowed_ids = { + item["id"] for group in ("required_facts", "required_actions") for item in task[group] + } + expected = {rows[(task_id, arm)]["output_id"] for arm in ("baseline", "candidate")} + found = {entry.get("id") for entry in pair.get("outputs", [])} + if found != expected: + raise EvidenceError(f"judge:{task_id}: output ids do not match the run") + for entry in pair["outputs"]: + acceptable = entry.get("acceptable") + missing = entry.get("missing_required_ids") + if not isinstance(acceptable, bool) or not isinstance(missing, list): + incomplete.append(f"judge:{task_id}:{entry['id'][:12]}") + continue + unknown = set(missing) - allowed_ids + if unknown: + raise EvidenceError(f"judge:{task_id}: unknown required ids {sorted(unknown)}") + labels[entry["id"]] = acceptable + if pair.get("winner") not in expected | {"tie"}: + incomplete.append(f"judge:{task_id}:winner") + return pairs, labels, incomplete, meta + + +def cohen_kappa(a, b): + keys = sorted(set(a) & set(b)) + if len(keys) < 2: + return None, 0.0, len(keys) + agree = sum(a[key] == b[key] for key in keys) + observed = agree / len(keys) + a_true = sum(a[key] for key in keys) / len(keys) + b_true = sum(b[key] for key in keys) / len(keys) + expected = a_true * b_true + (1 - a_true) * (1 - b_true) + kappa = None if abs(1 - expected) < 1e-12 else (observed - expected) / (1 - expected) + return kappa, observed, len(keys) + + +def evaluate(run, human, judge=None): + experiment, _, tasks, held_out, rows = validate_run(run) + human_pairs, human_labels, human_missing = validate_human( + human, experiment, tasks, held_out, rows + ) + judge_pairs, judge_labels, judge_missing, judge_meta = validate_judge( + judge, experiment, tasks, held_out, rows, run["actor"] + ) + + missing_evidence = sorted(set(human_missing + judge_missing)) + thresholds = experiment["thresholds"] + arm_by_id = {row["output_id"]: arm for (_, arm), row in rows.items()} + wins = {"baseline": 0, "candidate": 0, "tie": 0} + comprehension_deltas = [] + edit_deltas = [] + candidate_missing = [] + word_deltas = [] + human_complete = not human_missing + + if human_complete: + for task in held_out: + task_id = task["id"] + pair = human_pairs[task_id] + winner = pair["winner"] + wins["tie" if winner == "tie" else arm_by_id[winner]] += 1 + baseline_id = rows[(task_id, "baseline")]["output_id"] + candidate_id = rows[(task_id, "candidate")]["output_id"] + baseline_label = human_labels[baseline_id] + candidate_label = human_labels[candidate_id] + comprehension_deltas.append( + candidate_label["comprehension"] - baseline_label["comprehension"] + ) + edit_deltas.append( + baseline_label["edits_needed"] - candidate_label["edits_needed"] + ) + candidate_missing.extend(candidate_label["missing_required_ids"]) + word_deltas.append( + len(rows[(task_id, "baseline")]["output"].split()) + - len(rows[(task_id, "candidate")]["output"].split()) + ) + + kappa, agreement, calibration_n = cohen_kappa( + {key: value["acceptable"] for key, value in human_labels.items()}, + judge_labels, + ) + winner_agreements = 0 + winner_n = 0 + if not human_missing and not judge_missing: + for task in held_out: + winner_n += 1 + winner_agreements += human_pairs[task["id"]]["winner"] == judge_pairs[task["id"]]["winner"] + + actor_usage = { + "input_tokens": sum(row["usage"]["input_tokens"] for row in rows.values()), + "output_tokens": sum(row["usage"]["output_tokens"] for row in rows.values()), + "cost_usd": round(sum(row["usage"]["cost_usd"] for row in rows.values()), 12), + } + judge_usage = judge_meta.get("usage") if judge_meta else None + cutoffs = [ + {"task_id": task_id, "arm": arm, "finish_reason": row.get("finish_reason")} + for (task_id, arm), row in rows.items() + if row.get("finish_reason") != "stop" + ] + if cutoffs: + missing_evidence.append("actor:non-stop-finish") + + metrics = { + "held_out_pairs": len(held_out), + "wins": wins if human_complete else None, + "mean_comprehension_lift": ( + sum(comprehension_deltas) / len(comprehension_deltas) + if comprehension_deltas + else None + ), + "median_edit_reduction": ( + statistics.median(edit_deltas) if edit_deltas else None + ), + "mean_word_reduction": ( + sum(word_deltas) / len(word_deltas) if word_deltas else None + ), + "candidate_missing_required": len(candidate_missing) if human_complete else None, + "judge_human_agreement": agreement if calibration_n else None, + "judge_human_kappa": kappa, + "judge_human_n": calibration_n, + "winner_agreement": winner_agreements / winner_n if winner_n else None, + "actor_usage": actor_usage, + "judge_usage": judge_usage, + "cutoffs": cutoffs, + } + + regression = human_complete and ( + len(candidate_missing) > thresholds["candidate_missing_required_max"] + or wins["baseline"] > wins["candidate"] + or metrics["mean_comprehension_lift"] < 0 + or metrics["median_edit_reduction"] < 0 + ) + improvement = ( + not missing_evidence + and human_complete + and wins["candidate"] >= thresholds["candidate_wins"] + and wins["baseline"] <= thresholds["baseline_wins_max"] + and metrics["mean_comprehension_lift"] >= thresholds["mean_comprehension_lift"] + and metrics["median_edit_reduction"] >= thresholds["median_edit_reduction"] + and len(candidate_missing) <= thresholds["candidate_missing_required_max"] + and agreement >= thresholds["judge_human_agreement"] + and kappa is not None + and kappa >= thresholds["judge_human_kappa"] + ) + if regression: + outcome = "regression" + recommendation = "reject-candidate" + elif improvement: + outcome = "improvement" + recommendation = "eligible-for-scoped-human-approval" + else: + outcome = "inconclusive" + recommendation = "do-not-install; collect-or-correct-evidence" + + return { + "schema_version": "adaptive-review-report/v1", + "experiment_id": experiment["experiment_id"], + "outcome": outcome, + "recommendation": recommendation, + "automatic_install": False, + "thresholds": thresholds, + "metrics": metrics, + "missing_evidence": sorted(set(missing_evidence)), + "limitations": [ + "The frozen corpus is small and repository-specific.", + "Human edit counts and comprehension ratings remain subjective despite blinding.", + "A passing result qualifies only a review-scoped approval; it does not justify learning or coding transfer." + ], + } + + +def synthetic_run(experiment, corpus, template): + images = {arm: expected_image_hash(experiment, arm) for arm in ("baseline", "candidate")} + outputs = [] + for task in corpus["tasks"]: + for arm in ("baseline", "candidate"): + text = f"{arm.title()} response for {task['id']} with retained requirements." + outputs.append( + { + "task_id": task["id"], + "arm": arm, + "output": text, + "output_id": output_id(task["id"], text), + "prompt_sha256": prompt_sha256(task, template), + "agent_image_hash": images[arm], + "finish_reason": "stop", + "usage": {"input_tokens": 100, "output_tokens": 20, "cost_usd": 0}, + } + ) + return { + "schema_version": "adaptive-review-run/v1", + "experiment_id": experiment["experiment_id"], + "provenance": { + "git_commit": "0" * 40, + "corpus_sha256": file_sha256(HERE / experiment["corpus"]), + "baseline_image_hash": images["baseline"], + "candidate_image_hash": images["candidate"], + }, + "actor": { + "provider": "local", + "model_family": "Qwen 27B", + "model": "self-test", + "model_digest": "sha256:" + "0" * 64, + "context_window": 32768, + "sequences": 1, + "external_fallback": False, + }, + "outputs": outputs, + } + + +def fill_labels(sheet, run, mode): + arm_by_id = {row["output_id"]: row["arm"] for row in run["outputs"]} + for index, pair in enumerate(sheet["pairs"]): + ids = {arm_by_id[entry["id"]]: entry["id"] for entry in pair["outputs"]} + for entry in pair["outputs"]: + arm = arm_by_id[entry["id"]] + entry["acceptable"] = arm == "candidate" or index % 2 == 0 + entry["missing_required_ids"] = [] + if mode == "improvement": + entry["comprehension"] = 5 if arm == "candidate" else 3 + entry["edits_needed"] = 0 if arm == "candidate" else 2 + elif mode == "regression": + entry["comprehension"] = 2 if arm == "candidate" else 4 + entry["edits_needed"] = 3 if arm == "candidate" else 0 + else: + entry["comprehension"] = 4 + entry["edits_needed"] = 1 + pair["winner"] = ( + ids["candidate"] + if mode == "improvement" + else ids["baseline"] + if mode == "regression" + else ("tie" if index >= 3 else ids["candidate"]) + ) + if mode == "regression": + first = sheet["pairs"][0] + candidate = next(entry for entry in first["outputs"] if arm_by_id[entry["id"]] == "candidate") + candidate["missing_required_ids"] = ["no-prompt-metadata"] + return sheet + + +def synthetic_judge(sheet, run): + labels = { + pair["task_id"]: { + "task_id": pair["task_id"], + "winner": pair["winner"], + "outputs": [ + { + "id": entry["id"], + "acceptable": entry["acceptable"], + "missing_required_ids": entry["missing_required_ids"], + } + for entry in pair["outputs"] + ], + } + for pair in sheet["pairs"] + } + _, _, _, _, rows = validate_run(run) + return { + "schema_version": "adaptive-review-judge-labels/v1", + "experiment_id": run["experiment_id"], + "judge": { + "tier": "frontier", + "external": True, + "provider": "self-test", + "model_family": "independent-family", + "model": "self-test", + "authorization_id": "self-test-authorization", + "authorized_input_sha256": judge_input_sha256(rows), + "max_spend_usd": 1, + "usage": {"input_tokens": 100, "output_tokens": 10, "cost_usd": 0.01}, + }, + "pairs": list(labels.values()), + } + + +def self_test(): + experiment, corpus, template, _, _ = load_contract() + run = synthetic_run(experiment, corpus, template) + checks = [] + for mode, expected in ( + ("improvement", "improvement"), + ("regression", "regression"), + ("inconclusive", "inconclusive"), + ): + sheet = fill_labels(make_blind_sheet(run), run, mode) + report = evaluate(run, sheet, synthetic_judge(sheet, run)) + checks.append((f"{mode} fixture reports {expected}", report["outcome"] == expected)) + sheet = fill_labels(make_blind_sheet(run), run, "improvement") + report = evaluate(run, sheet) + checks.append(("missing judge stays inconclusive", report["outcome"] == "inconclusive")) + bad_judge = synthetic_judge(sheet, run) + del bad_judge["judge"]["authorization_id"] + try: + evaluate(run, sheet, bad_judge) + checks.append(("external judge without authorization fails closed", False)) + except EvidenceError: + checks.append(("external judge without authorization fails closed", True)) + ok = True + for name, passed in checks: + print(f"{'PASS' if passed else 'FAIL'}: {name}") + ok = ok and passed + return 0 if ok else 1 + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument("--self-test", action="store_true") + sub = parser.add_subparsers(dest="command") + blind = sub.add_parser("blind") + blind.add_argument("--run", required=True) + blind.add_argument("--out", required=True) + score = sub.add_parser("evaluate") + score.add_argument("--run", required=True) + score.add_argument("--human", required=True) + score.add_argument("--judge") + score.add_argument("--out", required=True) + args = parser.parse_args(argv) + try: + if args.self_test: + return self_test() + if args.command == "blind": + write_json(args.out, make_blind_sheet(read_json(args.run))) + return 0 + if args.command == "evaluate": + judge = read_json(args.judge) if args.judge else None + report = evaluate(read_json(args.run), read_json(args.human), judge) + write_json(args.out, report) + print(f"adaptive-review: {report['outcome']}") + return 0 + parser.print_help() + return 2 + except (EvidenceError, KeyError, TypeError, json.JSONDecodeError) as error: + print(f"adaptive-review: invalid evidence: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/evals/paid/adaptive-review/experiment.json b/evals/paid/adaptive-review/experiment.json new file mode 100644 index 00000000..f2de60bb --- /dev/null +++ b/evals/paid/adaptive-review/experiment.json @@ -0,0 +1,62 @@ +{ + "schema_version": "adaptive-review-experiment/v1", + "experiment_id": "issue-context-v1", + "status": "frozen", + "hypothesis": "A concise-review behavior reduces editing and improves comprehension without dropping required facts or actions.", + "feedback_signal": { + "shape": "Issue drafts contain enough context to obscure the decision and next action.", + "selected_by": "https://github.com/JRichlen/agent-plugins/issues/117", + "recurrence_threshold": 3, + "qualified_sightings": [], + "qualification": "explicit-user-directed experiment; no recurrence claim" + }, + "preference": { + "statement": "For issue review, lead with the decision and retain only context needed to act.", + "scope": { + "task": "issue-review", + "repository": "JRichlen/agent-plugins" + }, + "status": "candidate", + "inspect": "experiment.json#/preference", + "correct": "edit the candidate record and create a new experiment version", + "delete": "remove this experiment candidate; no hidden or remote preference state exists" + }, + "corpus": "corpus.json", + "prompt_template": "prompt.template.txt", + "arms": { + "baseline": { + "query": "queries/baseline.json", + "image": "compiled/baseline.image.json", + "agent": "compiled/baseline.md" + }, + "candidate": { + "query": "queries/candidate.json", + "image": "compiled/candidate.image.json", + "agent": "compiled/candidate.md" + } + }, + "actor_qualification": { + "model_family": "Qwen 27B", + "model_digest_required": true, + "context_window": 32768, + "sequences": 1, + "external_fallback": false + }, + "thresholds": { + "minimum_held_out_pairs": 6, + "candidate_wins": 4, + "baseline_wins_max": 1, + "mean_comprehension_lift": 0.75, + "median_edit_reduction": 1.0, + "candidate_missing_required_max": 0, + "judge_human_agreement": 0.8, + "judge_human_kappa": 0.6 + }, + "advancement": { + "automatic_install": false, + "eligible_scope": "issue-review only", + "approval_required": true, + "monitor": "repeat the held-out comparison after material feedback or model changes", + "rollback": "remove the scoped candidate agent and restore the recorded baseline image hash" + } +} diff --git a/evals/paid/adaptive-review/prompt.template.txt b/evals/paid/adaptive-review/prompt.template.txt new file mode 100644 index 00000000..93de148e --- /dev/null +++ b/evals/paid/adaptive-review/prompt.template.txt @@ -0,0 +1,11 @@ +Review this GitHub issue draft for a maintainer who needs to decide what happens next. + +Return: +1. a ready-to-paste issue body; and +2. one short `Preservation note:` naming any required fact or action you could not keep. + +Do not invent facts, decisions, owners, dates, or evidence. + + +{{input}} + diff --git a/evals/paid/adaptive-review/queries/baseline.json b/evals/paid/adaptive-review/queries/baseline.json new file mode 100644 index 00000000..8948a647 --- /dev/null +++ b/evals/paid/adaptive-review/queries/baseline.json @@ -0,0 +1,9 @@ +{ + "name": "issue-review-baseline", + "role": "reviewer", + "task": "issue-review", + "domains": [], + "views": ["view.issue-review-baseline"], + "stance": ["goal-aligned"], + "effectCeiling": ["filesystem:read"] +} diff --git a/evals/paid/adaptive-review/queries/candidate.json b/evals/paid/adaptive-review/queries/candidate.json new file mode 100644 index 00000000..c62971ff --- /dev/null +++ b/evals/paid/adaptive-review/queries/candidate.json @@ -0,0 +1,9 @@ +{ + "name": "issue-review-concise", + "role": "reviewer", + "task": "issue-review", + "domains": [], + "views": ["view.issue-review-concise"], + "stance": ["goal-aligned"], + "effectCeiling": ["filesystem:read"] +} diff --git a/evals/paid/adaptive-review/registry/behaviors/concise-review.md b/evals/paid/adaptive-review/registry/behaviors/concise-review.md new file mode 100644 index 00000000..b4e9c454 --- /dev/null +++ b/evals/paid/adaptive-review/registry/behaviors/concise-review.md @@ -0,0 +1,39 @@ +--- +id: behavior.issue-review-concise +kind: behavior +version: 1.0.0 +--- + +# Concise issue review + + +Lead with the decision, requested outcome, or next action the issue exists to support. + + + +Preserve every requirement, constraint, unresolved risk, and explicit action that can change the decision or its execution. + + + +Use only domain lenses that affect the stated goal; omit generic review categories that produce no finding. + + + +Replace chronology and repeated rationale with the shortest causal context needed to understand why each retained fact or action matters. + + + +Separate decisions, actions, gates, and deferred work so a maintainer can scan what happens next without rereading the background. + + + +Would deleting this sentence change a decision, constraint, risk, or action? If not, remove it. + + + +Can every required fact and action in the source be located in the revision without inference? + + + +Never call an issue concise by dropping a required fact, action, limitation, or approval boundary. + diff --git a/evals/paid/adaptive-review/registry/views/baseline.md b/evals/paid/adaptive-review/registry/views/baseline.md new file mode 100644 index 00000000..c1d2fc77 --- /dev/null +++ b/evals/paid/adaptive-review/registry/views/baseline.md @@ -0,0 +1,11 @@ +--- +id: view.issue-review-baseline +kind: view +version: 1.0.0 +traits: [goal-aligned] +max_effects: [filesystem:read] +--- + +# Issue review baseline + +The untreated review arm. It adds no behavior beyond the shared task prompt. diff --git a/evals/paid/adaptive-review/registry/views/concise-review.md b/evals/paid/adaptive-review/registry/views/concise-review.md new file mode 100644 index 00000000..ef4f9465 --- /dev/null +++ b/evals/paid/adaptive-review/registry/views/concise-review.md @@ -0,0 +1,12 @@ +--- +id: view.issue-review-concise +kind: view +version: 1.0.0 +requires: [behavior.issue-review-concise] +traits: [goal-aligned] +max_effects: [filesystem:read] +--- + +# Concise issue reviewer + +The single candidate treatment for the frozen issue-context experiment. diff --git a/evals/paid/adaptive-review/self-test.sh b/evals/paid/adaptive-review/self-test.sh new file mode 100644 index 00000000..1c7a2301 --- /dev/null +++ b/evals/paid/adaptive-review/self-test.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/../../.." && pwd)" +COMPILER="$ROOT/plugins/agent-compiler/scripts/compile.py" +RENDERER="$ROOT/plugins/agent-compiler/scripts/render_claude_agent.py" +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +for arm in baseline candidate; do + python3 "$COMPILER" compile \ + --registry "$HERE/registry" \ + --query "$HERE/queries/$arm.json" \ + --out "$tmp/$arm.image.json" + diff -u "$HERE/compiled/$arm.image.json" "$tmp/$arm.image.json" + python3 "$RENDERER" --image "$tmp/$arm.image.json" --out "$tmp/$arm.md" + diff -u "$HERE/compiled/$arm.md" "$tmp/$arm.md" +done + +(cd "$HERE" && sha256sum -c corpus.sha256) +python3 "$HERE/evaluate.py" --self-test +python3 - "$HERE/status.json" <<'PY' +import json, sys +status = json.load(open(sys.argv[1], encoding="utf-8")) +assert status["outcome"] == "inconclusive" +assert status["advance"] is False +assert status["evidence"] == "not-run" +PY diff --git a/evals/paid/adaptive-review/status.json b/evals/paid/adaptive-review/status.json new file mode 100644 index 00000000..e2ac98bd --- /dev/null +++ b/evals/paid/adaptive-review/status.json @@ -0,0 +1,8 @@ +{ + "experiment_id": "issue-context-v1", + "outcome": "inconclusive", + "evidence": "not-run", + "advance": false, + "reason": "No authorized pinned local Qwen 27B actor run, blinded human labels, or separately authorized independent frontier-judge run exists.", + "next_step": "Run the frozen two-arm experiment through the approved agent-request capability, retain the private artifacts, then evaluate them without changing the preregistered thresholds." +} From 4f46cb3f648e41f2ff661f76ac65b968bc58bae2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:21:42 +0000 Subject: [PATCH 3/5] fix: isolate adaptive experiment self-test Co-authored-by: JRichlen <9574264+JRichlen@users.noreply.github.com> --- evals/cheap/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/evals/cheap/run.sh b/evals/cheap/run.sh index bd5907d1..7543ea4f 100755 --- a/evals/cheap/run.sh +++ b/evals/cheap/run.sh @@ -458,7 +458,8 @@ fi # The experiment never calls a model here. This recompiles both treatment # images, verifies the frozen corpus, and exercises the evidence scorer's # improvement/regression/inconclusive and authorization branches. -if [ -f "evals/paid/adaptive-review/self-test.sh" ]; then +if [ -f "evals/paid/adaptive-review/self-test.sh" ] \ + && [ -f "plugins/agent-compiler/scripts/compile.py" ]; then group "adaptive-review experiment self-test" if bash evals/paid/adaptive-review/self-test.sh >/dev/null 2>&1; then ok "adaptive-review frozen experiment and scorer self-test" From 0aa0fdad380b6afbc5a3e3f03b1a4ebf0383ef9a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:35:10 +0000 Subject: [PATCH 4/5] fix: harden adaptive experiment evidence Co-authored-by: JRichlen <9574264+JRichlen@users.noreply.github.com> --- evals/paid/adaptive-review/README.md | 54 +- evals/paid/adaptive-review/evaluate.py | 525 ++++++++++++++++--- evals/paid/adaptive-review/experiment.json | 6 +- evals/paid/adaptive-review/experiment.sha256 | 1 + evals/paid/adaptive-review/self-test.sh | 2 +- 5 files changed, 508 insertions(+), 80 deletions(-) mode change 100644 => 100755 evals/paid/adaptive-review/evaluate.py create mode 100644 evals/paid/adaptive-review/experiment.sha256 mode change 100644 => 100755 evals/paid/adaptive-review/self-test.sh diff --git a/evals/paid/adaptive-review/README.md b/evals/paid/adaptive-review/README.md index eb6e3e1f..bf86d095 100644 --- a/evals/paid/adaptive-review/README.md +++ b/evals/paid/adaptive-review/README.md @@ -16,7 +16,7 @@ qualified run supplies complete evidence. `agent-compiler`; `self-test.sh` recompiles and compares them byte for byte. - `experiment.json`: the feedback source, scoped inspectable preference, qualified actor route, thresholds, approval boundary, monitoring rule, and - rollback. + rollback. `experiment.sha256` makes threshold changes explicit. The recurrence record is intentionally honest: #117 explicitly selected this experiment, but no three cited sightings exist, so it makes no @@ -29,14 +29,20 @@ Only the six held-out pairs count. **Improvement** requires all of: - at least 4 candidate wins and at most 1 baseline win under blind human review; - mean comprehension lift at least `+0.75` on a 1–5 scale; - median reduction of at least 1 substantive edit before posting; -- zero required facts or actions missing from candidate outputs; +- every candidate output judged acceptable by the human; +- zero required facts or actions missing from candidate outputs according to + either human or judge; - no truncated/non-stop actor response; - independent frontier-judge agreement with human acceptability labels of at - least `0.80` and Cohen's kappa of at least `0.60`. + least `0.80`, Cohen's kappa of at least `0.60` when label variance makes it + defined, at least `0.80` winner agreement, and at least 4 judge candidate + wins. A lost candidate requirement, more baseline than candidate wins, or negative -comprehension/edit delta is a **regression**. Everything else is -**inconclusive**. Even improvement produces only +comprehension/edit delta is a **regression** and may reject the candidate +without paying for a judge. Missing judge evidence blocks an improvement but +does not erase a human-observed regression. Everything else is **inconclusive**. +Even improvement produces only `eligible-for-scoped-human-approval`; the scorer never installs anything. ## Run contract @@ -47,10 +53,13 @@ comprehension/edit delta is a **regression**. Everything else is `agent-request/v1` capability when it becomes available; do not duplicate its transport or authority semantics here. 2. Retain a private `run.json` with schema `adaptive-review-run/v1`: exact git, - corpus, prompt, and image hashes; actor model digest; every raw output; - finish reason; and numeric input tokens, output tokens, and cost for every - call. Generated run directories are git-ignored because detailed lab - evidence stays private. + experiment, corpus, prompt, and image hashes; actor model digest; a private + random blinding salt; every raw output; finish reason; and numeric input + tokens, output tokens, and cost for every call. Generated run directories + are git-ignored because detailed lab evidence stays private. Keep `run.json` + hidden from the human labeler because it contains the arm mapping. + Run from a clean committed checkout: the evaluator resolves the recorded + commit and verifies its experiment, corpus, prompt, and compiled images. 3. Create the human sheet: ```sh @@ -58,13 +67,17 @@ comprehension/edit delta is a **regression**. Everything else is --out runs//human-labels.json ``` - The sheet contains output hashes and texts but no arm names. For each output, - label comprehension (1–5), minimum substantive edits before posting, - acceptability, and missing required IDs; then choose a winner or tie. + The sheet contains the source task, required facts/actions, frozen rubric, + and output hashes/texts, but no arm names. Output order is randomized with + system entropy and IDs are salted, so the mapping cannot be reconstructed + from the public experiment. For each output, label comprehension (1–5), + minimum substantive edits before posting, acceptability, and missing + required IDs; then choose a winner or tie. 4. Give the exact same output set and task requirements to an independent - frontier judge. If grading is external, its record must bind a separate - authorization to the exact input digest and a positive spending limit. - Preserve its labels and total usage as + frontier judge from another model family. Its record must include pinned + model identity and a separate authorization receipt bound to the exact + canonical review-input digest and a finite positive spending limit. Preserve + the receipt, labels, and total usage as `adaptive-review-judge-labels/v1`. 5. Evaluate without rerunning either model: @@ -76,9 +89,14 @@ comprehension/edit delta is a **regression**. Everything else is --out runs//report.json ``` -The evaluator rejects contaminated hashes, missing pairs, unknown requirement -IDs, unqualified actors, same-family judges, unbound external inputs, unknown -usage, and overspend. Missing labels or judge evidence stays inconclusive. +The evaluator rejects contaminated hashes, missing pairs, changed review +inputs, unknown requirement IDs, unqualified actors, missing or same-family +judge identity, unbound external inputs, non-finite/unknown usage, and +overspend. Missing judge evidence cannot advance the candidate. + +The authorization receipt is inspectable provenance, not cryptographic +attestation. “Held-out” means excluded from development runs; these public +repository excerpts were not hidden from the experiment author. ## Lifecycle and rollback diff --git a/evals/paid/adaptive-review/evaluate.py b/evals/paid/adaptive-review/evaluate.py old mode 100644 new mode 100755 index ec995860..577bda82 --- a/evals/paid/adaptive-review/evaluate.py +++ b/evals/paid/adaptive-review/evaluate.py @@ -4,12 +4,17 @@ import argparse import hashlib import json +import math +import secrets import statistics +import subprocess import sys +from copy import deepcopy from pathlib import Path HERE = Path(__file__).resolve().parent +REPO_ROOT = HERE.parents[2] class EvidenceError(ValueError): @@ -31,8 +36,43 @@ def file_sha256(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() -def output_id(task_id, text): - return hashlib.sha256(f"{task_id}\0{text}".encode("utf-8")).hexdigest() +def validate_digest(value, where): + if ( + not isinstance(value, str) + or not value.startswith("sha256:") + or len(value) != 71 + or any(c not in "0123456789abcdef" for c in value[7:]) + ): + raise EvidenceError(f"{where} must be sha256 followed by 64 lowercase hex characters") + + +REVIEW_RUBRIC = ( + "Judge whether each revision is easy to comprehend and ready to post while " + "preserving every listed required fact and action. Count the minimum " + "substantive edits still needed; do not reward brevity that drops a requirement." +) +HUMAN_INSTRUCTIONS = ( + "Label without identifying arms. For each output set comprehension 1-5, " + "edits_needed to the minimum substantive edits before posting, acceptable " + "true/false, and missing_required_ids. Set winner to an output id or tie." +) + + +def require_exact_keys(value, expected, where): + if not isinstance(value, dict): + raise EvidenceError(f"{where} must be an object") + extra = set(value) - set(expected) + missing = set(expected) - set(value) + if extra or missing: + raise EvidenceError( + f"{where} keys differ: missing={sorted(missing)} extra={sorted(extra)}" + ) + + +def output_id(task_id, arm, text, blinding_salt): + return hashlib.sha256( + f"{blinding_salt}\0{task_id}\0{arm}\0{text}".encode("utf-8") + ).hexdigest() def rendered_prompt(task, template): @@ -69,11 +109,52 @@ def validate_usage(usage, where): if not isinstance(value, int) or isinstance(value, bool) or value < 0: raise EvidenceError(f"{where}: {key} must be a non-negative integer") cost = usage.get("cost_usd") - if not isinstance(cost, (int, float)) or isinstance(cost, bool) or cost < 0: + if ( + not isinstance(cost, (int, float)) + or isinstance(cost, bool) + or not math.isfinite(cost) + or cost < 0 + ): raise EvidenceError(f"{where}: cost_usd must be a known non-negative number") -def validate_run(run): +def committed_bytes(commit, relative_path): + result = subprocess.run( + ["git", "-C", str(REPO_ROOT), "show", f"{commit}:{relative_path}"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if result.returncode: + raise EvidenceError(f"git_commit does not contain {relative_path}") + return result.stdout + + +def validate_git_provenance(commit, provenance, experiment): + paths = { + "experiment_sha256": "evals/paid/adaptive-review/experiment.json", + "corpus_sha256": "evals/paid/adaptive-review/corpus.json", + } + for key, relative_path in paths.items(): + digest = hashlib.sha256(committed_bytes(commit, relative_path)).hexdigest() + if digest != provenance[key]: + raise EvidenceError(f"git_commit {relative_path} does not match {key}") + for arm in ("baseline", "candidate"): + relative_path = f"evals/paid/adaptive-review/compiled/{arm}.image.json" + try: + image = json.loads(committed_bytes(commit, relative_path)) + except json.JSONDecodeError as error: + raise EvidenceError(f"git_commit has invalid {arm} image JSON") from error + if image.get("hash") != provenance[f"{arm}_image_hash"]: + raise EvidenceError(f"git_commit {arm} image does not match its recorded hash") + committed_template = committed_bytes( + commit, "evals/paid/adaptive-review/prompt.template.txt" + ) + if committed_template != (HERE / experiment["prompt_template"]).read_bytes(): + raise EvidenceError("git_commit prompt template differs from the evaluator checkout") + + +def validate_run(run, verify_git=True): experiment, corpus, template, tasks, held_out = load_contract() if run.get("schema_version") != "adaptive-review-run/v1": raise EvidenceError("run schema_version must be adaptive-review-run/v1") @@ -81,8 +162,16 @@ def validate_run(run): raise EvidenceError("run experiment_id does not match the frozen experiment") provenance = run.get("provenance", {}) + blinding_salt = provenance.get("blinding_salt", "") + if ( + len(blinding_salt) != 64 + or any(c not in "0123456789abcdef" for c in blinding_salt) + ): + raise EvidenceError("run blinding_salt must be a private random 64-hex value") if provenance.get("corpus_sha256") != file_sha256(HERE / experiment["corpus"]): raise EvidenceError("run corpus hash does not match the frozen corpus") + if provenance.get("experiment_sha256") != file_sha256(HERE / "experiment.json"): + raise EvidenceError("run experiment hash does not match the frozen thresholds") for arm in ("baseline", "candidate"): key = f"{arm}_image_hash" if provenance.get(key) != expected_image_hash(experiment, arm): @@ -90,6 +179,8 @@ def validate_run(run): commit = provenance.get("git_commit", "") if len(commit) != 40 or any(c not in "0123456789abcdef" for c in commit): raise EvidenceError("run git_commit must be a lowercase 40-hex commit") + if verify_git: + validate_git_provenance(commit, provenance, experiment) actor = run.get("actor", {}) required = experiment["actor_qualification"] @@ -103,8 +194,10 @@ def validate_run(run): for key, expected in checks.items(): if actor.get(key) != expected: raise EvidenceError(f"actor {key} must be {expected!r}") - if required["model_digest_required"] and not actor.get("model_digest"): - raise EvidenceError("actor model_digest is required") + if not isinstance(actor.get("model"), str) or not actor["model"].strip(): + raise EvidenceError("actor model is required") + if required["model_digest_required"]: + validate_digest(actor.get("model_digest"), "actor model_digest") expected_pairs = {(task_id, arm) for task_id in tasks for arm in ("baseline", "candidate")} rows = {} @@ -117,7 +210,7 @@ def validate_run(run): text = row.get("output") if not isinstance(text, str) or not text.strip(): raise EvidenceError(f"{pair}: output must be non-empty text") - if row.get("output_id") != output_id(pair[0], text): + if row.get("output_id") != output_id(pair[0], pair[1], text, blinding_salt): raise EvidenceError(f"{pair}: output_id does not match task and output") if row.get("prompt_sha256") != prompt_sha256(tasks[pair[0]], template): raise EvidenceError(f"{pair}: prompt hash does not match the shared frozen prompt") @@ -131,16 +224,53 @@ def validate_run(run): return experiment, corpus, tasks, held_out, rows -def make_blind_sheet(run): - experiment, _, _, held_out, rows = validate_run(run) +def review_payload(experiment, held_out, rows): + return { + "schema_version": "adaptive-review-review-input/v1", + "experiment_id": experiment["experiment_id"], + "rubric": REVIEW_RUBRIC, + "pairs": [ + { + "task_id": task["id"], + "source": task["source"], + "title": task["title"], + "input": task["input"], + "required_facts": task["required_facts"], + "required_actions": task["required_actions"], + "outputs": sorted( + [ + { + "id": rows[(task["id"], arm)]["output_id"], + "text": rows[(task["id"], arm)]["output"], + } + for arm in ("baseline", "candidate") + ], + key=lambda item: item["id"], + ), + } + for task in held_out + ], + } + + +def review_input_sha256(experiment, held_out, rows): + encoded = json.dumps( + review_payload(experiment, held_out, rows), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + +def make_blind_sheet(run, verify_git=True): + experiment, _, _, held_out, rows = validate_run(run, verify_git=verify_git) result = { "schema_version": "adaptive-review-human-labels/v1", "experiment_id": experiment["experiment_id"], - "instructions": ( - "Label without identifying arms. For each output set comprehension 1-5, " - "edits_needed to the minimum substantive edits before posting, acceptable " - "true/false, and missing_required_ids. Set winner to an output id or tie." - ), + "review_input_sha256": review_input_sha256(experiment, held_out, rows), + "rubric": REVIEW_RUBRIC, + "instructions": HUMAN_INSTRUCTIONS, "pairs": [], } for task in held_out: @@ -156,18 +286,61 @@ def make_blind_sheet(run): } for arm in ("baseline", "candidate") ] - if int(hashlib.sha256(f"{experiment['experiment_id']}:{task_id}".encode()).hexdigest(), 16) % 2: - candidates.reverse() - result["pairs"].append({"task_id": task_id, "outputs": candidates, "winner": None}) + secrets.SystemRandom().shuffle(candidates) + result["pairs"].append( + { + "task_id": task_id, + "source": task["source"], + "title": task["title"], + "input": task["input"], + "required_facts": task["required_facts"], + "required_actions": task["required_actions"], + "outputs": candidates, + "winner": None, + } + ) return result +def index_pairs(raw_pairs, held_out, where): + if not isinstance(raw_pairs, list): + raise EvidenceError(f"{where} pairs must be a list") + expected = {task["id"] for task in held_out} + ids = [pair.get("task_id") for pair in raw_pairs if isinstance(pair, dict)] + if len(ids) != len(raw_pairs): + raise EvidenceError(f"{where} pairs must all be objects") + if len(ids) != len(set(ids)): + raise EvidenceError(f"{where} contains duplicate task labels") + unknown = set(ids) - expected + if unknown: + raise EvidenceError(f"{where} contains unknown tasks {sorted(unknown)}") + return {pair["task_id"]: pair for pair in raw_pairs} + + def validate_human(sheet, experiment, tasks, held_out, rows): + require_exact_keys( + sheet, + { + "schema_version", + "experiment_id", + "review_input_sha256", + "rubric", + "instructions", + "pairs", + }, + "human labels", + ) if sheet.get("schema_version") != "adaptive-review-human-labels/v1": raise EvidenceError("human labels have the wrong schema_version") if sheet.get("experiment_id") != experiment["experiment_id"]: raise EvidenceError("human labels refer to another experiment") - pairs = {pair.get("task_id"): pair for pair in sheet.get("pairs", [])} + if sheet.get("review_input_sha256") != review_input_sha256(experiment, held_out, rows): + raise EvidenceError("human labels do not match the canonical review input") + if sheet.get("rubric") != REVIEW_RUBRIC: + raise EvidenceError("human labels do not carry the frozen rubric") + if sheet.get("instructions") != HUMAN_INSTRUCTIONS: + raise EvidenceError("human labels do not carry the frozen instructions") + pairs = index_pairs(sheet["pairs"], held_out, "human labels") labels = {} incomplete = [] for task in held_out: @@ -176,14 +349,52 @@ def validate_human(sheet, experiment, tasks, held_out, rows): if not pair: incomplete.append(f"human:{task_id}") continue + require_exact_keys( + pair, + { + "task_id", + "source", + "title", + "input", + "required_facts", + "required_actions", + "outputs", + "winner", + }, + f"human:{task_id}", + ) + for key in ("source", "title", "input", "required_facts", "required_actions"): + if pair.get(key) != task[key]: + raise EvidenceError(f"human:{task_id}: frozen {key} changed") allowed_ids = { item["id"] for group in ("required_facts", "required_actions") for item in task[group] } expected = {rows[(task_id, arm)]["output_id"] for arm in ("baseline", "candidate")} - found = {entry.get("id") for entry in pair.get("outputs", [])} - if found != expected: + entries = pair.get("outputs", []) + if not isinstance(entries, list): + raise EvidenceError(f"human:{task_id}: outputs must be a list") + found_ids = [entry.get("id") for entry in entries if isinstance(entry, dict)] + found = set(found_ids) + if ( + len(found_ids) != len(entries) + or len(found_ids) != len(expected) + or len(found_ids) != len(found) + or found != expected + ): raise EvidenceError(f"human:{task_id}: output ids do not match the run") for entry in pair["outputs"]: + require_exact_keys( + entry, + { + "id", + "text", + "comprehension", + "edits_needed", + "acceptable", + "missing_required_ids", + }, + f"human:{task_id}:output", + ) output = next(r for (tid, _), r in rows.items() if tid == task_id and r["output_id"] == entry["id"]) if entry.get("text") != output["output"]: raise EvidenceError(f"human:{task_id}: retained output text changed") @@ -217,40 +428,78 @@ def validate_human(sheet, experiment, tasks, held_out, rows): return pairs, labels, incomplete -def judge_input_sha256(rows): - payload = [ - {"output_id": row["output_id"], "output": row["output"]} - for row in sorted(rows.values(), key=lambda item: item["output_id"]) - ] - encoded = json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True) - return hashlib.sha256(encoded.encode("utf-8")).hexdigest() - - def validate_judge(judge, experiment, tasks, held_out, rows, actor): if judge is None: return {}, {}, ["judge:missing"], {} + require_exact_keys( + judge, + { + "schema_version", + "experiment_id", + "review_input_sha256", + "authorization", + "judge", + "pairs", + }, + "judge labels", + ) if judge.get("schema_version") != "adaptive-review-judge-labels/v1": raise EvidenceError("judge labels have the wrong schema_version") if judge.get("experiment_id") != experiment["experiment_id"]: raise EvidenceError("judge labels refer to another experiment") + expected_input_hash = review_input_sha256(experiment, held_out, rows) + if judge.get("review_input_sha256") != expected_input_hash: + raise EvidenceError("judge labels do not match the canonical review input") meta = judge.get("judge", {}) + require_exact_keys( + meta, + { + "tier", + "external", + "provider", + "model_family", + "model", + "model_digest", + "usage", + }, + "judge", + ) if meta.get("tier") != "frontier": raise EvidenceError("judge tier must be frontier") - if meta.get("model_family") == actor.get("model_family"): + for key in ("provider", "model_family", "model", "model_digest"): + if not isinstance(meta.get(key), str) or not meta[key].strip(): + raise EvidenceError(f"judge {key} is required") + validate_digest(meta["model_digest"], "judge model_digest") + if meta["provider"].strip().casefold() == "local": + raise EvidenceError("the frontier judge must use an external provider") + if meta["model_family"].strip().casefold() == actor.get("model_family", "").strip().casefold(): raise EvidenceError("judge must be from a different model family than the actor") validate_usage(meta.get("usage"), "judge") - if meta.get("external"): - if not meta.get("authorization_id"): - raise EvidenceError("external judge requires a separate authorization_id") - if meta.get("authorized_input_sha256") != judge_input_sha256(rows): - raise EvidenceError("external judge authorization is not bound to the exact output inputs") - cap = meta.get("max_spend_usd") - if not isinstance(cap, (int, float)) or isinstance(cap, bool) or cap <= 0: - raise EvidenceError("external judge requires a positive max_spend_usd") - if meta["usage"]["cost_usd"] > cap: - raise EvidenceError("external judge cost exceeds its authorized spending limit") - - pairs = {pair.get("task_id"): pair for pair in judge.get("pairs", [])} + if meta.get("external") is not True: + raise EvidenceError("the frontier judge must be recorded as external") + authorization = judge.get("authorization", {}) + require_exact_keys( + authorization, + {"id", "approved_by", "approved_at", "input_sha256", "max_spend_usd"}, + "judge authorization", + ) + for key in ("id", "approved_by", "approved_at"): + if not isinstance(authorization.get(key), str) or not authorization[key].strip(): + raise EvidenceError(f"external judge authorization {key} is required") + if authorization.get("input_sha256") != expected_input_hash: + raise EvidenceError("external judge authorization is not bound to the exact review input") + cap = authorization.get("max_spend_usd") + if ( + not isinstance(cap, (int, float)) + or isinstance(cap, bool) + or not math.isfinite(cap) + or cap <= 0 + ): + raise EvidenceError("external judge requires a finite positive max_spend_usd") + if meta["usage"]["cost_usd"] > cap: + raise EvidenceError("external judge cost exceeds its authorized spending limit") + + pairs = index_pairs(judge["pairs"], held_out, "judge labels") labels = {} incomplete = [] for task in held_out: @@ -259,14 +508,31 @@ def validate_judge(judge, experiment, tasks, held_out, rows, actor): if not pair: incomplete.append(f"judge:{task_id}") continue + require_exact_keys( + pair, {"task_id", "winner", "outputs"}, f"judge:{task_id}" + ) allowed_ids = { item["id"] for group in ("required_facts", "required_actions") for item in task[group] } expected = {rows[(task_id, arm)]["output_id"] for arm in ("baseline", "candidate")} - found = {entry.get("id") for entry in pair.get("outputs", [])} - if found != expected: + entries = pair.get("outputs", []) + if not isinstance(entries, list): + raise EvidenceError(f"judge:{task_id}: outputs must be a list") + found_ids = [entry.get("id") for entry in entries if isinstance(entry, dict)] + found = set(found_ids) + if ( + len(found_ids) != len(entries) + or len(found_ids) != len(expected) + or len(found_ids) != len(found) + or found != expected + ): raise EvidenceError(f"judge:{task_id}: output ids do not match the run") for entry in pair["outputs"]: + require_exact_keys( + entry, + {"id", "acceptable", "missing_required_ids"}, + f"judge:{task_id}:output", + ) acceptable = entry.get("acceptable") missing = entry.get("missing_required_ids") if not isinstance(acceptable, bool) or not isinstance(missing, list): @@ -294,8 +560,8 @@ def cohen_kappa(a, b): return kappa, observed, len(keys) -def evaluate(run, human, judge=None): - experiment, _, tasks, held_out, rows = validate_run(run) +def evaluate(run, human, judge=None, verify_git=True): + experiment, _, tasks, held_out, rows = validate_run(run, verify_git=verify_git) human_pairs, human_labels, human_missing = validate_human( human, experiment, tasks, held_out, rows ) @@ -310,6 +576,7 @@ def evaluate(run, human, judge=None): comprehension_deltas = [] edit_deltas = [] candidate_missing = [] + candidate_acceptable = [] word_deltas = [] human_complete = not human_missing @@ -330,6 +597,7 @@ def evaluate(run, human, judge=None): baseline_label["edits_needed"] - candidate_label["edits_needed"] ) candidate_missing.extend(candidate_label["missing_required_ids"]) + candidate_acceptable.append(candidate_label["acceptable"]) word_deltas.append( len(rows[(task_id, "baseline")]["output"].split()) - len(rows[(task_id, "candidate")]["output"].split()) @@ -341,10 +609,22 @@ def evaluate(run, human, judge=None): ) winner_agreements = 0 winner_n = 0 + judge_wins = {"baseline": 0, "candidate": 0, "tie": 0} + judge_candidate_missing = [] if not human_missing and not judge_missing: for task in held_out: winner_n += 1 - winner_agreements += human_pairs[task["id"]]["winner"] == judge_pairs[task["id"]]["winner"] + human_winner = human_pairs[task["id"]]["winner"] + judge_winner = judge_pairs[task["id"]]["winner"] + winner_agreements += human_winner == judge_winner + judge_wins["tie" if judge_winner == "tie" else arm_by_id[judge_winner]] += 1 + candidate_id = rows[(task["id"], "candidate")]["output_id"] + judge_candidate = next( + entry + for entry in judge_pairs[task["id"]]["outputs"] + if entry["id"] == candidate_id + ) + judge_candidate_missing.extend(judge_candidate["missing_required_ids"]) actor_usage = { "input_tokens": sum(row["usage"]["input_tokens"] for row in rows.values()), @@ -352,6 +632,13 @@ def evaluate(run, human, judge=None): "cost_usd": round(sum(row["usage"]["cost_usd"] for row in rows.values()), 12), } judge_usage = judge_meta.get("usage") if judge_meta else None + total_usage = dict(actor_usage) + if judge_usage: + total_usage = { + "input_tokens": actor_usage["input_tokens"] + judge_usage["input_tokens"], + "output_tokens": actor_usage["output_tokens"] + judge_usage["output_tokens"], + "cost_usd": round(actor_usage["cost_usd"] + judge_usage["cost_usd"], 12), + } cutoffs = [ {"task_id": task_id, "arm": arm, "finish_reason": row.get("finish_reason")} for (task_id, arm), row in rows.items() @@ -375,12 +662,22 @@ def evaluate(run, human, judge=None): sum(word_deltas) / len(word_deltas) if word_deltas else None ), "candidate_missing_required": len(candidate_missing) if human_complete else None, + "candidate_acceptable_rate": ( + sum(candidate_acceptable) / len(candidate_acceptable) + if candidate_acceptable + else None + ), "judge_human_agreement": agreement if calibration_n else None, "judge_human_kappa": kappa, "judge_human_n": calibration_n, "winner_agreement": winner_agreements / winner_n if winner_n else None, + "judge_wins": judge_wins if winner_n else None, + "judge_candidate_missing_required": ( + len(judge_candidate_missing) if winner_n else None + ), "actor_usage": actor_usage, "judge_usage": judge_usage, + "total_usage": total_usage, "cutoffs": cutoffs, } @@ -397,10 +694,14 @@ def evaluate(run, human, judge=None): and wins["baseline"] <= thresholds["baseline_wins_max"] and metrics["mean_comprehension_lift"] >= thresholds["mean_comprehension_lift"] and metrics["median_edit_reduction"] >= thresholds["median_edit_reduction"] + and metrics["candidate_acceptable_rate"] >= thresholds["candidate_acceptable_rate"] and len(candidate_missing) <= thresholds["candidate_missing_required_max"] and agreement >= thresholds["judge_human_agreement"] - and kappa is not None - and kappa >= thresholds["judge_human_kappa"] + and (kappa is None or kappa >= thresholds["judge_human_kappa"]) + and metrics["winner_agreement"] >= thresholds["judge_winner_agreement"] + and judge_wins["candidate"] >= thresholds["judge_candidate_wins"] + and len(judge_candidate_missing) + <= thresholds["judge_candidate_missing_required_max"] ) if regression: outcome = "regression" @@ -423,7 +724,9 @@ def evaluate(run, human, judge=None): "missing_evidence": sorted(set(missing_evidence)), "limitations": [ "The frozen corpus is small and repository-specific.", + "Held-out means excluded from development runs, not hidden from the experiment author.", "Human edit counts and comprehension ratings remain subjective despite blinding.", + "The authorization record is an auditable receipt, not cryptographic attestation.", "A passing result qualifies only a review-scoped approval; it does not justify learning or coding transfer." ], } @@ -431,6 +734,7 @@ def evaluate(run, human, judge=None): def synthetic_run(experiment, corpus, template): images = {arm: expected_image_hash(experiment, arm) for arm in ("baseline", "candidate")} + blinding_salt = "0" * 64 outputs = [] for task in corpus["tasks"]: for arm in ("baseline", "candidate"): @@ -440,7 +744,7 @@ def synthetic_run(experiment, corpus, template): "task_id": task["id"], "arm": arm, "output": text, - "output_id": output_id(task["id"], text), + "output_id": output_id(task["id"], arm, text, blinding_salt), "prompt_sha256": prompt_sha256(task, template), "agent_image_hash": images[arm], "finish_reason": "stop", @@ -452,7 +756,9 @@ def synthetic_run(experiment, corpus, template): "experiment_id": experiment["experiment_id"], "provenance": { "git_commit": "0" * 40, + "blinding_salt": blinding_salt, "corpus_sha256": file_sha256(HERE / experiment["corpus"]), + "experiment_sha256": file_sha256(HERE / "experiment.json"), "baseline_image_hash": images["baseline"], "candidate_image_hash": images["candidate"], }, @@ -516,19 +822,24 @@ def synthetic_judge(sheet, run): } for pair in sheet["pairs"] } - _, _, _, _, rows = validate_run(run) return { "schema_version": "adaptive-review-judge-labels/v1", "experiment_id": run["experiment_id"], + "review_input_sha256": sheet["review_input_sha256"], + "authorization": { + "id": "self-test-authorization", + "approved_by": "self-test", + "approved_at": "2000-01-01T00:00:00Z", + "input_sha256": sheet["review_input_sha256"], + "max_spend_usd": 1, + }, "judge": { "tier": "frontier", "external": True, "provider": "self-test", "model_family": "independent-family", "model": "self-test", - "authorization_id": "self-test-authorization", - "authorized_input_sha256": judge_input_sha256(rows), - "max_spend_usd": 1, + "model_digest": "sha256:" + "1" * 64, "usage": {"input_tokens": 100, "output_tokens": 10, "cost_usd": 0.01}, }, "pairs": list(labels.values()), @@ -544,19 +855,113 @@ def self_test(): ("regression", "regression"), ("inconclusive", "inconclusive"), ): - sheet = fill_labels(make_blind_sheet(run), run, mode) - report = evaluate(run, sheet, synthetic_judge(sheet, run)) + sheet = fill_labels(make_blind_sheet(run, verify_git=False), run, mode) + report = evaluate(run, sheet, synthetic_judge(sheet, run), verify_git=False) checks.append((f"{mode} fixture reports {expected}", report["outcome"] == expected)) - sheet = fill_labels(make_blind_sheet(run), run, "improvement") - report = evaluate(run, sheet) + sheet = fill_labels(make_blind_sheet(run, verify_git=False), run, "improvement") + report = evaluate(run, sheet, verify_git=False) checks.append(("missing judge stays inconclusive", report["outcome"] == "inconclusive")) + unacceptable = deepcopy(sheet) + for pair in unacceptable["pairs"]: + candidate_id = next( + row["output_id"] + for row in run["outputs"] + if row["task_id"] == pair["task_id"] and row["arm"] == "candidate" + ) + next(entry for entry in pair["outputs"] if entry["id"] == candidate_id)["acceptable"] = False + report = evaluate( + run, unacceptable, synthetic_judge(unacceptable, run), verify_git=False + ) + checks.append( + ("unacceptable candidates cannot advance", report["outcome"] == "inconclusive") + ) bad_judge = synthetic_judge(sheet, run) - del bad_judge["judge"]["authorization_id"] + del bad_judge["authorization"]["id"] try: - evaluate(run, sheet, bad_judge) + evaluate(run, sheet, bad_judge, verify_git=False) checks.append(("external judge without authorization fails closed", False)) except EvidenceError: checks.append(("external judge without authorization fails closed", True)) + bad_judge = synthetic_judge(sheet, run) + bad_judge["judge"]["external"] = False + try: + evaluate(run, sheet, bad_judge, verify_git=False) + checks.append(("external flag cannot bypass authorization", False)) + except EvidenceError: + checks.append(("external flag cannot bypass authorization", True)) + bad_judge = synthetic_judge(sheet, run) + bad_judge["judge"]["usage"]["cost_usd"] = float("nan") + try: + evaluate(run, sheet, bad_judge, verify_git=False) + checks.append(("non-finite judge cost fails closed", False)) + except EvidenceError: + checks.append(("non-finite judge cost fails closed", True)) + bad_judge = synthetic_judge(sheet, run) + del bad_judge["judge"]["model_family"] + try: + evaluate(run, sheet, bad_judge, verify_git=False) + checks.append(("missing judge identity fails closed", False)) + except EvidenceError: + checks.append(("missing judge identity fails closed", True)) + corrupted = deepcopy(sheet) + corrupted["pairs"][0]["required_facts"] = [] + try: + evaluate( + run, corrupted, synthetic_judge(corrupted, run), verify_git=False + ) + checks.append(("changed human review inputs fail closed", False)) + except EvidenceError: + checks.append(("changed human review inputs fail closed", True)) + leaked = deepcopy(sheet) + leaked["pairs"][0]["outputs"][0]["arm"] = "candidate" + try: + evaluate(run, leaked, synthetic_judge(sheet, run), verify_git=False) + checks.append(("arm metadata in human labels fails closed", False)) + except EvidenceError: + checks.append(("arm metadata in human labels fails closed", True)) + judge_omission = synthetic_judge(sheet, run) + first_task = sheet["pairs"][0]["task_id"] + candidate_id = next( + row["output_id"] + for row in run["outputs"] + if row["task_id"] == first_task and row["arm"] == "candidate" + ) + next( + entry + for entry in judge_omission["pairs"][0]["outputs"] + if entry["id"] == candidate_id + )["missing_required_ids"] = ["no-prompt-metadata"] + report = evaluate(run, sheet, judge_omission, verify_git=False) + checks.append( + ("judge-reported candidate omission blocks advancement", report["outcome"] == "inconclusive") + ) + duplicated = deepcopy(sheet) + duplicated["pairs"][0]["outputs"].append(deepcopy(duplicated["pairs"][0]["outputs"][0])) + try: + evaluate(run, duplicated, synthetic_judge(sheet, run), verify_git=False) + checks.append(("duplicate human output labels fail closed", False)) + except EvidenceError: + checks.append(("duplicate human output labels fail closed", True)) + duplicate_judge = synthetic_judge(sheet, run) + duplicate_judge["pairs"].append(deepcopy(duplicate_judge["pairs"][0])) + try: + evaluate(run, sheet, duplicate_judge, verify_git=False) + checks.append(("duplicate judge task labels fail closed", False)) + except EvidenceError: + checks.append(("duplicate judge task labels fail closed", True)) + try: + validate_run(run, verify_git=True) + checks.append(("nonexistent git provenance fails closed", False)) + except EvidenceError: + checks.append(("nonexistent git provenance fails closed", True)) + salt = run["provenance"]["blinding_salt"] + checks.append( + ( + "identical arm text still gets distinct opaque ids", + output_id("same-task", "baseline", "same", salt) + != output_id("same-task", "candidate", "same", salt), + ) + ) ok = True for name, passed in checks: print(f"{'PASS' if passed else 'FAIL'}: {name}") diff --git a/evals/paid/adaptive-review/experiment.json b/evals/paid/adaptive-review/experiment.json index f2de60bb..4ca9fcef 100644 --- a/evals/paid/adaptive-review/experiment.json +++ b/evals/paid/adaptive-review/experiment.json @@ -48,9 +48,13 @@ "baseline_wins_max": 1, "mean_comprehension_lift": 0.75, "median_edit_reduction": 1.0, + "candidate_acceptable_rate": 1.0, "candidate_missing_required_max": 0, + "judge_candidate_missing_required_max": 0, "judge_human_agreement": 0.8, - "judge_human_kappa": 0.6 + "judge_human_kappa": 0.6, + "judge_winner_agreement": 0.8, + "judge_candidate_wins": 4 }, "advancement": { "automatic_install": false, diff --git a/evals/paid/adaptive-review/experiment.sha256 b/evals/paid/adaptive-review/experiment.sha256 new file mode 100644 index 00000000..d487a387 --- /dev/null +++ b/evals/paid/adaptive-review/experiment.sha256 @@ -0,0 +1 @@ +8ab641a85bb55ee186b84f7d09e96fbc66e0c4e744b79866cd290140d93a2fc4 experiment.json diff --git a/evals/paid/adaptive-review/self-test.sh b/evals/paid/adaptive-review/self-test.sh old mode 100644 new mode 100755 index 1c7a2301..b9c0e341 --- a/evals/paid/adaptive-review/self-test.sh +++ b/evals/paid/adaptive-review/self-test.sh @@ -18,7 +18,7 @@ for arm in baseline candidate; do diff -u "$HERE/compiled/$arm.md" "$tmp/$arm.md" done -(cd "$HERE" && sha256sum -c corpus.sha256) +(cd "$HERE" && sha256sum -c corpus.sha256 experiment.sha256) python3 "$HERE/evaluate.py" --self-test python3 - "$HERE/status.json" <<'PY' import json, sys From 48e1d885aa2665ed1faecf87a81d44419b2bae7d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:40:19 +0000 Subject: [PATCH 5/5] fix: bind adaptive run provenance Co-authored-by: JRichlen <9574264+JRichlen@users.noreply.github.com> --- evals/paid/adaptive-review/evaluate.py | 97 +++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/evals/paid/adaptive-review/evaluate.py b/evals/paid/adaptive-review/evaluate.py index 577bda82..f6f2d0f7 100755 --- a/evals/paid/adaptive-review/evaluate.py +++ b/evals/paid/adaptive-review/evaluate.py @@ -101,7 +101,26 @@ def expected_image_hash(experiment, arm): return image["hash"] +def expected_agent_sha256(experiment, arm): + return file_sha256(HERE / experiment["arms"][arm]["agent"]) + + +def canonical_image_hash(image): + body = { + key: value + for key, value in image.items() + if key not in ("hash", "registryRevision") + } + encoded = json.dumps( + body, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ) + return "sha256:" + hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + def validate_usage(usage, where): + require_exact_keys( + usage, {"input_tokens", "output_tokens", "cost_usd"}, f"{where}:usage" + ) if not isinstance(usage, dict): raise EvidenceError(f"{where}: missing usage") for key in ("input_tokens", "output_tokens"): @@ -145,8 +164,14 @@ def validate_git_provenance(commit, provenance, experiment): image = json.loads(committed_bytes(commit, relative_path)) except json.JSONDecodeError as error: raise EvidenceError(f"git_commit has invalid {arm} image JSON") from error + if canonical_image_hash(image) != image.get("hash"): + raise EvidenceError(f"git_commit {arm} image has an invalid canonical hash") if image.get("hash") != provenance[f"{arm}_image_hash"]: raise EvidenceError(f"git_commit {arm} image does not match its recorded hash") + agent_path = f"evals/paid/adaptive-review/compiled/{arm}.md" + agent_digest = hashlib.sha256(committed_bytes(commit, agent_path)).hexdigest() + if agent_digest != provenance[f"{arm}_agent_sha256"]: + raise EvidenceError(f"git_commit {arm} agent does not match its recorded hash") committed_template = committed_bytes( commit, "evals/paid/adaptive-review/prompt.template.txt" ) @@ -156,12 +181,31 @@ def validate_git_provenance(commit, provenance, experiment): def validate_run(run, verify_git=True): experiment, corpus, template, tasks, held_out = load_contract() + require_exact_keys( + run, + {"schema_version", "experiment_id", "provenance", "actor", "outputs"}, + "run", + ) if run.get("schema_version") != "adaptive-review-run/v1": raise EvidenceError("run schema_version must be adaptive-review-run/v1") if run.get("experiment_id") != experiment["experiment_id"]: raise EvidenceError("run experiment_id does not match the frozen experiment") provenance = run.get("provenance", {}) + require_exact_keys( + provenance, + { + "git_commit", + "blinding_salt", + "corpus_sha256", + "experiment_sha256", + "baseline_image_hash", + "candidate_image_hash", + "baseline_agent_sha256", + "candidate_agent_sha256", + }, + "run provenance", + ) blinding_salt = provenance.get("blinding_salt", "") if ( len(blinding_salt) != 64 @@ -176,6 +220,9 @@ def validate_run(run, verify_git=True): key = f"{arm}_image_hash" if provenance.get(key) != expected_image_hash(experiment, arm): raise EvidenceError(f"run {key} does not match the compiled image") + agent_key = f"{arm}_agent_sha256" + if provenance.get(agent_key) != expected_agent_sha256(experiment, arm): + raise EvidenceError(f"run {agent_key} does not match the rendered agent") commit = provenance.get("git_commit", "") if len(commit) != 40 or any(c not in "0123456789abcdef" for c in commit): raise EvidenceError("run git_commit must be a lowercase 40-hex commit") @@ -183,6 +230,19 @@ def validate_run(run, verify_git=True): validate_git_provenance(commit, provenance, experiment) actor = run.get("actor", {}) + require_exact_keys( + actor, + { + "provider", + "model_family", + "model", + "model_digest", + "context_window", + "sequences", + "external_fallback", + }, + "actor", + ) required = experiment["actor_qualification"] checks = { "provider": "local", @@ -201,7 +261,23 @@ def validate_run(run, verify_git=True): expected_pairs = {(task_id, arm) for task_id in tasks for arm in ("baseline", "candidate")} rows = {} - for row in run.get("outputs", []): + if not isinstance(run["outputs"], list): + raise EvidenceError("run outputs must be a list") + for row in run["outputs"]: + require_exact_keys( + row, + { + "task_id", + "arm", + "output", + "output_id", + "prompt_sha256", + "agent_image_hash", + "finish_reason", + "usage", + }, + "run output", + ) pair = (row.get("task_id"), row.get("arm")) if pair in rows: raise EvidenceError(f"duplicate output for {pair}") @@ -761,6 +837,8 @@ def synthetic_run(experiment, corpus, template): "experiment_sha256": file_sha256(HERE / "experiment.json"), "baseline_image_hash": images["baseline"], "candidate_image_hash": images["candidate"], + "baseline_agent_sha256": expected_agent_sha256(experiment, "baseline"), + "candidate_agent_sha256": expected_agent_sha256(experiment, "candidate"), }, "actor": { "provider": "local", @@ -954,6 +1032,23 @@ def self_test(): checks.append(("nonexistent git provenance fails closed", False)) except EvidenceError: checks.append(("nonexistent git provenance fails closed", True)) + open_run = deepcopy(run) + open_run["unbound"] = "field" + try: + validate_run(open_run, verify_git=False) + checks.append(("unknown run fields fail closed", False)) + except EvidenceError: + checks.append(("unknown run fields fail closed", True)) + image = read_json(HERE / experiment["arms"]["candidate"]["image"]) + valid_image_hash = canonical_image_hash(image) == image["hash"] + tampered_image = deepcopy(image) + tampered_image["behavior"][0]["content"] += " changed" + checks.append( + ( + "canonical image hash detects behavior mutation", + valid_image_hash and canonical_image_hash(tampered_image) != image["hash"], + ) + ) salt = run["provenance"]["blinding_salt"] checks.append( (