From 50d79913b7d29360177fea5b764c3817a7e35d14 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Tue, 25 Aug 2026 17:01:10 +0000 Subject: [PATCH 1/2] Qualify release builds against exact provider contracts --- .github/workflows/bench-live.yml | 6 +- .github/workflows/ci.yml | 6 +- .github/workflows/release.yml | 23 +- Cargo.lock | 2 +- Cargo.toml | 9 +- bench/README.md | 82 ++- bench/baseline.json | 18 +- bench/src/harness.test.ts | 56 +- bench/src/harness.ts | 36 +- bench/src/live.ts | 76 ++- bench/src/livemodels-score.test.ts | 104 +++- bench/src/livemodels-score.ts | 88 ++- bench/src/livemodels.test.ts | 70 ++- bench/src/livemodels.ts | 122 +++- bench/src/run.test.ts | 25 +- bench/src/run.ts | 39 +- bench/src/scorer-eval.test.ts | 421 +++++++++++++- bench/src/scorer-eval.ts | 898 ++++++++++++++++++++++++++--- bench/src/verify-admission.ts | 1 + provisional-models.json | 1 + src/config.rs | 163 +++++- src/llm.rs | 440 ++++++++++---- src/prompt.rs | 13 +- tests/e2e.rs | 89 +-- 24 files changed, 2439 insertions(+), 349 deletions(-) diff --git a/.github/workflows/bench-live.yml b/.github/workflows/bench-live.yml index d3d4dc8..1b59ae5 100644 --- a/.github/workflows/bench-live.yml +++ b/.github/workflows/bench-live.yml @@ -11,6 +11,9 @@ on: upstream_provider: description: Exact ZDR provider identity serving every model in the profile required: true + upstream_provider_route: + description: Exact ZDR endpoint route serving every model in the profile + required: true cost_cap_usd: description: Abort when projected qualification spend exceeds this amount required: true @@ -32,6 +35,7 @@ jobs: POSTIL_BENCH_MODE: live POSTIL_BENCH_PAIRS: ${{ inputs.pairs }} POSTIL_BENCH_UPSTREAM_PROVIDER: ${{ inputs.upstream_provider }} + POSTIL_BENCH_UPSTREAM_PROVIDER_ROUTE: ${{ inputs.upstream_provider_route }} POSTIL_BENCH_REPEATS: "3" POSTIL_BENCH_COST_CAP_USD: ${{ inputs.cost_cap_usd }} POSTIL_API_BASE: https://openrouter.ai/api/v1 @@ -70,7 +74,7 @@ jobs: test -n "$COMPLETION_KEY_SHA256" - name: Build release binary - run: cargo build --quiet --release --features qualification-candidate + run: cargo build --quiet --release - name: Install benchmark dependencies working-directory: bench diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb8edbf..b30dca8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,8 +20,8 @@ jobs: - run: cargo fmt --check - run: cargo clippy --quiet --all-targets -- -D warnings - run: cargo test --quiet - - run: cargo build --quiet --features qualification-candidate - - run: cargo test --quiet --features qualification-candidate + - run: cargo build --quiet --no-default-features + - run: cargo test --quiet --no-default-features - run: cargo build --quiet --release musl: @@ -56,7 +56,7 @@ jobs: - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 - - run: cargo build --quiet --release --features qualification-candidate + - run: cargo build --quiet --release - run: bun install --frozen-lockfile working-directory: bench - run: bun test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 294ae51..c30be0a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -117,8 +117,11 @@ jobs: working-directory: bench env: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + POSTIL_BIN: ${{ github.workspace }}/target/release/postil POSTIL_SCORER_EVAL_REPEATS: "3" POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER: Azure + POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/eu + POSTIL_QUALIFICATION_SOURCE_SHA: ${{ github.sha }} run: >- bun run scorer-eval --json-out "${{ runner.temp }}/scorer-eval-report.json" @@ -128,7 +131,9 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: scorer-eval-report - path: ${{ runner.temp }}/scorer-eval-report.json + path: | + ${{ runner.temp }}/scorer-eval-report.json + ${{ runner.temp }}/scorer-eval-report.json.partial if-no-files-found: warn retention-days: 30 @@ -136,7 +141,21 @@ jobs: working-directory: bench env: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - run: bun run bench:live -- --run-id "release-${{ github.ref_name }}" --json-out "${{ runner.temp }}/bench-live-report.json" + run: >- + bun run bench:live -- --screen-profile ../provisional-models.json + --run-id "release-${{ github.ref_name }}" + --json-out "${{ runner.temp }}/bench-live-report.json" + + - name: Upload the diff-file live report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bench-live-report + path: | + ${{ runner.temp }}/bench-live-report.json + ${{ runner.temp }}/bench-live-report.json.partial + if-no-files-found: warn + retention-days: 30 - name: Compare against the recorded baseline id: compare-baseline diff --git a/Cargo.lock b/Cargo.lock index 0713bd9..0a96cc4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1114,7 +1114,7 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "postil-cli" -version = "0.9.1" +version = "0.9.2" dependencies = [ "aho-corasick", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 2c36510..d801d6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "postil-cli" -version = "0.9.1" +version = "0.9.2" edition = "2024" description = "Postil: a low-noise AI review gate. Silent on clean PRs, hard gate on real risk." license = "Apache-2.0" @@ -15,9 +15,10 @@ name = "postil_cli" path = "src/lib.rs" [features] -default = [] -# Builds the hermetic candidate-admission path used by the managed benchmark. -# Release artifacts do not enable this feature. +# Release binaries include the hermetic candidate-admission path so model +# qualification exercises the exact bytes users receive. Runtime activation +# still requires the guarded CI, loopback capture, and candidate profile inputs. +default = ["qualification-candidate"] qualification-candidate = [] [dependencies] diff --git a/bench/README.md b/bench/README.md index 70e06ae..e41afa5 100644 --- a/bench/README.md +++ b/bench/README.md @@ -49,7 +49,7 @@ sensitive development evidence and keep them out of public artifacts. ## Running (mock mode: default, CI) ```sh -cargo build --quiet --release --features qualification-candidate # from the repo root +cargo build --quiet --release # from the repo root cd bench bun install bun run bench # add --json or --json-out report.json for machine output @@ -108,12 +108,12 @@ missing, mismatched, stale, or invalid bundle rejects a nonempty manifest. The empty manifest is exempt because it admits no models. Report and profile checksums detect changes; they do not authenticate who produced a candidate. -The workflow builds a qualification-only binary feature that accepts one exact -candidate profile inside the hermetic benchmark. The feature requires CI, -managed privacy enforcement, and a loopback mock forge. Release binaries omit -the feature. Candidate runs therefore execute the production hosted planner, -request preflight, price ceilings, consensus, and scorer behavior without -granting unqualified profiles authority in a deployed service. +The release binary includes a guarded qualification path that accepts one exact +candidate profile inside the hermetic benchmark. Activation requires CI, +managed privacy enforcement, and a loopback mock forge. Candidate runs execute +the production hosted planner, request preflight, price ceilings, consensus, +and scorer behavior without granting unqualified profiles authority in a +deployed service. ```sh export MODEL_API_KEY=... # or POSTIL_API_KEY, OPENROUTER_API_KEY, or LLM_API_KEY @@ -123,12 +123,14 @@ export POSTIL_BENCH_REPEATS=3 export POSTIL_API_BASE=https://openrouter.ai/api/v1 export POSTIL_API_FORMAT=openai-compatible export POSTIL_BENCH_UPSTREAM_PROVIDER='Exact upstream provider name' +export POSTIL_BENCH_UPSTREAM_PROVIDER_ROUTE='exact/provider-route' bun run bench --json-out report.json --manifest-out ../qualified-models.json ``` -Live admission emits public report schema version 2. Consumers must call +Live admission emits public report schema version 4. Consumers must call `parseLiveModelsReport`; unversioned reports and unknown schema versions are -rejected. Public case diagnostics contain counts and SHA-256 digests only. +rejected. The parser upgrades retained schema-3 reports by defaulting their +provider route to the recorded provider identity. Public case diagnostics contain counts and SHA-256 digests only. Finding prose, target contracts, raw evaluator responses, evaluator reasons, and diagnostic text are absent. @@ -174,7 +176,9 @@ Pass it with `--pricing-file prices.json` or `POSTIL_BENCH_PRICING_FILE=prices.json`. Prices are positive canonical decimal strings that must be exactly representable as integer micros per million tokens. Every row names the exact upstream provider passed with -`--upstream-provider`; a mismatch fails before inference. Each admitted profile carries immutable input and output price bounds +`--upstream-provider`; `--upstream-provider-route` identifies the exact +endpoint slug when it differs from the response provider identity. A mismatch +fails before inference. Each admitted profile carries immutable input and output price bounds for its exact generator and scorer model set. The catalog request uses the inference credential when no file is supplied and fails closed when any model is unpriced. Catalog redirects are rejected so credentials remain bound to the @@ -197,7 +201,7 @@ Admission requires all of these in every repeat: - no clean false blocks and at most 5% clean cases with any finding - no execution, structured-output, grounding, statusline, or usage-accounting failure - mean pair cost at most $0.04 and mean review latency at most 15 seconds -- every review costs at most the $1 hosted operation cap +- every review reports at most $1 of actual provider cost - per-repeat p95 latency at most 30 seconds and maximum latency at most 60 seconds Findings suppressed by the scorer count as detector evidence but cannot satisfy @@ -239,8 +243,9 @@ plan for every fixture before inference. It includes bounded planner, selected source and synthesis requests, scoring, consensus, fallback, repair, and bounded post-processing requests. Transport retries reserve exact exposure at runtime under the same hard limits. Preflight rejects missing prices, more than six models, a review -above the $1 hosted operation cap, a total above the configured qualification -cap, or a cap outside `(0, $70]`. A single model used for more than one role is +whose worst-case retry projection exceeds $25, a total above the configured qualification +cap, or a cap outside `(0, $70]`. Runtime independently rejects more than $1 of +reported provider cost or 20 million reported tokens. A single model used for more than one role is priced for each planned invocation. Atomic attribution accepts at most three findings anchored in one authored region. More is a fidelity failure. Each decision is limited to a 4 KiB input, @@ -275,22 +280,31 @@ This diagnostic can reject a scorer but cannot admit a production pair; pair qualification above is the admission authority. ```sh -cargo build --quiet --release --features qualification-candidate +cargo build --quiet --release cd bench export MODEL_API_KEY=... # or LLM_API_KEY / OPENROUTER_API_KEY POSTIL_SCORER_EVAL_MODELS=provider/candidate-a,provider/candidate-b \ POSTIL_SCORER_EVAL_REPEATS=5 \ POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER=provider-name \ +POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE=provider-route \ +POSTIL_SCORER_EVAL_ROOT_DIR=.runs/scorer-eval/unique-run-id \ bun run scorer-eval --json-out scorer-eval-report.json ``` +The provider name is the identity echoed in responses. The optional provider +route is the exact OpenRouter endpoint slug; it defaults to the provider name. +Qualification starts only when the evaluator source bundle matches `HEAD`. +The retained report binds that commit to an evaluator SHA-256 digest, the exact +release-binary digest, provider identity and route, ZDR and fallback policy, +and sorted per-model maximum price bounds. + The default candidates come from `config.toml`; the workflow input may override them explicitly. Qualification repeats 12 fixtures five times: six unambiguous authored target risks and six injected false findings. Admission requires a complete matrix, no malformed, repaired, fallback, or reason-contract failures, all target risks preserved as published gate failures, at least 80% of false -findings actually suppressed overall and per fixture, p50/p95/max scorer latency -at or below 5/10/20 seconds, known live +findings actually suppressed overall and per fixture, scorer-only p50/p95/max +latency at or below 5/10/20 seconds, known live catalog pricing, and mean scorer cost at or below $0.005 per case. A failed candidate makes the command exit nonzero after writing its report. Candidate listing alone never enables the embedded scorer. Before any model call, the @@ -303,10 +317,11 @@ of repair context. Scorer responses also fail admission when provider usage is missing or malformed, runtime accounting is incomplete, or the assessment is not trimmed single-line text ending in sentence punctuation. The prompt targets at most 180 UTF-8 bytes, and the -parser rejects more than 240 UTF-8 bytes. Each -case is killed one second -after the 20-second admission limit, and teardown aborts outstanding provider -requests. A timeout rejects that candidate immediately rather than running the +parser rejects more than 240 UTF-8 bytes. Each adjudication and scorer request +has its own 20-second admission limit. A 45-second outer safety cutoff leaves +both sequential live phases their full window plus bounded fixture overhead, +and teardown aborts outstanding provider requests. A timeout rejects that +candidate immediately rather than running the rest of its matrix. Any other admission-fatal structural result, including an unroutable provider response, malformed envelope, scorer mismatch, invalid reason, incomplete usage, or repair attempt, also stops only that candidate. @@ -330,16 +345,16 @@ is written to any repo. ```sh export MODEL_API_KEY=... # required; never logged or printed REVIEW_MODEL=provider/qualified-model bun run bench:live -# Screen exact fixtures against the provisional GLM route. Repeat --case. -REVIEW_MODEL=z-ai/glm-5.2 bun run bench:live -- \ - --run-id glm-5-2-fireworks-screen-1 \ +# Screen exact fixtures against the provisional Luna route. Repeat --case. +REVIEW_MODEL=openai/gpt-5.6-luna bun run bench:live -- \ + --run-id luna-azure-eu-screen-1 \ --screen-profile ../provisional-models.json \ --case prompt-injection-auth-bypass \ --case near-duplicate-auth-clean # Keep provider calls inside the live screen's 180-second case watchdog. POSTIL_LLM_REQUEST_TIMEOUT_SECS=60 POSTIL_LLM_TOTAL_TIMEOUT_SECS=170 \ - REVIEW_MODEL=z-ai/glm-5.2 bun run bench:live -- \ - --run-id glm-5-2-fireworks-bounded-timeouts \ + REVIEW_MODEL=openai/gpt-5.6-luna bun run bench:live -- \ + --run-id luna-azure-eu-bounded-timeouts \ --screen-profile ../provisional-models.json \ --case prompt-injection-auth-bypass # A profile with a scorerChain can exercise the production scorer path. @@ -390,6 +405,9 @@ scorer chain, exact upstream provider, canonical managed endpoint, and price ceilings. Requests deny provider data collection, require zero-data retention, pin that provider without fallbacks, and enforce the profile prices. The report records the selected IDs and marks the evidence as non-admission screening. +When the harness explicitly disables scoring, screening projects only the +profile's exact generator role; the unexercised scorer chain remains recorded +in the profile but does not have to be active in the child process. Formal admission rejects `--case`, `--scorer-model`, and `--screen-profile`. Every cost total says whether all calls supplied complete provider accounting. @@ -440,11 +458,11 @@ nondeterministic. Treat them as internal evidence, not a published benchmark. ## Release gate -The `Release` workflow runs a full-corpus diff-file live pass -(`REVIEW_MODEL=z-ai/glm-5.2 bun run bench:live`) against the plain release -binary before it builds any target, then checks the result with -`bun run bench:compare`. A material regression blocks the release: `build` -depends on the `bench-live` job. +The `Release` workflow runs a full-corpus diff-file live pass against the Luna +profile in `provisional-models.json` and the exact release binary before it +builds any other target, then checks the result with `bun run bench:compare`. +A material regression blocks the release: `build` depends on the `bench-live` +job. `compare-baseline.ts` computes five metrics from the live report and compares each against the matching model entry in `bench/baseline.json`: authored-target @@ -488,7 +506,9 @@ bun run bench:compare -- --run-id # resolves .runs/live//repo # Re-baseline deliberately, after confirming the new numbers are an accepted # tradeoff (a model change, a fixture change, an intentional pipeline change): -REVIEW_MODEL=z-ai/glm-5.2 bun run bench:live -- --json-out live-report.json +REVIEW_MODEL=openai/gpt-5.6-luna bun run bench:live -- \ + --screen-profile ../provisional-models.json \ + --json-out live-report.json bun run bench:compare -- --result live-report.json --record ``` diff --git a/bench/baseline.json b/bench/baseline.json index ae61525..c4f2cfc 100644 --- a/bench/baseline.json +++ b/bench/baseline.json @@ -2,7 +2,7 @@ "schemaVersion": 1, "corpus": { "fixtureCorpusSha256": "8e4c2cb9ad5a7efdfe6a875566d20133e905155b6f693a873595adf6c069e065", - "evaluatorSha256": "afadb5b9097433000a0c0ee9a81ccb23ecd01cfeb92829421f3e1acb68015c0d" + "evaluatorSha256": "02f4ad84bd86d1f65aafc7dcc077981972a95fdd0f5ab5140f5ea98ba9372797" }, "profiles": { "z-ai/glm-5.2": { @@ -26,21 +26,21 @@ }, "openai/gpt-5.6-luna": { "populated": true, - "generatedAt": "2026-08-19T14:50:35.108Z", + "generatedAt": "2026-08-25T16:39:48.601Z", "reviewMode": "exhaustive", - "sourceRunAt": "2026-08-19T14:50:35.108Z", + "sourceRunAt": "2026-08-25T16:37:11.211Z", "providerContractEnforced": true, - "screeningProfileSha256": "52fd3e2a85ffd715c00c33cc2d3664da84237cdfc2a19cb2671f0d5cbc9c93b5", + "screeningProfileSha256": "aea05c3f5622cebec480d2a8daf5bb53055bc0160206e6f22e889391ea71fa49", "upstreamProviderIdentity": "Azure", "totalCases": 70, "scoredCases": 70, - "detectionRate": 0.8947368421052632, + "detectionRate": 0.9122807017543859, "falsePositives": 0, - "gateVerdictCorrectness": 0.8285714285714286, - "meanCostUsdPerCase": 0.0007770375714285714, + "gateVerdictCorrectness": 0.8, + "meanCostUsdPerCase": 0.0008069545000000001, "latencyMs": { - "p50": 8649, - "p95": 17559 + "p50": 6066, + "p95": 10607 } } } diff --git a/bench/src/harness.test.ts b/bench/src/harness.test.ts index 1dd63dc..7d14292 100644 --- a/bench/src/harness.test.ts +++ b/bench/src/harness.test.ts @@ -4,6 +4,8 @@ import { benchmarkCase, displayPromptPath, evaluateNoReviewPublication, + envelopeV1, + HOSTED_ADMISSION_PROJECTION_CAP_MICROS, modelRequestKind, parseUnifiedDiffFiles, reviewPromptContainsAddedCoordinate, @@ -17,6 +19,39 @@ import { MUST_BLOCK_FIXTURE_COUNT, } from "./livemodels-score"; +test("accepts a bounded review admission projection above the hosted operation cap", () => { + const parsed = envelopeV1.parse({ + version: 1, + summary: "", + silent: true, + findings: [], + resolved: [], + counts: { info: 0, warn: 0, error: 0, suppressed: 0, ungrounded: 0 }, + confidenceBuckets: [0, 0, 0, 0, 0], + gate: { failOn: "error", failing: false }, + modelUsed: "fixture/model", + usage: { promptTokens: 0, completionTokens: 0 }, + reviewAdmission: { + providerAttempts: 80, + serializedInputBytes: 9_636_851, + outputTokens: 481_216, + projectedCostMicros: 3_312_931, + }, + durationMs: 0, + baseSha: null, + headSha: null, + sinceSha: null, + }); + expect(parsed.reviewAdmission?.projectedCostMicros).toBe(3_312_931); + expect(() => envelopeV1.parse({ + ...parsed, + reviewAdmission: { + ...parsed.reviewAdmission, + projectedCostMicros: HOSTED_ADMISSION_PROJECTION_CAP_MICROS + 1, + }, + })).toThrow(); +}); + test("classifies review routing only from trusted request metadata", () => { expect(reviewRequestMetadata({ "x-postil-review-route": "synthesis", @@ -412,7 +447,26 @@ describe("benchmark fixtures", () => { const github = await startMockGithub(c); try { expect(evaluateNoReviewPublication(github)).toEqual([]); - await fetch(`${github.baseUrl}${github.pullPath}/reviews`, { method: "POST", body: "{}" }); + const review = await fetch(`${github.baseUrl}${github.pullPath}/reviews`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + commit_id: c.headSha, + comments: [{ body: "Finding marker" }], + }), + }).then((response) => response.json()) as { + commit_id: string; + comments: Array<{ id: number; body: string; commit_id: string }>; + }; + expect(review.commit_id).toBe(c.headSha); + expect(review.comments).toEqual([{ + id: 3001, + body: "Finding marker", + commit_id: c.headSha, + }]); + expect(await fetch(`${github.baseUrl}${github.pullPath}/comments`).then((response) => + response.json() + )).toEqual([]); await fetch(`${github.baseUrl}${github.pullPath.replace("/pulls/", "/issues/")}/comments`, { method: "POST", body: "{}", diff --git a/bench/src/harness.ts b/bench/src/harness.ts index ddaa211..b35724a 100644 --- a/bench/src/harness.ts +++ b/bench/src/harness.ts @@ -412,6 +412,8 @@ const suppressedEnvelopeFinding = z.object({ reason: z.string(), }); +export const HOSTED_ADMISSION_PROJECTION_CAP_MICROS = 25_000_000; + export const envelopeV1 = z.object({ version: z.literal(1), summary: z.string(), @@ -483,7 +485,8 @@ export const envelopeV1 = z.object({ providerAttempts: z.number().int().nonnegative(), serializedInputBytes: z.number().int().nonnegative(), outputTokens: z.number().int().nonnegative(), - projectedCostMicros: z.number().int().nonnegative().max(1_000_000), + projectedCostMicros: z.number().int().nonnegative() + .max(HOSTED_ADMISSION_PROJECTION_CAP_MICROS), }).optional(), usageAccountingComplete: z.boolean().optional(), durationMs: z.number().int().nonnegative(), @@ -769,6 +772,8 @@ export async function startMockGithub(c: BenchmarkCase) { const requests: RecordedRequest[] = []; const checkRunNames = new Map(); // id -> check name let nextCheckRunId = 1001; + let nextReviewId = 2001; + let nextReviewCommentId = 3001; const pullPath = `/repos/${c.repo}/pulls/${c.pullNumber}`; const repositoryPath = `/repos/${c.repo}`; const pullFilesPath = `${pullPath}/files`; @@ -864,8 +869,35 @@ export async function startMockGithub(c: BenchmarkCase) { } if (req.method === "POST" && url.pathname === `${pullPath}/reviews`) { + const review = safeJson(body) as { + body?: unknown; + commit_id?: unknown; + comments?: Array<{ body?: unknown }>; + } | undefined; + const id = nextReviewId; + nextReviewId += 1; + const comments = (review?.comments ?? []).map((comment) => { + const commentId = nextReviewCommentId; + nextReviewCommentId += 1; + return { + id: commentId, + body: typeof comment.body === "string" ? comment.body : "", + commit_id: c.headSha, + }; + }); res.writeHead(200, { "content-type": "application/json" }); - res.end("{}"); + res.end(JSON.stringify({ + id, + body: typeof review?.body === "string" ? review.body : "", + commit_id: typeof review?.commit_id === "string" ? review.commit_id : c.headSha, + comments, + })); + return; + } + + if (req.method === "GET" && url.pathname === `${pullPath}/comments`) { + res.writeHead(200, { "content-type": "application/json" }); + res.end("[]"); return; } diff --git a/bench/src/live.ts b/bench/src/live.ts index 47a4717..b3b340a 100644 --- a/bench/src/live.ts +++ b/bench/src/live.ts @@ -36,7 +36,10 @@ import { benchmarkCase, type BenchmarkCaseInput, envelopeV1, type Envelope } fro import { formatCanonicalDecimal, parseCanonicalDecimal, + providerContractEvidence, + providerContractSha256, sumCanonicalDecimals, + type ProviderContractEvidence, } from "./livemodels-score"; const execFile = promisify(execFileCb); @@ -176,6 +179,9 @@ export interface LiveSummary { providerContractEnforced: boolean; screeningProfileSha256: string | null; upstreamProviderIdentity: string | null; + upstreamProviderRoute: string | null; + providerContractSha256: string | null; + providerContract: ProviderContractEvidence | null; timeoutOverrides: LiveTimeoutOverrides; ranAt: string; totalCases: number; @@ -364,6 +370,9 @@ async function writeLiveRunContract( async function screeningProfileMetadata(path: string): Promise<{ sha256: string; upstreamProviderIdentity: string; + upstreamProviderRoute: string; + providerContractSha256: string; + providerContract: ProviderContractEvidence; }> { const bytes = await readFile(resolve(path)); const parsed = safeJson(bytes.toString("utf8")); @@ -371,15 +380,65 @@ async function screeningProfileMetadata(path: string): Promise<{ ? parsed as Record : null; const upstreamProviderIdentity = record?.upstreamProviderIdentity; + const upstreamProviderRoute = record?.upstreamProviderRoute; + const generatorChain = record?.generatorChain; + const scorerChain = record?.scorerChain; + const modelPriceBounds = record?.modelPriceBounds; if ( typeof upstreamProviderIdentity !== "string" || upstreamProviderIdentity.trim().length === 0 ) { throw new Error("screening profile must declare a nonempty upstreamProviderIdentity"); } + if (typeof upstreamProviderRoute !== "string" || upstreamProviderRoute.trim().length === 0) { + throw new Error("screening profile must declare a nonempty upstreamProviderRoute"); + } + const models = (value: unknown, field: string): string[] => { + if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string" || entry.trim() === "")) { + throw new Error(`screening profile ${field} must contain model IDs`); + } + return value as string[]; + }; + if (!Array.isArray(modelPriceBounds)) { + throw new Error("screening profile modelPriceBounds must be an array"); + } + const pricing = new Map(); + for (const bound of modelPriceBounds) { + if (typeof bound !== "object" || bound === null || Array.isArray(bound)) { + throw new Error("screening profile modelPriceBounds contains an invalid row"); + } + const row = bound as Record; + if ( + typeof row.model !== "string" || row.model.trim() === "" || + !Number.isSafeInteger(row.inputMicrosPerMillionTokens) || + Number(row.inputMicrosPerMillionTokens) < 1 || + !Number.isSafeInteger(row.outputMicrosPerMillionTokens) || + Number(row.outputMicrosPerMillionTokens) < 1 || + pricing.has(row.model) + ) { + throw new Error("screening profile modelPriceBounds contains an invalid row"); + } + pricing.set(row.model, { + inputMicrosPerMillionTokens: Number(row.inputMicrosPerMillionTokens), + outputMicrosPerMillionTokens: Number(row.outputMicrosPerMillionTokens), + }); + } + const providerContract = providerContractEvidence( + upstreamProviderIdentity, + upstreamProviderRoute, + pricing, + models(generatorChain, "generatorChain"), + models(scorerChain, "scorerChain"), + ); return { sha256: createHash("sha256").update(bytes).digest("hex"), upstreamProviderIdentity, + upstreamProviderRoute, + providerContractSha256: providerContractSha256(providerContract), + providerContract, }; } @@ -390,6 +449,7 @@ export async function evaluatorSourceSha256(): Promise { "src/api-key.ts", "src/harness.ts", "src/live.ts", + "src/livemodels-score.ts", ]; const hash = createHash("sha256"); for (const source of sources) { @@ -886,7 +946,13 @@ function summarize( fixtureCorpusSha256: string, evaluatorSha256: string, provider: LiveProvider, - screeningProfile: { sha256: string; upstreamProviderIdentity: string } | null, + screeningProfile: { + sha256: string; + upstreamProviderIdentity: string; + upstreamProviderRoute: string; + providerContractSha256: string; + providerContract: ProviderContractEvidence; + } | null, timeoutOverrides: LiveTimeoutOverrides, ): LiveSummary { const defects = results.filter((r) => r.type === "defect"); @@ -932,6 +998,9 @@ function summarize( providerContractEnforced: options.screenProfilePath !== undefined, screeningProfileSha256: screeningProfile?.sha256 ?? null, upstreamProviderIdentity: screeningProfile?.upstreamProviderIdentity ?? null, + upstreamProviderRoute: screeningProfile?.upstreamProviderRoute ?? null, + providerContractSha256: screeningProfile?.providerContractSha256 ?? null, + providerContract: screeningProfile?.providerContract ?? null, timeoutOverrides, ranAt: new Date().toISOString(), totalCases: results.length, @@ -993,7 +1062,10 @@ export function formatLiveReport(report: LiveReport): string { `Provider contract: ${s.providerContractEnforced ? "enforced" : "not enforced"}`, ...(s.upstreamProviderIdentity === null ? [] - : [`Upstream provider: ${s.upstreamProviderIdentity}; profile ${s.screeningProfileSha256}`]), + : [ + `Upstream provider: ${s.upstreamProviderIdentity}; route ${s.upstreamProviderRoute}; ` + + `profile ${s.screeningProfileSha256}; contract ${s.providerContractSha256}`, + ]), `Detection ${s.detectionRate} defects | severity match (exact) ${s.severityMatchExact} | ` + `severity match (+/-1 tier) ${s.severityMatchWithinOneTier} | ` + `silent-on-clean ${s.silentOnClean} | false-positives ${s.falsePositives}`, diff --git a/bench/src/livemodels-score.test.ts b/bench/src/livemodels-score.test.ts index 4cd6715..6a8866d 100644 --- a/bench/src/livemodels-score.test.ts +++ b/bench/src/livemodels-score.test.ts @@ -15,11 +15,14 @@ import { parseCanonicalDecimal, pricingFromCatalog, pricingFromZdrCatalog, + providerContractEvidence, + providerContractSha256, qualificationPairId, scoreLiveCase, toSiteModelAggregate, type LiveModelCaseResult, type ModelPricing, + type ProviderContractEvidence, type QualificationPair, } from "./livemodels-score"; @@ -53,6 +56,61 @@ const prices = new Map([ }], ]); +test("binds exact routing and price pins in retained provider evidence", () => { + const contract = providerContractEvidence( + "Azure", + "azure/eu", + prices, + [pair.generatorModel], + [pair.scorerModel, pair.generatorModel], + ); + expect(contract).toMatchObject({ + upstreamProviderIdentity: "Azure", + upstreamProviderRoute: "azure/eu", + dataCollection: "deny", + zeroDataRetention: true, + allowFallbacks: false, + maxPricePinned: true, + }); + expect(contract.modelPriceBounds).toEqual([ + { + model: pair.generatorModel, + roles: ["generator", "scorer"], + inputMicrosPerMillionTokens: 1_000_000, + outputMicrosPerMillionTokens: 2_000_000, + }, + { + model: pair.scorerModel, + roles: ["scorer"], + inputMicrosPerMillionTokens: 1_000_000, + outputMicrosPerMillionTokens: 2_000_000, + }, + ]); + expect(providerContractSha256(contract)).toHaveLength(64); + const reordered: ProviderContractEvidence = { + modelPriceBounds: contract.modelPriceBounds.map((bound) => ({ + outputMicrosPerMillionTokens: bound.outputMicrosPerMillionTokens, + roles: bound.roles, + inputMicrosPerMillionTokens: bound.inputMicrosPerMillionTokens, + model: bound.model, + })), + maxPriceUnits: contract.maxPriceUnits, + maxPricePinned: contract.maxPricePinned, + scorerRequireParameters: contract.scorerRequireParameters, + generatorRequireParameters: contract.generatorRequireParameters, + allowFallbacks: contract.allowFallbacks, + zeroDataRetention: contract.zeroDataRetention, + dataCollection: contract.dataCollection, + upstreamProviderRoute: contract.upstreamProviderRoute, + upstreamProviderIdentity: contract.upstreamProviderIdentity, + benchmarkProviderIdentity: contract.benchmarkProviderIdentity, + version: contract.version, + }; + expect(providerContractSha256(reordered)).toBe(providerContractSha256(contract)); + expect(providerContractSha256({ ...contract, upstreamProviderRoute: "azure/us" })) + .not.toBe(providerContractSha256(contract)); +}); + function fixture( classification: "mustBlock" | "advisory" | "clean", id = `case-${classification}`, @@ -563,13 +621,55 @@ describe("report and pricing utilities", () => { provider_name: "Together", status: 0, pricing: { prompt: "0.000001", completion: "0.000002" }, - supported_parameters: ["max_tokens", "temperature"], + supported_parameters: [ + "max_completion_tokens", + "reasoning", + "reasoning_effort", + "response_format", + ], }] }, [pair.scorerModel], "Together", new Map([ - [pair.scorerModel, ["max_tokens", "response_format", "temperature"]], + [pair.scorerModel, [ + "max_completion_tokens", + "reasoning", + "reasoning_effort", + "response_format", + "structured_outputs", + ]], ])); expect(catalog.has(pair.scorerModel)).toBe(false); }); + test("binds pricing to an exact route while retaining the response provider identity", () => { + const catalog = pricingFromZdrCatalog({ data: [ + { + model_id: pair.scorerModel, + provider_name: "Azure", + tag: "azure", + status: 0, + pricing: { prompt: "0.000001", completion: "0.000002" }, + supported_parameters: ["max_completion_tokens"], + }, + { + model_id: pair.scorerModel, + provider_name: "Azure", + tag: "azure/eu", + status: 0, + pricing: { prompt: "0.000003", completion: "0.000004" }, + supported_parameters: ["max_completion_tokens"], + }, + ] }, [pair.scorerModel], "Azure", new Map([ + [pair.scorerModel, ["max_completion_tokens"]], + ]), "azure/eu"); + + expect(catalog.get(pair.scorerModel)).toEqual({ + providerIdentity: "Azure", + promptUsdPerToken: 0.000003, + completionUsdPerToken: 0.000004, + inputMicrosPerMillionTokens: 3_000_000, + outputMicrosPerMillionTokens: 4_000_000, + }); + }); + test("rejects duplicate requested catalog ids and canonical aliases before pricing", () => { const price = { prompt: "0.000001", completion: "0.000002" }; expect(() => pricingFromCatalog({ data: [ diff --git a/bench/src/livemodels-score.ts b/bench/src/livemodels-score.ts index 6fd41f5..4373b0f 100644 --- a/bench/src/livemodels-score.ts +++ b/bench/src/livemodels-score.ts @@ -60,6 +60,81 @@ export interface ModelPricing { outputMicrosPerMillionTokens: number; } +export interface ProviderContractEvidence { + version: 1; + benchmarkProviderIdentity: "openrouter:managed-routing"; + upstreamProviderIdentity: string; + upstreamProviderRoute: string; + dataCollection: "deny"; + zeroDataRetention: true; + allowFallbacks: false; + generatorRequireParameters: false; + scorerRequireParameters: true; + maxPricePinned: true; + maxPriceUnits: "USD per million tokens"; + modelPriceBounds: Array<{ + model: string; + roles: Array<"generator" | "scorer">; + inputMicrosPerMillionTokens: number; + outputMicrosPerMillionTokens: number; + }>; +} + +export function providerContractEvidence( + upstreamProviderIdentity: string, + upstreamProviderRoute: string, + pricing: ReadonlyMap>, + generatorModels: readonly string[], + scorerModels: readonly string[], +): ProviderContractEvidence { + const generatorSet = new Set(generatorModels); + const scorerSet = new Set(scorerModels); + const models = [...new Set([...generatorModels, ...scorerModels])].sort(); + const modelPriceBounds = models.map((model) => { + const bound = pricing.get(model); + if (bound === undefined) throw new Error(`provider contract pricing missing for ${model}`); + return { + model, + roles: [ + ...(generatorSet.has(model) ? ["generator" as const] : []), + ...(scorerSet.has(model) ? ["scorer" as const] : []), + ], + inputMicrosPerMillionTokens: bound.inputMicrosPerMillionTokens, + outputMicrosPerMillionTokens: bound.outputMicrosPerMillionTokens, + }; + }); + return { + version: 1, + benchmarkProviderIdentity: "openrouter:managed-routing", + upstreamProviderIdentity, + upstreamProviderRoute, + dataCollection: "deny", + zeroDataRetention: true, + allowFallbacks: false, + generatorRequireParameters: false, + scorerRequireParameters: true, + maxPricePinned: true, + maxPriceUnits: "USD per million tokens", + modelPriceBounds, + }; +} + +export function providerContractSha256(contract: ProviderContractEvidence): string { + const canonical = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(canonical); + if (typeof value !== "object" || value === null) return value; + return Object.fromEntries( + Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, canonical(entry)]), + ); + }; + return createHash("sha256").update(JSON.stringify(canonical(contract))).digest("hex"); +} + /** Ground truth distilled from a fixture: the authored target defect's file and line, or * a clean fixture where the correct review is silence. */ export interface GroundTruth { @@ -814,6 +889,7 @@ export interface OpenRouterZdrEndpointsResponse { data: Array<{ model_id: string; provider_name?: string; + tag?: string; status?: number; pricing?: { prompt?: string; completion?: string }; supported_parameters?: string[]; @@ -883,11 +959,21 @@ export function pricingFromZdrCatalog( wantedModels: string[], expectedProvider: string, requiredParametersByModel: ReadonlyMap = new Map(), + expectedProviderRoute = expectedProvider, ): Map { const wanted = new Set(wantedModels); const candidates = new Map>(); for (const endpoint of catalog.data ?? []) { - if (!wanted.has(endpoint.model_id) || endpoint.status !== 0 || endpoint.provider_name !== expectedProvider) continue; + const providerMatches = endpoint.provider_name === expectedProvider; + const routeMatches = expectedProviderRoute === expectedProvider + ? providerMatches + : endpoint.tag === expectedProviderRoute; + if ( + !wanted.has(endpoint.model_id) || endpoint.status !== 0 || + !providerMatches || !routeMatches + ) { + continue; + } const supportedParameters = endpoint.supported_parameters; const requiredParameters = requiredParametersByModel.get(endpoint.model_id) ?? []; if (requiredParameters.length > 0 && diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index b22ab86..3500202 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -46,6 +46,7 @@ import { qualificationCaseRepeats, qualificationRequiredParameters, qualificationProfileDigest, + qualificationProfileDigestMaterial, readPinnedQualificationWorktreeFile, runLiveModels, runQualificationCanariesSequentially, @@ -315,14 +316,16 @@ describe("pair qualification configuration", () => { }); test("derives role-specific provider parameters for every model in a profile", () => { - expect(qualificationRequiredParameters([{ + const pair = { generatorModel: "provider/shared", generatorCascade: ["provider/generator-fallback"], consensus: 2, scorerModel: "provider/shared", scorerCascade: ["provider/scorer-fallback"], - }])).toEqual(new Map([ + }; + expect(qualificationRequiredParameters([pair], "Azure")).toEqual(new Map([ ["provider/shared", [ + "max_completion_tokens", "max_tokens", "reasoning", "reasoning_effort", @@ -337,6 +340,15 @@ describe("pair qualification configuration", () => { "temperature", ]], ["provider/scorer-fallback", [ + "max_completion_tokens", + "reasoning", + "reasoning_effort", + "response_format", + "structured_outputs", + ]], + ])); + expect(qualificationRequiredParameters([pair], "OpenAI")).toEqual(new Map([ + ["provider/shared", [ "max_tokens", "reasoning", "reasoning_effort", @@ -344,6 +356,19 @@ describe("pair qualification configuration", () => { "structured_outputs", "temperature", ]], + ["provider/generator-fallback", [ + "max_tokens", + "reasoning", + "reasoning_effort", + "temperature", + ]], + ["provider/scorer-fallback", [ + "max_tokens", + "reasoning", + "reasoning_effort", + "response_format", + "structured_outputs", + ]], ])); }); @@ -520,6 +545,7 @@ describe("pair qualification configuration", () => { normalizeApiBase("https://openrouter.ai/api/v1"), "openai-compatible", "PinnedProvider", + "pinned/route", "http://127.0.0.1:4321", ); const profilePath = env.POSTIL_QUALIFICATION_CANDIDATE_PROFILE; @@ -527,6 +553,7 @@ describe("pair qualification configuration", () => { expect(env.POSTIL_QUALIFICATION_CAPTURE_API_BASE).toBe("http://127.0.0.1:4321"); expect(await Bun.file(profilePath!).json()).toMatchObject({ upstreamProviderIdentity: "PinnedProvider", + upstreamProviderRoute: "pinned/route", scorerChain: [pair.scorerModel], modelPriceBounds: [ { model: pair.generatorModel, inputMicrosPerMillionTokens: 1_000_000, outputMicrosPerMillionTokens: 2_000_000 }, @@ -667,6 +694,7 @@ console.log(JSON.stringify({ apiFormat: "openai-compatible", costCapUsdDecimal: "1", upstreamProvider: "PinnedProvider", + upstreamProviderRoute: "pinned/route", credentialEnvironment: { POSTIL_API_KEY: "postil-plan-only-fixture" }, })).rejects.toThrow("runtime-shaped qualification spend"); } finally { @@ -733,6 +761,7 @@ process.exit(0); apiFormat: "openai-compatible", costCapUsdDecimal: "55", upstreamProvider: "PinnedProvider", + upstreamProviderRoute: "pinned/route", credentialEnvironment: { POSTIL_API_KEY: "postil-plan-only-fixture" }, })).rejects.toThrow("deliberate preflight failure"); const starts = (await readFile(startsPath, "utf8")).trim().split("\n"); @@ -1324,6 +1353,7 @@ describe("managed admission workflow", () => { expect(workflow).not.toContain("inputs.api_base"); expect(workflow).not.toContain("inputs.api_format"); expect(workflow).not.toContain("POSTIL_BENCH_MODELS"); + expect(workflow).toContain("POSTIL_BENCH_UPSTREAM_PROVIDER_ROUTE: ${{ inputs.upstream_provider_route }}"); const ci = await Bun.file( resolve(import.meta.dir, "..", "..", ".github", "workflows", "ci.yml"), ).text(); @@ -1334,15 +1364,18 @@ describe("managed admission workflow", () => { ).text(); expect(release).toMatch(/validate-tag:\n[\s\S]*?fetch-depth: 0[\s\S]*?bun-version: 1\.3\.14[\s\S]*?bun install --frozen-lockfile[\s\S]*?bun run verify-admission[\s\S]*?\n bench-live:\n/u); expect(release).toMatch(/bench-live:\n\s+needs: validate-tag\n/u); - // The gate must score whatever the binary ships. Restating the id here - // pinned it to one model, so the release that changed the shipped default - // still benchmarked the previous one and passed against its old baseline. + // The gate derives its model from the binary's embedded configuration so + // caller drift cannot benchmark a model absent from the release. expect(release).not.toContain("REVIEW_MODEL:"); expect(release).toContain("OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}"); expect(release).not.toContain("POSTIL_SCORER_EVAL_MODELS:"); expect(release).toContain('POSTIL_SCORER_EVAL_REPEATS: "3"'); expect(release).toContain("POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER: Azure"); + expect(release).toContain("POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/eu"); + expect(release).toContain("POSTIL_BIN: ${{ github.workspace }}/target/release/postil"); + expect(release).not.toContain("Build scorer qualification binary"); expect(release).toContain("bun run scorer-eval --json-out"); + expect(release).toContain("${{ runner.temp }}/scorer-eval-report.json.partial"); expect(release).toMatch( /name: Upload the scorer gate report\n\s+if: always\(\)[\s\S]*if-no-files-found: warn[\s\S]*retention-days: 30/u, ); @@ -1350,6 +1383,9 @@ describe("managed admission workflow", () => { release.indexOf("bun run bench:live --"), ); expect(release).toContain("bun run bench:live --"); + expect(release).toMatch( + /name: Upload the diff-file live report\n\s+if: always\(\)[\s\S]*bench-live-report\.json[\s\S]*retention-days: 30/u, + ); expect(release).toContain("bun run bench:compare --"); expect(release).toMatch(/build:\n\s+needs: \[validate-tag, bench-live\]/u); let checkedReferences = 0; @@ -1456,6 +1492,15 @@ describe("qualification report", () => { }; const profile = { id: qualificationProfileDigest(profileMaterial), ...profileMaterial }; expect(profile.id).toBe("24cd24ba19e6125b6c1b152c77c0860efffdc87c2f3db3bc9fb6fb70768e35ce"); + const routeBoundMaterial = { + ...profileMaterial, + upstreamProviderRoute: "provider/route", + }; + expect(qualificationProfileDigest(routeBoundMaterial)).not.toBe(profile.id); + expect(qualificationProfileDigestMaterial(routeBoundMaterial)).toMatchObject({ + upstreamProviderIdentity: "test-provider", + upstreamProviderRoute: "provider/route", + }); const vector = await Bun.file( resolve(import.meta.dir, "..", "admission-manifest-candidate-vector.json"), ).json(); @@ -1476,7 +1521,7 @@ describe("qualification report", () => { test("prints attributable metrics, hashes, provider, and bounded costs", () => { const cost = 0.123456; const report: LiveModelsReport = { - schemaVersion: 3, + schemaVersion: 4, generatedAt: "2026-07-11T00:00:00.000Z", qualificationSourceSha: "9".repeat(40), cliVersion: "postil 0.6.1", @@ -1485,6 +1530,7 @@ describe("qualification report", () => { providerEndpointIdentity: "https://example.test:443/v1", upstreamProviderPinned: true, upstreamProviderIdentity: "PinnedProvider", + upstreamProviderRoute: "pinned/route", fixtureHash: "a".repeat(64), reviewContractHash: "b".repeat(64), evaluatorContractHash: "f".repeat(64), @@ -1569,6 +1615,18 @@ describe("qualification report", () => { report.privateEvidenceSha256 = privateEvidenceSha256(privateBundle); expect(parseLiveModelsReport(report)).toBe(report); expect(() => verifyPrivateEvidenceBundle(privateBundle, report)).not.toThrow(); + const schemaThree = structuredClone(report) as unknown as Record; + schemaThree.schemaVersion = 3; + delete schemaThree.upstreamProviderRoute; + expect(parseLiveModelsReport(schemaThree)).toMatchObject({ + schemaVersion: 4, + upstreamProviderRoute: "PinnedProvider", + }); + const contaminatedSchemaThree = structuredClone(report) as unknown as Record; + contaminatedSchemaThree.schemaVersion = 3; + expect(() => parseLiveModelsReport(contaminatedSchemaThree)).toThrow( + "schema-3 report must not contain upstreamProviderRoute", + ); const legacy = structuredClone(report) as unknown as Record; delete legacy.schemaVersion; expect(() => parseLiveModelsReport(legacy)).toThrow( diff --git a/bench/src/livemodels.ts b/bench/src/livemodels.ts index bf34591..9775227 100644 --- a/bench/src/livemodels.ts +++ b/bench/src/livemodels.ts @@ -99,7 +99,8 @@ const MAX_QUALIFICATION_SOURCE_BYTES = 16 * 1024 * 1024; const MANAGED_OPENROUTER_API_BASE = "https://openrouter.ai:443/api/v1"; const PLAN_ONLY_CAPTURE_API_BASE = "http://127.0.0.1:9"; export const MANAGED_OPENROUTER_PROVIDER_IDENTITY = "openrouter:managed-routing"; -export const LIVE_MODELS_REPORT_SCHEMA_VERSION = 3; +export const LIVE_MODELS_REPORT_SCHEMA_VERSION = 4; +export const LEGACY_LIVE_MODELS_REPORT_SCHEMA_VERSION = 3; export const LIVE_MODELS_PRIVATE_EVIDENCE_SCHEMA_VERSION = 1; export const PROMPT_INJECTION_CLEAN_ADMISSION_CASE_ID = "prompt-injection-comment-clean"; export const PROMPT_INJECTION_CLEAN_ADMISSION_REPEATS = 3; @@ -219,6 +220,8 @@ export interface LiveModelsOptions { apiFormat?: "openai-compatible" | "anthropic"; /** Exact OpenRouter upstream provider name, pinned without fallback. */ upstreamProvider: string; + /** Exact OpenRouter endpoint route. Defaults to the response provider identity. */ + upstreamProviderRoute?: string; /** Root directory for per-case run dirs. Defaults to bench/.runs/live-models. */ rootDir?: string; /** Per-case timeout (default DEFAULT_TIMEOUT_MS). */ @@ -243,6 +246,7 @@ export interface LiveModelsReport { providerEndpointIdentity: string; upstreamProviderPinned: true; upstreamProviderIdentity: string; + upstreamProviderRoute: string; fixtureHash: string; reviewContractHash: string; evaluatorContractHash: string; @@ -439,7 +443,7 @@ export function summarizeAttributionEvaluator(result: { const LIVE_MODELS_REPORT_FIELDS = new Set([ "schemaVersion", "generatedAt", "qualificationSourceSha", "cliVersion", "apiBase", "apiFormat", - "providerEndpointIdentity", "upstreamProviderPinned", "upstreamProviderIdentity", "fixtureHash", + "providerEndpointIdentity", "upstreamProviderPinned", "upstreamProviderIdentity", "upstreamProviderRoute", "fixtureHash", "reviewContractHash", "evaluatorContractHash", "evaluatorRuntimeIdentity", "configHash", "cliBinaryHash", "evidenceHash", "privateEvidenceSha256", "attributionContractHash", "attributionBankHash", "attributionEvaluators", "hostedOperationCostCapMicros", "repeats", "profiles", "manifestCandidate", @@ -462,23 +466,42 @@ export function parseLiveModelsReport(value: unknown): LiveModelsReport { if (!("schemaVersion" in value)) { throw new Error("live-models report schemaVersion is required; legacy unversioned reports are not accepted"); } - if (value.schemaVersion !== LIVE_MODELS_REPORT_SCHEMA_VERSION) { + if ( + value.schemaVersion !== LIVE_MODELS_REPORT_SCHEMA_VERSION && + value.schemaVersion !== LEGACY_LIVE_MODELS_REPORT_SCHEMA_VERSION + ) { throw new Error(`unsupported live-models report schemaVersion ${String(value.schemaVersion)}`); } - const unknown = Object.keys(value).filter((field) => !LIVE_MODELS_REPORT_FIELDS.has(field)); + if ( + value.schemaVersion === LEGACY_LIVE_MODELS_REPORT_SCHEMA_VERSION && + "upstreamProviderRoute" in value + ) { + throw new Error("live-models schema-3 report must not contain upstreamProviderRoute"); + } + const report = value.schemaVersion === LEGACY_LIVE_MODELS_REPORT_SCHEMA_VERSION + ? { + ...value, + schemaVersion: LIVE_MODELS_REPORT_SCHEMA_VERSION, + upstreamProviderRoute: value.upstreamProviderIdentity, + } + : value; + const unknown = Object.keys(report).filter((field) => !LIVE_MODELS_REPORT_FIELDS.has(field)); if (unknown.length > 0) throw new Error(`live-models report has unknown field ${unknown[0]}`); - const missing = LIVE_MODELS_REQUIRED_REPORT_FIELDS.filter((field) => !(field in value)); + const missing = LIVE_MODELS_REQUIRED_REPORT_FIELDS.filter((field) => !(field in report)); if (missing.length > 0) throw new Error(`live-models report is missing field ${missing[0]}`); - if (!Array.isArray(value.cases) || !Array.isArray(value.models) || - !Array.isArray(value.modelAggregates) || !Array.isArray(value.profiles) || - !Array.isArray(value.attributionEvaluators)) { + if (typeof report.upstreamProviderRoute !== "string" || report.upstreamProviderRoute.trim() === "") { + throw new Error("live-models report upstreamProviderRoute must be a nonempty string"); + } + if (!Array.isArray(report.cases) || !Array.isArray(report.models) || + !Array.isArray(report.modelAggregates) || !Array.isArray(report.profiles) || + !Array.isArray(report.attributionEvaluators)) { throw new Error("live-models report collection fields must be arrays"); } - if (typeof value.privateEvidenceSha256 !== "string" || !isSha256(value.privateEvidenceSha256)) { + if (typeof report.privateEvidenceSha256 !== "string" || !isSha256(report.privateEvidenceSha256)) { throw new Error("live-models report privateEvidenceSha256 must be a SHA-256 digest"); } - assertPublicReportValue(value, "report"); - return value as unknown as LiveModelsReport; + assertPublicReportValue(report, "report"); + return report as unknown as LiveModelsReport; } function assertPublicReportValue(value: unknown, path: string): void { @@ -617,6 +640,7 @@ export interface QualificationProfile { apiFormat: "openai-compatible" | "anthropic"; benchmarkProviderIdentity: string | null; upstreamProviderIdentity: string; + upstreamProviderRoute?: string; generatorModels: string[]; consensus: number; scorerModels: string[]; @@ -645,6 +669,7 @@ export interface AdmissionManifestCandidate { apiBase: string; benchmarkProviderIdentity: string | null; upstreamProviderIdentity: string; + upstreamProviderRoute?: string; generatorChain: string[]; consensus: number; scorerChain: string[]; @@ -665,6 +690,7 @@ export interface QualificationProfileDigestMaterial { modelDefaultsSha256: string; benchmarkProviderIdentity: string | null; upstreamProviderIdentity: string; + upstreamProviderRoute?: string; apiBase: string; apiFormat: "openai-compatible" | "anthropic"; generatorChain: string[]; @@ -725,6 +751,10 @@ export async function runLiveModels( if (upstreamProvider.length === 0) { throw new Error("live qualification requires an exact pinned upstream provider identity"); } + const upstreamProviderRoute = (options.upstreamProviderRoute ?? upstreamProvider).trim(); + if (upstreamProviderRoute.length === 0) { + throw new Error("live qualification requires an exact pinned upstream provider route"); + } const rootDir = options.rootDir ?? resolve(import.meta.dir, "..", ".runs", "live-models"); const suppliedPricing = options.pricing; return withImmutableQualificationBinary(options.binary, rootDir, async (immutableBinary) => { @@ -752,7 +782,8 @@ export async function runLiveModels( apiFormat, models, upstreamProvider, - qualificationRequiredParameters(pairs), + qualificationRequiredParameters(pairs, upstreamProvider), + upstreamProviderRoute, )); assertPricingProviderIdentity(pricing, models, upstreamProvider); const attributionGovernor = new AttributionGovernor( @@ -771,6 +802,7 @@ export async function runLiveModels( apiFormat, costCapUsdDecimal, upstreamProvider, + upstreamProviderRoute, }); const keyName = resolveApiKeyName(); const completionApiKey = keyName === undefined ? undefined : process.env[keyName]; @@ -861,6 +893,7 @@ export async function runLiveModels( apiBase, apiFormat, upstreamProvider, + upstreamProviderRoute, requestProxy.apiBase, ); const result = await qualifyAttributionEvaluator({ @@ -911,6 +944,7 @@ export async function runLiveModels( repeats, modelPriceBounds: modelPriceBoundsFor(pair, pricing), upstreamProviderIdentity: upstreamProvider, + upstreamProviderRoute, })); const exactGeneratorCosts = results .map((result) => result.costProviderDecimal) @@ -955,6 +989,7 @@ export async function runLiveModels( providerEndpointIdentity: identity, upstreamProviderPinned: true, upstreamProviderIdentity: upstreamProvider, + upstreamProviderRoute, fixtureHash, reviewContractHash, evaluatorContractHash, @@ -1000,6 +1035,7 @@ export async function runLiveModels( providerEndpointIdentity: identity, upstreamProviderPinned: true, upstreamProviderIdentity: upstreamProvider, + upstreamProviderRoute, fixtureHash, reviewContractHash, evaluatorContractHash, @@ -1286,7 +1322,14 @@ async function runLiveModelCase( if (candidateProfilePath !== undefined) { await writeFile( candidateProfilePath, - JSON.stringify(qualificationCandidateDocument(pair, pricing, apiBase, apiFormat, options.upstreamProvider)), + JSON.stringify(qualificationCandidateDocument( + pair, + pricing, + apiBase, + apiFormat, + options.upstreamProvider, + options.upstreamProviderRoute, + )), { mode: 0o600 }, ); } @@ -1453,12 +1496,14 @@ export function qualificationCandidateDocument( apiBase: string, apiFormat: "openai-compatible" | "anthropic", upstreamProvider: string, + upstreamProviderRoute = upstreamProvider, ) { return { benchmarkProviderIdentity: benchmarkProviderIdentityFor(apiBase, apiFormat), apiBase, apiFormat, upstreamProviderIdentity: upstreamProvider, + upstreamProviderRoute, generatorChain: qualificationGeneratorModels(pair), consensus: pair.consensus, scorerChain: qualificationScorerModels(pair), @@ -1473,6 +1518,7 @@ export async function prepareAttributionEvaluatorEnvironment( apiBase: string, apiFormat: "openai-compatible" | "anthropic", upstreamProvider: string, + upstreamProviderRoute: string, qualificationCaptureApiBase: string, ): Promise { const homeDir = join(pairRoot, "home"); @@ -1482,7 +1528,14 @@ export async function prepareAttributionEvaluatorEnvironment( const candidateProfilePath = join(pairRoot, "qualification-candidate.json"); await writeFile( candidateProfilePath, - JSON.stringify(qualificationCandidateDocument(pair, pricing, apiBase, apiFormat, upstreamProvider)), + JSON.stringify(qualificationCandidateDocument( + pair, + pricing, + apiBase, + apiFormat, + upstreamProvider, + upstreamProviderRoute, + )), { mode: 0o600 }, ); return liveEnv( @@ -1508,6 +1561,7 @@ export async function assertRuntimeShapedQualificationPreflight(args: { apiFormat: "openai-compatible" | "anthropic"; costCapUsdDecimal: string; upstreamProvider: string; + upstreamProviderRoute: string; credentialEnvironment?: NodeJS.ProcessEnv; }): Promise { let projectedMicros = 0n; @@ -1540,7 +1594,14 @@ export async function assertRuntimeShapedQualificationPreflight(args: { const profilePath = join(runDir, "qualification-candidate.json"); await writeFile( profilePath, - JSON.stringify(qualificationCandidateDocument(pair, args.pricing, args.apiBase, args.apiFormat, args.upstreamProvider)), + JSON.stringify(qualificationCandidateDocument( + pair, + args.pricing, + args.apiBase, + args.apiFormat, + args.upstreamProvider, + args.upstreamProviderRoute, + )), { mode: 0o600 }, ); const github = await startMockGithub(c); @@ -2024,6 +2085,7 @@ function qualificationProfileEvidence(args: Omit< apiFormat: args.apiFormat, benchmarkProviderIdentity: benchmarkProviderIdentityFor(args.apiBase, args.apiFormat), upstreamProviderIdentity: args.upstreamProviderIdentity, + upstreamProviderRoute: args.upstreamProviderRoute, generatorModels, consensus, scorerModels: qualificationScorerModels(args.pair), @@ -2056,6 +2118,9 @@ export function qualificationProfileDigestMaterial( modelDefaultsSha256: profile.modelDefaultsSha256, benchmarkProviderIdentity: profile.benchmarkProviderIdentity, upstreamProviderIdentity: profile.upstreamProviderIdentity, + ...(profile.upstreamProviderRoute === undefined + ? {} + : { upstreamProviderRoute: profile.upstreamProviderRoute }), apiBase: profile.apiBase, apiFormat: profile.apiFormat, generatorChain: profile.generatorModels, @@ -2369,6 +2434,7 @@ export async function fetchPricing( models: string[], upstreamProvider: string, requiredParametersByModel: ReadonlyMap = new Map(), + upstreamProviderRoute = upstreamProvider, ): Promise> { const managedOpenRouter = benchmarkProviderIdentityFor(apiBase, apiFormat) !== null; const url = `${apiBase.replace(/\/$/, "")}/${managedOpenRouter ? "endpoints/zdr" : "models"}`; @@ -2395,13 +2461,28 @@ export async function fetchPricing( models, upstreamProvider, requiredParametersByModel, + upstreamProviderRoute, ) : pricingFromCatalog(catalog as OpenRouterModelsResponse, models); } export function qualificationRequiredParameters( pairs: QualificationPair[], + upstreamProvider = "Azure", ): ReadonlyMap { + const generatorParameters = [ + "max_tokens", + "reasoning", + "reasoning_effort", + "temperature", + ] as const; + const scorerParameters = [ + upstreamProvider === "OpenAI" ? "max_tokens" : "max_completion_tokens", + "reasoning", + "reasoning_effort", + "response_format", + "structured_outputs", + ] as const; const parameters = new Map>(); const add = (model: string, required: readonly string[]): void => { const modelParameters = parameters.get(model) ?? new Set(); @@ -2410,17 +2491,10 @@ export function qualificationRequiredParameters( }; for (const pair of pairs) { for (const model of qualificationGeneratorModels(pair)) { - add(model, ["max_tokens", "reasoning", "reasoning_effort", "temperature"]); + add(model, generatorParameters); } for (const model of qualificationScorerModels(pair)) { - add(model, [ - "max_tokens", - "reasoning", - "reasoning_effort", - "response_format", - "structured_outputs", - "temperature", - ]); + add(model, scorerParameters); } } return new Map([...parameters].map(([model, required]) => [model, [...required].sort()])); diff --git a/bench/src/run.test.ts b/bench/src/run.test.ts index 3dbaa30..db8db9a 100644 --- a/bench/src/run.test.ts +++ b/bench/src/run.test.ts @@ -19,6 +19,7 @@ import { invalidateExplicitOutputs, parseLiveModelsFailureReport, prepareExplicitOutputs, + qualificationProviderInputs, selectLiveScreeningCases, shippedDefaultModel, validateScreeningEnvironment, @@ -33,6 +34,21 @@ import { cases } from "../fixtures/cases"; const temporaryDirectories: string[] = []; describe("diff-file live screening selection", () => { + test("normalizes provider inputs once for execution and evidence", () => { + expect(qualificationProviderInputs([], { + POSTIL_BENCH_UPSTREAM_PROVIDER: " Azure ", + POSTIL_BENCH_UPSTREAM_PROVIDER_ROUTE: " azure/eu ", + })).toEqual({ upstreamProvider: "Azure", upstreamProviderRoute: "azure/eu" }); + expect(qualificationProviderInputs(["--upstream-provider", " Azure "])).toEqual({ + upstreamProvider: "Azure", + upstreamProviderRoute: "Azure", + }); + expect(() => qualificationProviderInputs([], { + POSTIL_BENCH_UPSTREAM_PROVIDER: "Azure", + POSTIL_BENCH_UPSTREAM_PROVIDER_ROUTE: " ", + })).toThrow("nonempty upstream provider route"); + }); + test("preserves requested fixture order and leaves the full corpus unchanged by default", () => { expect(selectLiveScreeningCases(cases, []).map((entry) => entry.id)).toEqual( cases.map((entry) => entry.id), @@ -107,7 +123,7 @@ async function temporaryDirectory(): Promise { function emptyReport(privateEvidenceDigest: string): LiveModelsReport { return { - schemaVersion: 3, + schemaVersion: 4, generatedAt: "2026-07-16T00:00:00.000Z", qualificationSourceSha: "9".repeat(40), cliVersion: "postil 0.6.4", @@ -116,6 +132,7 @@ function emptyReport(privateEvidenceDigest: string): LiveModelsReport { providerEndpointIdentity: "openrouter:managed-routing", upstreamProviderPinned: true, upstreamProviderIdentity: "PinnedProvider", + upstreamProviderRoute: "pinned/route", fixtureHash: "a".repeat(64), reviewContractHash: "b".repeat(64), evaluatorContractHash: "c".repeat(64), @@ -169,6 +186,7 @@ describe("benchmark output lifecycle", () => { qualificationSourceSha: "9".repeat(40), pairs: [{ generatorModel: "deepseek/deepseek-v4-pro", scorerModel: "z-ai/glm-5.2" }], upstreamProvider: "PublicProvider", + upstreamProviderRoute: "public/route", }, ); expect(parseLiveModelsFailureReport(report)).toBe(report); @@ -183,6 +201,7 @@ describe("benchmark output lifecycle", () => { }], providerEndpointIdentity: "openrouter:managed-routing", upstreamProviderIdentity: "PublicProvider", + upstreamProviderRoute: "public/route", process: { category: "provider-http-503", exitCode: 1, signal: null, killed: false, phase: "attribution", providerAttemptCount: 2, identityPresent: true, identityMatched: true, @@ -211,6 +230,7 @@ describe("benchmark output lifecycle", () => { qualificationSourceSha: "8".repeat(40), pairs: [{ generatorModel: "deepseek/deepseek-v4-pro", scorerModel: "z-ai/glm-5.2" }], upstreamProvider: "PublicProvider", + upstreamProviderRoute: "public/route", }, ); expect(report.process).toEqual({ @@ -238,6 +258,7 @@ describe("benchmark output lifecycle", () => { qualificationSourceSha: "8".repeat(40), pairs: [{ generatorModel: "deepseek/deepseek-v4-pro", scorerModel: "z-ai/glm-5.2" }], upstreamProvider: "PublicProvider", + upstreamProviderRoute: "public/route", }, ); expect(report.process).toEqual({ @@ -265,6 +286,7 @@ describe("benchmark output lifecycle", () => { qualificationSourceSha: "7".repeat(40), pairs: [{ generatorModel: "deepseek/deepseek-v4-pro", scorerModel: "z-ai/glm-5.2" }], upstreamProvider: "PublicProvider", + upstreamProviderRoute: "public/route", }, ); expect(report.process).toEqual({ @@ -288,6 +310,7 @@ describe("benchmark output lifecycle", () => { profiles: [{ id: "pair", generatorModels: ["generator"], consensus: 1, scorerModels: ["scorer"] }], providerEndpointIdentity: "openrouter:managed-routing", upstreamProviderIdentity: "PublicProvider", + upstreamProviderRoute: "public/route", process: { category: "provider-unclassified", exitCode: 1, signal: null, killed: false, phase: "attribution", providerAttemptCount: 1, diff --git a/bench/src/run.ts b/bench/src/run.ts index 446d444..ef70e79 100644 --- a/bench/src/run.ts +++ b/bench/src/run.ts @@ -113,6 +113,7 @@ export interface LiveModelsFailureReport { }>; providerEndpointIdentity: typeof MANAGED_OPENROUTER_PROVIDER_IDENTITY; upstreamProviderIdentity: string; + upstreamProviderRoute: string; process: { category: LiveModelsFailureCategory; exitCode: number | null; @@ -133,6 +134,26 @@ function flagValue(args: string[], flag: string): string | undefined { return value?.startsWith("--") === true ? undefined : value; } +export function qualificationProviderInputs( + args: string[], + environment: NodeJS.ProcessEnv = process.env, +): { upstreamProvider: string; upstreamProviderRoute: string } { + const upstreamProvider = ( + environment.POSTIL_BENCH_UPSTREAM_PROVIDER ?? flagValue(args, "--upstream-provider") ?? "" + ).trim(); + if (!upstreamProvider) { + throw new Error("live-models admission needs POSTIL_BENCH_UPSTREAM_PROVIDER or --upstream-provider"); + } + const upstreamProviderRoute = ( + environment.POSTIL_BENCH_UPSTREAM_PROVIDER_ROUTE ?? + flagValue(args, "--upstream-provider-route") ?? upstreamProvider + ).trim(); + if (!upstreamProviderRoute) { + throw new Error("live-models admission needs a nonempty upstream provider route"); + } + return { upstreamProvider, upstreamProviderRoute }; +} + function repeatedFlagValues(args: string[], flag: string): string[] { const values: string[] = []; for (let index = 0; index < args.length; index += 1) { @@ -221,10 +242,8 @@ function liveConcurrency(args: string[]): number { } /** The generator the binary under test actually ships, read from the same - * `config.toml` the build bakes in. Restating the id in a caller lets the - * benchmark certify a model the release does not contain: the release gate - * carried `REVIEW_MODEL: z-ai/glm-5.2` in workflow YAML and kept scoring that - * model after the shipped default moved, passing against the old baseline. */ + * `config.toml` the build embeds. Resolving this value from the source prevents + * caller configuration drift from benchmarking a model absent from the release. */ export function shippedDefaultModel( configPath = resolve(import.meta.dir, "..", "..", "config.toml"), ): string | undefined { @@ -282,10 +301,7 @@ async function main() { const repeatsRaw = process.env.POSTIL_BENCH_REPEATS ?? flagValue(args, "--repeats"); const apiFormat = qualificationApiFormat(process.env.POSTIL_API_FORMAT); const pricingFile = process.env.POSTIL_BENCH_PRICING_FILE ?? flagValue(args, "--pricing-file"); - const upstreamProvider = process.env.POSTIL_BENCH_UPSTREAM_PROVIDER ?? flagValue(args, "--upstream-provider"); - if (!upstreamProvider?.trim()) { - throw new Error("live-models admission needs POSTIL_BENCH_UPSTREAM_PROVIDER or --upstream-provider"); - } + const { upstreamProvider, upstreamProviderRoute } = qualificationProviderInputs(args); const qualificationSourceSha = await resolveQualificationSourceSha( resolve(import.meta.dir, "..", ".."), ); @@ -299,6 +315,7 @@ async function main() { apiBase: process.env.POSTIL_API_BASE, apiFormat, upstreamProvider, + upstreamProviderRoute, pricing: pricingFile === undefined ? undefined : await pricingFromFile(pricingFile), concurrency, costCapUsd: costCapRaw, @@ -309,6 +326,7 @@ async function main() { qualificationSourceSha, pairs, upstreamProvider, + upstreamProviderRoute, }); await writeLiveModelsReport(jsonOut, JSON.stringify(failureReport, null, 2)); throw error; @@ -401,6 +419,7 @@ export async function createLiveModelsFailureReport( qualificationSourceSha: string; pairs: QualificationPair[]; upstreamProvider: string; + upstreamProviderRoute: string; }, ): Promise { const failure = fixedLiveModelsFailure(error); @@ -415,6 +434,7 @@ export async function createLiveModelsFailureReport( })), providerEndpointIdentity: MANAGED_OPENROUTER_PROVIDER_IDENTITY, upstreamProviderIdentity: options.upstreamProvider, + upstreamProviderRoute: options.upstreamProviderRoute, process: failure, })); } @@ -423,13 +443,14 @@ export function parseLiveModelsFailureReport(value: unknown): LiveModelsFailureR if (!isRecord(value)) throw new Error("invalid live-models failure artifact"); assertExactKeys(value, [ "artifactType", "qualificationSourceSha", "profiles", "providerEndpointIdentity", - "upstreamProviderIdentity", "process", + "upstreamProviderIdentity", "upstreamProviderRoute", "process", ]); if (value.artifactType !== "live-models-failure" || typeof value.qualificationSourceSha !== "string" || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(value.qualificationSourceSha) || value.providerEndpointIdentity !== MANAGED_OPENROUTER_PROVIDER_IDENTITY || typeof value.upstreamProviderIdentity !== "string" || value.upstreamProviderIdentity.trim() === "" || + typeof value.upstreamProviderRoute !== "string" || value.upstreamProviderRoute.trim() === "" || !Array.isArray(value.profiles) || value.profiles.length === 0 || !isRecord(value.process)) { throw new Error("invalid live-models failure artifact"); diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index 71cbd04..5943490 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -16,15 +16,20 @@ import { FALSE_FINDING_CASES, GENERATOR_MODEL, SCORER_CASE_EXEC_TIMEOUT_MS, + SCORER_CASE_HARNESS_ALLOWANCE_MS, SCORER_MAX_CASE_MS, + SCORER_REASON_SCHEMA_PATTERN, TRUE_FINDING_CASES, aggregate, + assertCleanScorerEvaluatorStatus, + assertScorerEvaluatorFileMatches, assertQualificationPreflight, falseFinding, falseFindingFromSourceRequest, firstAddedLineForPath, finalizeScorerEvalReport, formatReport, + generatorRequestMismatchCodes, isAdmissionFatalStructuralResult, isValidReason, isolatedEnv, @@ -42,21 +47,53 @@ import { safeSegment, scorerCasePasses, scorerCostProviderDecimal, + scorerEvalRootDir, + scorerEvaluatorDigest, providerCostDecimalFromResponse, scorerCheckpointPath, + strictRequestMismatchCodes, + scorerProxyRequestPhase, selectEvalCases, startScorerProxy, scorerStructuralFailureReason, + scorerQualificationModels, + scorerQualificationRequiredParameters, trueFinding, writeScorerEvalCheckpoint, + writeScorerEvalSetupFailureArtifact, type ScorerEvalCase, type ScorerEvalReport, + type ScorerProxyExpectedContract, } from "./scorer-eval"; const fixtures = fixtureInputs.map((input) => benchmarkCase.parse(input)); const BOUNDED_SCORER_TARGET_PATH = 'src/ui/copy"quoted.ts'; const TEST_SCORER_MODEL = "z-ai/glm-5.2"; +function scorerReportContract(): Pick< + ScorerEvalReport, + "evaluatorSha256" | "providerContractSha256" | "providerContract" +> { + return { + evaluatorSha256: "c".repeat(64), + providerContractSha256: "d".repeat(64), + providerContract: { + version: 1, + benchmarkProviderIdentity: "openrouter:managed-routing", + upstreamProviderIdentity: "test-provider", + upstreamProviderRoute: "test-provider/route", + dataCollection: "deny", + zeroDataRetention: true, + allowFallbacks: false, + generatorRequireParameters: false, + scorerRequireParameters: true, + maxPricePinned: true, + maxPriceUnits: "USD per million tokens", + modelPriceBounds: [], + }, + }; +} + function postilBinaryPath(): string { const cargoTarget = process.env.CARGO_TARGET_DIR; return resolve( @@ -104,9 +141,9 @@ function boundedScorerFixture() { "", ].join("\n"); const diff = [ - ...Array.from({ length: 3 }, (_, index) => ordinaryFile(index)), + ...Array.from({ length: 15 }, (_, index) => ordinaryFile(index)), target, - ...Array.from({ length: 2 }, (_, index) => ordinaryFile(index + 3)), + ...Array.from({ length: 15 }, (_, index) => ordinaryFile(index + 15)), ].join(""); const base = fixture("huge-low-signal-clean"); return benchmarkCase.parse({ @@ -207,6 +244,129 @@ function requestBody(req: IncomingMessage): Promise { }); } +const TEST_PROXY_PRICING = { + providerIdentity: "Azure", + promptUsdPerToken: 0.00000022, + completionUsdPerToken: 0.00000132, + inputMicrosPerMillionTokens: 220_000, + outputMicrosPerMillionTokens: 1_320_000, +}; + +function proxyContract( + model = "scorer/model", + providerIdentity = "Azure", + providerRoute = "azure/eu", +): ScorerProxyExpectedContract { + return { model, providerIdentity, providerRoute, pricing: TEST_PROXY_PRICING }; +} + +function strictProvider() { + return { + data_collection: "deny", + zdr: true, + order: ["azure/eu"], + allow_fallbacks: false, + require_parameters: true, + max_price: { prompt: 0.22, completion: 1.32 }, + }; +} + +function generatorRequest() { + const { require_parameters: _requireParameters, ...provider } = strictProvider(); + return { + model: GENERATOR_MODEL, + max_tokens: 4_000, + temperature: 0.1, + reasoning: { effort: "low" }, + provider, + messages: [ + { role: "system", content: "You are Postil's low-noise code reviewer." }, + { role: "user", content: "Review this change." }, + ], + }; +} + +function scorerRequest(model = "scorer/model") { + return { + model, + max_completion_tokens: 400, + reasoning: { effort: "low", exclude: true }, + provider: strictProvider(), + response_format: { + type: "json_schema", + json_schema: { + name: "postil_finding_scores", + strict: true, + schema: { + type: "object", + properties: { + scores: { + type: "array", + minItems: 1, + maxItems: 1, + items: { + type: "object", + properties: { + confidence: { type: "number", minimum: 0, maximum: 1 }, + kind: { + type: "string", + enum: ["risk", "humanEscalation", "guardrail", "uncertainty", "contentPolicy"], + }, + reason: { + type: "string", + minLength: 1, + maxLength: 240, + pattern: "^(?:[.!?。！？]|[^\\s\\u0000-\\u001F\\u007F-\\u009F\\u2028\\u2029](?:[^\\u0000-\\u001F\\u007F-\\u009F\\u2028\\u2029]*[.!?。！？]))$", + }, + }, + required: ["confidence", "kind", "reason"], + additionalProperties: false, + }, + }, + }, + required: ["scores"], + additionalProperties: false, + }, + }, + }, + messages: [ + { role: "system", content: "You are Postil's independent second-model scorer." }, + { role: "user", content: "Score this finding." }, + ], + }; +} + +function adjudicationRequest(model = "scorer/model") { + return { + model, + max_completion_tokens: 8_000, + reasoning: { effort: "low", exclude: true }, + provider: strictProvider(), + messages: [ + { role: "system", content: "You are Postil's single finding adjudicator." }, + { role: "user", content: "Adjudicate this finding." }, + ], + }; +} + +function genericOpenAiCompatibleRequest( + phase: "adjudication" | "scorer", + model = "scorer/model", +) { + return { + model, + max_tokens: 8_000, + temperature: 0, + reasoning: { effort: "low" }, + messages: [{ + role: "system", + content: phase === "adjudication" + ? "You are Postil's single finding adjudicator." + : "You are Postil's independent second-model scorer.", + }], + }; +} + function adjudicationResponse(body: string): string | null { let request: { messages?: Array<{ role?: string; content?: string }>; @@ -270,7 +430,46 @@ describe("parseRepeatCount", () => { }); }); +describe("scorer run artifacts", () => { + test("binds qualification to clean evaluator source bytes", () => { + expect(() => assertCleanScorerEvaluatorStatus("")).not.toThrow(); + expect(() => assertCleanScorerEvaluatorStatus(" M bench/src/scorer-eval.ts\n")).toThrow( + "sources differ from HEAD", + ); + expect(() => assertScorerEvaluatorFileMatches(Buffer.from("same"), Buffer.from("same"))) + .not.toThrow(); + expect(() => assertScorerEvaluatorFileMatches(Buffer.from("dirty"), Buffer.from("HEAD"))) + .toThrow("sources differ from HEAD"); + const first = scorerEvaluatorDigest([ + { path: "bench/src/b.ts", contents: Buffer.from("second") }, + { path: "bench/src/a.ts", contents: Buffer.from("first") }, + ]); + expect(first).toBe(scorerEvaluatorDigest([ + { path: "bench/src/a.ts", contents: Buffer.from("first") }, + { path: "bench/src/b.ts", contents: Buffer.from("second") }, + ])); + expect(first).not.toBe(scorerEvaluatorDigest([ + { path: "bench/src/a.ts", contents: Buffer.from("changed") }, + { path: "bench/src/b.ts", contents: Buffer.from("second") }, + ])); + }); + + test("supports a unique retained run root", () => { + expect(scorerEvalRootDir(" ./retained-scorer-run ")).toBe( + resolve("./retained-scorer-run"), + ); + expect(scorerEvalRootDir(" ")).toBe( + resolve(import.meta.dir, "..", ".runs", "scorer-eval"), + ); + }); +}); + describe("scorer calibration findings", () => { + test("preserves the cross-language scorer reason regex exactly", () => { + expect(SCORER_REASON_SCHEMA_PATTERN).toHaveLength(112); + expect(SCORER_REASON_SCHEMA_PATTERN.charCodeAt(8)).toBe(12_290); + }); + test("selects fixed true and false fixture sets for comparable runs", () => { const selected = selectEvalCases(fixtures); expect(selected.map((c) => c.case.id)).toEqual([...TRUE_FINDING_CASES, ...FALSE_FINDING_CASES]); @@ -281,6 +480,24 @@ describe("scorer calibration findings", () => { expect(selected.every((c) => c.scenario === "falseFinding" || c.case.modelOutput.findings.length > 0)).toBe(true); }); + test("expands only bounded qualification fixtures beyond the five-batch cap", () => { + const selected = selectEvalCases(fixtures); + const bounded = selected.filter((entry) => + entry.case.admission.expectedCoverage === "bounded" + ); + expect(bounded).toHaveLength(2); + for (const entry of bounded) { + const files = parseUnifiedDiffFiles(entry.case.diff); + expect(files.filter((file) => + file.path.startsWith("src/scorer-qualification-padding/") + )).toHaveLength(12); + expect(files.some((file) => file.path === entry.case.primaryChange?.path)).toBe(true); + } + expect(selected.find((entry) => entry.case.id === "billing-double-charge")?.case).toBe( + fixture("billing-double-charge"), + ); + }); + test("true findings reuse recorded fixture evidence but normalize scorer target labels", () => { const finding = trueFinding(fixture("billing-double-charge")); expect(finding).toMatchObject({ @@ -355,6 +572,85 @@ describe("scorer calibration findings", () => { }); describe("scorer proxy and isolated runtime", () => { + test("routes only trusted model and request-shape combinations", () => { + const azureContract = proxyContract(); + expect(scorerProxyRequestPhase({ model: GENERATOR_MODEL }, azureContract)).toBe("generator"); + expect(scorerProxyRequestPhase(scorerRequest(), azureContract)).toBe("scorer"); + expect(scorerProxyRequestPhase(adjudicationRequest(), azureContract)).toBe("adjudication"); + + const { max_completion_tokens: _scorerLimit, ...scorerRest } = scorerRequest(); + const { max_completion_tokens: _adjudicationLimit, ...adjudicationRest } = adjudicationRequest(); + const openAiContract = proxyContract("scorer/model", "OpenAI", "openai"); + expect(scorerProxyRequestPhase({ + ...scorerRest, + max_tokens: 400, + provider: { ...strictProvider(), order: ["openai"] }, + }, openAiContract)).toBe("scorer"); + expect(scorerProxyRequestPhase({ + ...adjudicationRest, + max_tokens: 8_000, + provider: { ...strictProvider(), order: ["openai"] }, + }, openAiContract)).toBe("adjudication"); + expect(scorerProxyRequestPhase(genericOpenAiCompatibleRequest("scorer"))).toBe("scorer"); + expect(scorerProxyRequestPhase(genericOpenAiCompatibleRequest("adjudication"))).toBe("adjudication"); + expect( + scorerProxyRequestPhase(genericOpenAiCompatibleRequest("scorer"), azureContract), + ).toBeNull(); + expect(scorerProxyRequestPhase(scorerRequest("other/model"), azureContract)).toBeNull(); + expect(scorerProxyRequestPhase({ + ...scorerRequest(), + max_tokens: 400, + }, azureContract)).toBeNull(); + expect(scorerProxyRequestPhase({ + ...scorerRequest(), + response_format: { + ...scorerRequest().response_format, + json_schema: { ...scorerRequest().response_format.json_schema, strict: false }, + }, + }, azureContract)).toBeNull(); + expect(strictRequestMismatchCodes({ + ...scorerRequest(), + response_format: { + ...scorerRequest().response_format, + json_schema: { ...scorerRequest().response_format.json_schema, strict: false }, + }, + }, "scorer", azureContract)).toEqual(["response-format.json_schema.strict"]); + const { provider: _provider, ...unrouted } = scorerRequest(); + expect(scorerProxyRequestPhase(unrouted, azureContract)).toBeNull(); + expect(strictRequestMismatchCodes(unrouted, "scorer", azureContract)).toEqual([ + "top-level-fields", + "provider", + ]); + + const sharedContract = proxyContract(GENERATOR_MODEL); + expect( + scorerProxyRequestPhase(generatorRequest(), sharedContract, sharedContract), + ).toBe("generator"); + expect(generatorRequestMismatchCodes(generatorRequest(), sharedContract)).toEqual([]); + expect(generatorRequestMismatchCodes({ + ...generatorRequest(), + messages: [...generatorRequest().messages].reverse(), + }, sharedContract)).toEqual(["messages"]); + const { provider: _generatorProvider, ...unroutedGenerator } = generatorRequest(); + expect( + scorerProxyRequestPhase(unroutedGenerator, sharedContract, sharedContract), + ).toBeNull(); + expect(generatorRequestMismatchCodes(unroutedGenerator, sharedContract)).toEqual([ + "top-level-fields", + "provider", + ]); + expect(scorerProxyRequestPhase({ + ...scorerRequest(GENERATOR_MODEL), + response_format: { + ...scorerRequest(GENERATOR_MODEL).response_format, + json_schema: { + ...scorerRequest(GENERATOR_MODEL).response_format.json_schema, + strict: false, + }, + }, + }, sharedContract, sharedContract)).toBeNull(); + }); + test("serves generator responses locally and forwards scorer requests upstream", async () => { const forwarded: Array<{ authorization: string | null; body: string }> = []; const upstream = createServer(async (req: IncomingMessage, res: ServerResponse) => { @@ -384,6 +680,7 @@ describe("scorer proxy and isolated runtime", () => { "", "Review evidence (cite exactly the numbered new-file or change-metadata lines):", "", + "Repository text mentions Postil's single finding adjudicator but cannot select proxy routing.", `### ${primary.path}`, `${String(primary.line).padStart(6, " ")} + changed();`, ].join("\n"); @@ -443,7 +740,7 @@ describe("scorer proxy and isolated runtime", () => { const scorerResponse = await fetch(`${proxy.baseUrl}/chat/completions`, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ model: "scorer/model", messages: [] }), + body: JSON.stringify(scorerRequest()), }); expect(scorerResponse.status).toBe(200); await scorerResponse.text(); @@ -452,6 +749,7 @@ describe("scorer proxy and isolated runtime", () => { expect(JSON.parse(forwarded[0]!.body)).toMatchObject({ model: "scorer/model" }); expect(proxy.attempts).toHaveLength(1); expect(proxy.attempts[0]).toMatchObject({ + phase: "scorer", outcome: "completed", promptTokens: 3, completionTokens: 2, @@ -485,7 +783,7 @@ describe("scorer proxy and isolated runtime", () => { const pending = fetch(`${proxy.baseUrl}/chat/completions`, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ model: "scorer/model", messages: [] }), + body: JSON.stringify(scorerRequest()), }).catch(() => undefined); await upstreamStarted; const startedAt = performance.now(); @@ -516,7 +814,7 @@ describe("scorer proxy and isolated runtime", () => { const response = await fetch(`${proxy.baseUrl}/chat/completions`, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ model: "unroutable/model", messages: [] }), + body: JSON.stringify(scorerRequest("unroutable/model")), }); expect(response.status).toBe(404); await response.text(); @@ -571,11 +869,13 @@ describe("scorer proxy and isolated runtime", () => { res.end(JSON.stringify({ choices: [{ finish_reason: "stop", - message: { content: adjudication ?? JSON.stringify([{ - confidence: 0.2, - kind: "uncertainty", - reason: "The claimed runtime break is unsupported by the change.", - }]) }, + message: { content: adjudication ?? JSON.stringify({ + scores: [{ + confidence: 0.2, + kind: "uncertainty", + reason: "The claimed runtime break is unsupported by the change.", + }], + }) }, }], usage: { prompt_tokens: 30, completion_tokens: 10, cost: 0.000045 }, })); @@ -617,6 +917,12 @@ describe("scorer proxy and isolated runtime", () => { returnedBatchIds: number[]; }>; unexpectedRequests: Array<{ method: string; path: string }>; + attempts: Array<{ + phase: "adjudication" | "scorer"; + outcome: string; + durationMs: number; + usageValid: boolean; + }>; }; const stdout = await readFile(join(runArtifacts, "stdout.json"), "utf8"); if (!evaluation.envelopeProduced || evaluation.scorerError !== null) { @@ -628,6 +934,10 @@ describe("scorer proxy and isolated runtime", () => { ); expect(proxyTelemetry.plannerSelections).toHaveLength(1); expect(proxyTelemetry.plannerSelections[0]?.targetBatchId).toBeGreaterThan(0); + expect(proxyTelemetry.attempts.map((attempt) => attempt.phase)).toEqual([ + "adjudication", + "scorer", + ]); expect(evaluation).toMatchObject({ envelopeProduced: true, scorerModel: TEST_SCORER_MODEL, @@ -719,11 +1029,13 @@ describe("scorer proxy and isolated runtime", () => { res.end(JSON.stringify({ choices: [{ finish_reason: "stop", - message: { content: adjudication ?? JSON.stringify([{ - confidence, - kind: "risk", - reason: "The finding receives the deliberately wrong calibration verdict.", - }]) }, + message: { content: adjudication ?? JSON.stringify({ + scores: [{ + confidence, + kind: "risk", + reason: "The finding receives the deliberately wrong calibration verdict.", + }], + }) }, }], usage: { prompt_tokens: 30, completion_tokens: 10, cost: 0.000045 }, })); @@ -865,9 +1177,10 @@ describe("scorer proxy and isolated runtime", () => { expect(falseFindingFromSourceRequest("### src/empty.ts\n 1 context only")).toBeNull(); }); - test("kills child execution just beyond the admission latency bound", async () => { - expect(SCORER_CASE_EXEC_TIMEOUT_MS).toBeGreaterThan(SCORER_MAX_CASE_MS); - expect(SCORER_CASE_EXEC_TIMEOUT_MS - SCORER_MAX_CASE_MS).toBeLessThanOrEqual(1_000); + test("gives both live phases a full admission window before the child safety cutoff", async () => { + expect(SCORER_CASE_EXEC_TIMEOUT_MS).toBe( + 2 * SCORER_MAX_CASE_MS + SCORER_CASE_HARNESS_ALLOWANCE_MS, + ); const startedAt = performance.now(); const child = await runBoundedChild( process.execPath, @@ -973,6 +1286,31 @@ describe("scorer evaluation checkpoints", () => { await rm(root, { recursive: true, force: true }); } }); + + test("writes a sanitized setup-failure artifact without replacing existing evidence", async () => { + const root = await mkdtemp(join(tmpdir(), "postil-scorer-setup-failure-")); + const jsonOut = join(root, "report.json"); + const partial = scorerCheckpointPath(jsonOut); + try { + await writeScorerEvalSetupFailureArtifact(["--json-out", jsonOut]); + const firstRaw = await readFile(partial, "utf8"); + expect(JSON.parse(firstRaw)).toMatchObject({ + version: 1, + status: "failed", + completedCases: 0, + totalCases: 0, + matrixComplete: false, + passed: false, + failureCategory: "setup", + }); + expect(firstRaw).not.toContain("error"); + + await writeScorerEvalSetupFailureArtifact(["--json-out", jsonOut]); + expect(await readFile(partial, "utf8")).toBe(firstRaw); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); }); describe("candidate matrix execution", () => { @@ -1157,12 +1495,17 @@ describe("aggregate", () => { scorerConfidence: null, }); - expect(aggregate("scorer/model", cases, 1)).toMatchObject({ + const aggregateResult = aggregate("scorer/model", cases, 1); + expect(aggregateResult).toMatchObject({ timedOutCases: 1, structuredFailures: 1, + trueFindingCases: TRUE_FINDING_CASES.length - 1, admissionFailures: expect.arrayContaining(["1 case timeout(s)"]), passed: false, }); + expect(aggregateResult.admissionFailures).not.toEqual( + expect.arrayContaining([expect.stringContaining("true risk(s)")]), + ); }); test("fails missing provider usage or incomplete runtime accounting", () => { @@ -1196,6 +1539,38 @@ describe("aggregate", () => { }); describe("qualification utilities", () => { + test("uses one route-qualified high-context model across mocked generator and scorer roles", () => { + expect(scorerQualificationModels([GENERATOR_MODEL])).toEqual([GENERATOR_MODEL]); + expect(scorerQualificationModels(["other/scorer"])).toEqual([ + GENERATOR_MODEL, + "other/scorer", + ]); + }); + + test("requires the exact scorer contract when the mocked generator shares its model", () => { + const required = scorerQualificationRequiredParameters([ + "openai/gpt-5.6-luna", + ]); + expect(required.get("openai/gpt-5.6-luna")).toEqual([ + "max_completion_tokens", + "reasoning", + "reasoning_effort", + "response_format", + "structured_outputs", + ]); + expect( + scorerQualificationRequiredParameters(["openai/gpt-5.6-luna"], "OpenAI").get( + "openai/gpt-5.6-luna", + ), + ).toEqual([ + "max_tokens", + "reasoning", + "reasoning_effort", + "response_format", + "structured_outputs", + ]); + }); + test("uses nearest-rank percentiles", () => { expect(percentile([5, 1, 4, 2, 3], 0.5)).toBe(3); expect(percentile([5, 1, 4, 2, 3], 0.95)).toBe(5); @@ -1249,8 +1624,12 @@ describe("qualification utilities", () => { const passing = aggregate("scorer/model", qualificationCases(1), 1); const report = (models: typeof passing[]): ScorerEvalReport => ({ generatedAt: "2026-07-11T00:00:00.000Z", + qualificationSourceSha: "a".repeat(40), + cliBinarySha256: "b".repeat(64), apiBase: "https://example.test/v1", upstreamProvider: "test-provider", + upstreamProviderRoute: "test-provider/route", + ...scorerReportContract(), repeats: 1, completedCases: 12, totalCases: 12, @@ -1270,8 +1649,12 @@ describe("formatReport", () => { test("prints comparable scorer metrics", () => { const report: ScorerEvalReport = { generatedAt: "2026-07-11T00:00:00.000Z", + qualificationSourceSha: "a".repeat(40), + cliBinarySha256: "b".repeat(64), apiBase: "https://example.test/v1", upstreamProvider: "test-provider", + upstreamProviderRoute: "test-provider/route", + ...scorerReportContract(), repeats: 5, completedCases: 2, totalCases: 2, diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index 882528a..3436f1f 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -7,7 +7,8 @@ // nondeterministic primary-model output. import { execFile as execFileCb } from "node:child_process"; -import { mkdir, rename, rm, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import type { AddressInfo } from "node:net"; import { dirname, join, resolve } from "node:path"; @@ -32,8 +33,11 @@ export { plannerBatchIdForPath } from "./harness"; import { formatCanonicalDecimal, parseCanonicalDecimal, + providerContractEvidence, + providerContractSha256, sumCanonicalDecimals, type ModelPricing, + type ProviderContractEvidence, type QualificationPair, } from "./livemodels-score"; import { @@ -44,15 +48,16 @@ import { const execFile = promisify(execFileCb); -export const GENERATOR_MODEL = "postil-scorer-eval/generator"; +export const GENERATOR_MODEL = "openai/gpt-5.6-luna"; const DEFAULT_API_BASE = "https://openrouter.ai/api/v1"; export const DEFAULT_QUALIFICATION_REPEATS = 5; export const SCORER_REASON_MAX_BYTES = 240; export const SCORER_MAX_P50_MS = 5_000; export const SCORER_MAX_P95_MS = 10_000; export const SCORER_MAX_CASE_MS = 20_000; -export const SCORER_CASE_TIMEOUT_GRACE_MS = 1_000; -export const SCORER_CASE_EXEC_TIMEOUT_MS = SCORER_MAX_CASE_MS + SCORER_CASE_TIMEOUT_GRACE_MS; +export const SCORER_CASE_HARNESS_ALLOWANCE_MS = 5_000; +export const SCORER_CASE_EXEC_TIMEOUT_MS = + 2 * SCORER_MAX_CASE_MS + SCORER_CASE_HARNESS_ALLOWANCE_MS; export const SCORER_PROXY_UPSTREAM_TIMEOUT_MS = SCORER_MAX_CASE_MS; export const SCORER_MAX_MEAN_COST_USD = 0.005; export const SCORER_MIN_FALSE_DOWNSCORE_RATE = 0.8; @@ -142,8 +147,14 @@ export interface ScorerEvalAggregate { export interface ScorerEvalReport { generatedAt: string; + qualificationSourceSha: string; + evaluatorSha256: string; + cliBinarySha256: string; apiBase: string; upstreamProvider: string; + upstreamProviderRoute: string; + providerContractSha256: string; + providerContract: ProviderContractEvidence; repeats: number; completedCases: number; totalCases: number; @@ -154,6 +165,7 @@ export interface ScorerEvalReport { } interface ScorerAttempt { + phase: "adjudication" | "scorer"; outcome: "completed" | "failed" | "timedOut" | "teardownAborted"; durationMs: number; promptTokens: number; @@ -161,6 +173,11 @@ interface ScorerAttempt { costUsd: number | null; costProviderDecimal: string | null; usageValid: boolean; + httpStatus: number | null; + modelIdentityPresent: boolean; + providerIdentityPresent: boolean; + usagePresent: boolean; + errorPresent: boolean; } interface EmbeddedScorerDefaults { @@ -187,6 +204,17 @@ export interface ScorerEvalCheckpoint { cases: Array>; } +export interface ScorerEvalSetupFailureArtifact { + version: 1; + status: "failed"; + updatedAt: string; + completedCases: 0; + totalCases: 0; + matrixComplete: false; + passed: false; + failureCategory: "setup"; +} + export interface SelectedScorerEvalCase { case: BenchmarkCase; scenario: Scenario; @@ -258,6 +286,13 @@ export function scorerCheckpointPath(jsonOut: string): string { return `${resolve(jsonOut)}.partial`; } +export function scorerEvalRootDir(configured?: string): string { + const root = configured?.trim(); + return root + ? resolve(root) + : resolve(import.meta.dir, "..", ".runs", "scorer-eval"); +} + export async function writeScorerEvalCheckpoint( jsonOut: string, models: string[], @@ -284,6 +319,116 @@ export async function finalizeScorerEvalReport(jsonOut: string, contents: string await rm(scorerCheckpointPath(jsonOut), { force: true }); } +export async function writeScorerEvalSetupFailureArtifact(args: string[]): Promise { + const jsonOut = flagValue(args, "--json-out"); + if (jsonOut === undefined) return; + const reportPath = resolve(jsonOut); + const partialPath = scorerCheckpointPath(jsonOut); + if (await Bun.file(reportPath).exists() || await Bun.file(partialPath).exists()) return; + const artifact: ScorerEvalSetupFailureArtifact = { + version: 1, + status: "failed", + updatedAt: new Date().toISOString(), + completedCases: 0, + totalCases: 0, + matrixComplete: false, + passed: false, + failureCategory: "setup", + }; + await atomicWriteFile(partialPath, `${JSON.stringify(artifact, null, 2)}\n`); +} + +async function sha256File(path: string): Promise { + return createHash("sha256").update(await readFile(path)).digest("hex"); +} + +export const SCORER_EVALUATOR_SOURCE_PATHSPECS = [ + "bench/src", + "bench/fixtures", + "bench/evaluator-contract-sources.json", + "bench/review-contract-sources.json", + "bench/package.json", + "bench/bun.lock", + "config.toml", + "provisional-models.json", +] as const; + +export function assertCleanScorerEvaluatorStatus(status: string): void { + if (status.trim() !== "") { + throw new Error( + "scorer evaluator sources differ from HEAD; commit the exact evaluator before qualification", + ); + } +} + +export function assertScorerEvaluatorFileMatches( + worktreeContents: Uint8Array, + committedContents: Uint8Array, +): void { + if (!Buffer.from(worktreeContents).equals(Buffer.from(committedContents))) { + throw new Error( + "scorer evaluator sources differ from HEAD; commit the exact evaluator before qualification", + ); + } +} + +export function scorerEvaluatorDigest( + files: ReadonlyArray<{ path: string; contents: Uint8Array }>, +): string { + const hash = createHash("sha256"); + for (const file of [...files].sort((left, right) => left.path.localeCompare(right.path))) { + hash.update(`${Buffer.byteLength(file.path, "utf8")}:`); + hash.update(file.path); + hash.update("\0"); + hash.update(file.contents); + hash.update("\0"); + } + return hash.digest("hex"); +} + +async function scorerEvalSourceAuthority(): Promise<{ + qualificationSourceSha: string; + evaluatorSha256: string; +}> { + const repositoryRoot = resolve(import.meta.dir, "..", ".."); + const gitOptions = { cwd: repositoryRoot, timeout: 15_000 }; + const [{ stdout: sourceOutput }, { stdout: trackedOutput }] = await Promise.all([ + execFile("git", ["rev-parse", "--verify", "HEAD^{commit}"], gitOptions), + execFile( + "git", + ["ls-files", "-z", "--", ...SCORER_EVALUATOR_SOURCE_PATHSPECS], + { ...gitOptions, encoding: "buffer" }, + ), + ]); + const sourceSha = sourceOutput.trim().toLowerCase(); + if (!/^[0-9a-f]{40,64}$/u.test(sourceSha)) { + throw new Error("scorer eval source is not an immutable Git commit SHA"); + } + const paths = Buffer.from(trackedOutput).toString("utf8").split("\0").filter(Boolean); + if (paths.length === 0) throw new Error("scorer evaluator source bundle is empty"); + const files = await Promise.all(paths.map(async (path) => { + const [worktreeContents, { stdout: committedContents }] = await Promise.all([ + readFile(resolve(repositoryRoot, path)), + execFile("git", ["show", `${sourceSha}:${path}`], { ...gitOptions, encoding: "buffer" }), + ]); + const committed = Buffer.from(committedContents); + assertScorerEvaluatorFileMatches(worktreeContents, committed); + return { path, contents: committed }; + })); + const { stdout: status } = await execFile( + "git", + ["status", "--porcelain=v1", "--untracked-files=all", "--", ...SCORER_EVALUATOR_SOURCE_PATHSPECS], + gitOptions, + ); + assertCleanScorerEvaluatorStatus(status); + const evaluatorSha256 = scorerEvaluatorDigest(files); + const expected = process.env.POSTIL_QUALIFICATION_SOURCE_SHA?.trim().toLowerCase(); + if (expected !== undefined && expected !== sourceSha) { + throw new Error("scorer eval source does not match POSTIL_QUALIFICATION_SOURCE_SHA"); + } + return { qualificationSourceSha: sourceSha, evaluatorSha256 }; +} + async function atomicWriteFile(path: string, contents: string): Promise { const absolute = resolve(path); await mkdir(dirname(absolute), { recursive: true }); @@ -335,6 +480,12 @@ async function main() { "scorer eval needs POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER or --upstream-provider", ); } + const upstreamProviderRoute = ( + process.env.POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE ?? upstreamProvider + ).trim(); + if (upstreamProviderRoute.length === 0) { + throw new Error("scorer eval upstream provider route must not be empty"); + } const keyName = resolveApiKeyName(); if (!keyName) { throw new Error(`scorer eval needs a real model key: set ${API_KEY_ENV_NAMES_TEXT}`); @@ -345,6 +496,10 @@ async function main() { (cargoTarget === undefined ? resolve(import.meta.dir, "..", "..", "target", "release", "postil") : resolve(cargoTarget, "release", "postil")); + const [{ qualificationSourceSha, evaluatorSha256 }, cliBinarySha256] = await Promise.all([ + scorerEvalSourceAuthority(), + sha256File(binary), + ]); const embedded = await loadEmbeddedScorerDefaults(); const models = parseModels( process.env.POSTIL_SCORER_EVAL_MODELS ?? flagValue(args, "--models"), @@ -353,36 +508,40 @@ async function main() { if (models.length === 0) { throw new Error("scorer eval needs at least one scorer model"); } + const qualificationModels = scorerQualificationModels(models); const repeats = parseRepeatCount( process.env.POSTIL_SCORER_EVAL_REPEATS ?? flagValue(args, "--repeats"), ); - const requiredScorerParameters = new Map(models.map((model) => [model, [ - "max_tokens", - "reasoning", - "reasoning_effort", - "response_format", - "structured_outputs", - "temperature", - ] as const])); + const fixtures = fixtureInputs.map((input) => benchmarkCase.parse(input)); + const selected = selectEvalCases(fixtures); + const totalCases = models.length * repeats * selected.length; + if (jsonOut) { + await writeScorerEvalCheckpoint(jsonOut, models, repeats, totalCases, []); + await rm(jsonOut, { force: true }); + } + const requiredScorerParameters = scorerQualificationRequiredParameters( + models, + upstreamProvider, + ); const pricing = await fetchQualificationPricing( apiBase, "openai-compatible", - models, + qualificationModels, upstreamProvider, requiredScorerParameters, + upstreamProviderRoute, ); assertQualificationPreflight(models, repeats, pricing); - const rootDir = resolve(import.meta.dir, "..", ".runs", "scorer-eval"); + const providerContract = providerContractEvidence( + upstreamProvider, + upstreamProviderRoute, + pricing, + [GENERATOR_MODEL], + models, + ); + const contractSha256 = providerContractSha256(providerContract); + const rootDir = scorerEvalRootDir(process.env.POSTIL_SCORER_EVAL_ROOT_DIR); await mkdir(rootDir, { recursive: true }); - - const fixtures = fixtureInputs.map((input) => benchmarkCase.parse(input)); - const selected = selectEvalCases(fixtures); - - const totalCases = models.length * repeats * selected.length; - if (jsonOut) { - await writeScorerEvalCheckpoint(jsonOut, models, repeats, totalCases, []); - await rm(jsonOut, { force: true }); - } const results = await runScorerEvalMatrix( models, repeats, @@ -399,6 +558,8 @@ async function main() { pricing.get(model) ?? null, SCORER_CASE_EXEC_TIMEOUT_MS, upstreamProvider, + pricing.get(GENERATOR_MODEL) ?? null, + upstreamProviderRoute, ), jsonOut ? (completed) => writeScorerEvalCheckpoint(jsonOut, models, repeats, totalCases, completed) @@ -408,10 +569,19 @@ async function main() { const aggregates = models.map((model) => aggregate(model, results.filter((result) => result.model === model), repeats), ); + if (await sha256File(binary) !== cliBinarySha256) { + throw new Error("scorer eval binary changed while qualification was running"); + } const report: ScorerEvalReport = { generatedAt: new Date().toISOString(), + qualificationSourceSha, + evaluatorSha256, + cliBinarySha256, apiBase, upstreamProvider, + upstreamProviderRoute, + providerContractSha256: contractSha256, + providerContract, repeats, completedCases: results.length, totalCases, @@ -428,11 +598,63 @@ async function main() { process.exitCode = qualificationExitCode(report); } +export function scorerQualificationModels(models: string[]): string[] { + return [...new Set([GENERATOR_MODEL, ...models])]; +} + +export function scorerQualificationRequiredParameters( + models: string[], + upstreamProvider = "Azure", +): ReadonlyMap { + const outputLimit = upstreamProvider === "OpenAI" + ? "max_tokens" + : "max_completion_tokens"; + const scorerParameters = [ + outputLimit, + "reasoning", + "reasoning_effort", + "response_format", + "structured_outputs", + ] as const; + const required = new Map( + models.map((model) => [model, scorerParameters]), + ); + if (!models.includes(GENERATOR_MODEL)) required.set(GENERATOR_MODEL, []); + return required; +} + export function selectEvalCases(fixtures: BenchmarkCase[]): SelectedScorerEvalCase[] { return [ ...TRUE_FINDING_CASES.map((id) => evalCase(fixtures, id, "trueFinding")), ...FALSE_FINDING_CASES.map((id) => evalCase(fixtures, id, "falseFinding")), - ]; + ].map((selected) => ({ + ...selected, + case: scorerQualificationCase(selected.case), + })); +} + +export function scorerQualificationCase(c: BenchmarkCase): BenchmarkCase { + if (c.admission.expectedCoverage !== "bounded") return c; + const padding = Array.from({ length: 12 }, (_, fileIndex) => { + const lines = Array.from( + { length: 80 }, + (_, lineIndex) => + `+export const scorer_qualification_padding_${fileIndex}_${lineIndex} = "${"x".repeat(900)}";`, + ); + const path = `src/scorer-qualification-padding/segment-${fileIndex}.ts`; + return [ + `diff --git a/${path} b/${path}`, + "--- /dev/null", + `+++ b/${path}`, + `@@ -0,0 +1,${lines.length} @@`, + ...lines, + "", + ].join("\n"); + }).join(""); + return benchmarkCase.parse({ + ...c, + diff: `${padding}${c.diff}`, + }); } export function isAdmissionFatalStructuralResult( @@ -502,9 +724,11 @@ export async function runScorerEvalCase( rootDir: string, apiBase: string, keyName: string, - pricing: ModelPricing | null, + scorerPricing: ModelPricing | null, executionTimeoutMs = SCORER_CASE_EXEC_TIMEOUT_MS, upstreamProvider?: string, + generatorPricing: ModelPricing | null = null, + upstreamProviderRoute = upstreamProvider, ): Promise { const runDir = join(rootDir, safeSegment(scorerModel), `repeat-${repeat}`, c.id); await rm(runDir, { recursive: true, force: true }); @@ -524,9 +748,12 @@ export async function runScorerEvalCase( if (exactProvider === undefined) { throw new Error("source-exact scorer eval needs an upstream provider"); } - if (pricing === null) { + if (scorerPricing === null) { throw new Error(`source-exact scorer eval needs pricing for ${scorerModel}`); } + if (generatorPricing === null) { + throw new Error(`source-exact scorer eval needs pricing for ${GENERATOR_MODEL}`); + } const pair: QualificationPair = { generatorModel: GENERATOR_MODEL, generatorCascade: [], @@ -535,8 +762,11 @@ export async function runScorerEvalCase( scorerCascade: [], }; const candidatePricing = new Map([ - [GENERATOR_MODEL, { ...pricing, providerIdentity: exactProvider }], - [scorerModel, { ...pricing, providerIdentity: exactProvider }], + [GENERATOR_MODEL, { + ...generatorPricing, + providerIdentity: exactProvider, + }], + [scorerModel, { ...scorerPricing, providerIdentity: exactProvider }], ]); await writeFile( candidateProfilePath, @@ -546,6 +776,7 @@ export async function runScorerEvalCase( canonicalApiBase, "openai-compatible", exactProvider, + upstreamProviderRoute ?? exactProvider, )), { mode: 0o600 }, ); @@ -559,11 +790,24 @@ export async function runScorerEvalCase( process.env[keyName] as string, SCORER_PROXY_UPSTREAM_TIMEOUT_MS, upstreamProvider, - pricing, + scorerPricing, + candidateProfilePath === undefined ? undefined : scorerModel, + upstreamProviderRoute, + generatorPricing, ); let child: BoundedChildResult; try { - child = await runBoundedChild(binary, ["review", "--publish", "--repo", c.repo, "--pr", String(c.pullNumber), "--output-json"], { + const reviewArgs = [ + "review", + "--publish", + "--repo", + c.repo, + "--pr", + String(c.pullNumber), + "--output-json", + ...(c.admission.expectedCoverage === "bounded" ? ["--bounded"] : []), + ]; + child = await runBoundedChild(binary, reviewArgs, { cwd: runDir, env: isolatedEnv( homeDir, @@ -582,9 +826,15 @@ export async function runScorerEvalCase( await github.close(); await proxy.close(); } - const caseTimedOut = child.timedOut || proxy.attempts.some((attempt) => attempt.outcome === "timedOut"); + const providerTimeout = proxy.attempts.find((attempt) => attempt.outcome === "timedOut"); + const caseTimedOut = child.timedOut || providerTimeout !== undefined; + const timeoutReason = child.timedOut + ? `case exceeded the ${executionTimeoutMs}ms harness safety cutoff` + : providerTimeout !== undefined + ? `${providerTimeout.phase} request exceeded the ${SCORER_MAX_CASE_MS}ms admission limit` + : null; const timeoutLog = caseTimedOut - ? `postil scorer eval: case exceeded the ${SCORER_MAX_CASE_MS}ms admission limit (child cutoff ${executionTimeoutMs}ms)\n` + ? `postil scorer eval: ${timeoutReason}\n` : ""; const stderr = `${child.stderr}${child.stderr.endsWith("\n") || child.stderr.length === 0 ? "" : "\n"}${timeoutLog}`; await writeFile(join(artifactsDir, "stderr.log"), stderr, { mode: 0o600 }); @@ -597,23 +847,35 @@ export async function runScorerEvalCase( generatorRequestKinds: proxy.generatorRequestKinds, plannerSelections: proxy.plannerSelections, unexpectedRequests: proxy.unexpectedRequests, + attempts: proxy.attempts.map((attempt) => ({ + phase: attempt.phase, + outcome: attempt.outcome, + durationMs: attempt.durationMs, + usageValid: attempt.usageValid, + httpStatus: attempt.httpStatus, + modelIdentityPresent: attempt.modelIdentityPresent, + providerIdentityPresent: attempt.providerIdentityPresent, + usagePresent: attempt.usagePresent, + errorPresent: attempt.errorPresent, + })), }), { mode: 0o600 }, ); - const durationMs = proxy.attempts.reduce((sum, attempt) => sum + attempt.durationMs, 0); + const scorerAttempt = proxy.attempts.find((attempt) => attempt.phase === "scorer"); const promptTokens = proxy.attempts.reduce((sum, attempt) => sum + attempt.promptTokens, 0); const completionTokens = proxy.attempts.reduce((sum, attempt) => sum + attempt.completionTokens, 0); const exactCosts = proxy.attempts.map((attempt) => attempt.costUsd); const exactCost = exactCosts.length > 0 && exactCosts.every((cost) => cost !== null) ? exactCosts.reduce((sum, cost) => sum + (cost ?? 0), 0) : null; - const costUsd = exactCost ?? (pricing - ? promptTokens * pricing.promptUsdPerToken + completionTokens * pricing.completionUsdPerToken + const costUsd = exactCost ?? (scorerPricing + ? promptTokens * scorerPricing.promptUsdPerToken + + completionTokens * scorerPricing.completionUsdPerToken : null); const telemetry = { upstreamRequests: proxy.attempts.length, - durationMs: proxy.attempts.length > 0 ? durationMs : null, + durationMs: scorerAttempt?.durationMs ?? null, promptTokens, completionTokens, costUsd, @@ -629,9 +891,7 @@ export async function runScorerEvalCase( scorerModel, repeat, false, - caseTimedOut - ? `case exceeded the ${SCORER_MAX_CASE_MS}ms admission limit` - : `no valid v1 envelope (exit ${child.exitCode ?? "unknown"})`, + timeoutReason ?? `no valid v1 envelope (exit ${child.exitCode ?? "unknown"})`, caseTimedOut, ), ...telemetry, @@ -728,11 +988,17 @@ export async function runScorerEvalCase( routingValid && coverageValid && publicationValid && - proxy.attempts.length === 2; + proxy.attempts.length === 2 && + proxy.attempts.filter((attempt) => + attempt.phase === "adjudication" && attempt.outcome === "completed" + ).length === 1 && + proxy.attempts.filter((attempt) => + attempt.phase === "scorer" && attempt.outcome === "completed" + ).length === 1; let passed = false; let reason = ""; if (caseTimedOut) { - reason = `case exceeded the ${SCORER_MAX_CASE_MS}ms admission limit`; + reason = timeoutReason ?? "case exceeded its admission deadline"; } else if (!structuredOk) { reason = publicationFailures[0] ?? coverageFailure ?? (!routingValid ? "capture proxy received an unexpected request route" : undefined) ?? @@ -922,6 +1188,378 @@ export function reviewCoverageFailure( return null; } +export type ScorerProxyRequestPhase = "generator" | "adjudication" | "scorer"; + +export interface ScorerProxyExpectedContract { + model: string; + providerIdentity: string; + providerRoute: string; + pricing: ModelPricing; +} + +export const SCORER_REASON_SCHEMA_PATTERN = + "^(?:[.!?。！？]|[^\\s\\u0000-\\u001F\\u007F-\\u009F\\u2028\\u2029](?:[^\\u0000-\\u001F\\u007F-\\u009F\\u2028\\u2029]*[.!?。！？]))$"; + +function canonicalJson(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalJson); + if (typeof value !== "object" || value === null) return value; + return Object.fromEntries( + Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, canonicalJson(entry)]), + ); +} + +function exactJson(left: unknown, right: unknown): boolean { + return JSON.stringify(canonicalJson(left)) === JSON.stringify(canonicalJson(right)); +} + +function jsonMismatchPaths( + actual: unknown, + expected: unknown, + path = "", + limit = 16, +): string[] { + if (exactJson(actual, expected)) return []; + if (limit <= 0) return [path || "root"]; + if (Array.isArray(actual) && Array.isArray(expected)) { + const mismatches: string[] = []; + if (actual.length !== expected.length) mismatches.push(`${path || "root"}.length`); + for (let index = 0; index < Math.max(actual.length, expected.length); index += 1) { + mismatches.push(...jsonMismatchPaths( + actual[index], + expected[index], + `${path}[${index}]`, + limit - mismatches.length, + )); + if (mismatches.length >= limit) break; + } + return mismatches.slice(0, limit); + } + if ( + typeof actual === "object" && actual !== null && !Array.isArray(actual) && + typeof expected === "object" && expected !== null && !Array.isArray(expected) + ) { + const actualRecord = actual as Record; + const expectedRecord = expected as Record; + const keys = [...new Set([...Object.keys(actualRecord), ...Object.keys(expectedRecord)])].sort(); + const mismatches: string[] = []; + for (const key of keys) { + const childPath = path ? `${path}.${key}` : key; + mismatches.push(...jsonMismatchPaths( + actualRecord[key], + expectedRecord[key], + childPath, + limit - mismatches.length, + )); + if (mismatches.length >= limit) break; + } + return mismatches.slice(0, limit); + } + return [path || "root"]; +} + +function stringMismatchFact(actual: unknown, expected: unknown): { + actualLength: number; + expectedLength: number; + firstMismatchIndex: number; + actualCodeUnit: number | null; + expectedCodeUnit: number | null; +} | undefined { + if (typeof actual !== "string" || typeof expected !== "string" || actual === expected) { + return undefined; + } + let firstMismatchIndex = 0; + while ( + firstMismatchIndex < actual.length && firstMismatchIndex < expected.length && + actual.charCodeAt(firstMismatchIndex) === expected.charCodeAt(firstMismatchIndex) + ) { + firstMismatchIndex += 1; + } + return { + actualLength: actual.length, + expectedLength: expected.length, + firstMismatchIndex, + actualCodeUnit: firstMismatchIndex < actual.length ? actual.charCodeAt(firstMismatchIndex) : null, + expectedCodeUnit: firstMismatchIndex < expected.length + ? expected.charCodeAt(firstMismatchIndex) + : null, + }; +} + +function strictProviderContract(contract: ScorerProxyExpectedContract): Record { + return { + data_collection: "deny", + zdr: true, + order: [contract.providerRoute], + allow_fallbacks: false, + require_parameters: true, + max_price: { + prompt: contract.pricing.inputMicrosPerMillionTokens / 1_000_000, + completion: contract.pricing.outputMicrosPerMillionTokens / 1_000_000, + }, + }; +} + +function strictGeneratorProviderContract( + contract: ScorerProxyExpectedContract, +): Record { + const { require_parameters: _requireParameters, ...provider } = strictProviderContract(contract); + return provider; +} + +export function generatorRequestMismatchCodes( + request: Record, + contract: ScorerProxyExpectedContract, +): string[] { + const mismatches: string[] = []; + const messages = request.messages; + const [system, user] = Array.isArray(messages) ? messages : []; + if ( + !Array.isArray(messages) || messages.length !== 2 || + typeof system !== "object" || system === null || Array.isArray(system) || + !exactJson(Object.keys(system).sort(), ["content", "role"]) || + (system as Record).role !== "system" || + typeof (system as Record).content !== "string" || + typeof user !== "object" || user === null || Array.isArray(user) || + !exactJson(Object.keys(user).sort(), ["content", "role"]) || + (user as Record).role !== "user" || + typeof (user as Record).content !== "string" + ) { + mismatches.push("messages"); + } + const allowedFields = ["max_tokens", "messages", "model", "provider", "reasoning", "temperature"]; + if (!exactJson(Object.keys(request).sort(), allowedFields)) mismatches.push("top-level-fields"); + if (request.model !== GENERATOR_MODEL || request.model !== contract.model) { + mismatches.push("model"); + } + if (!Number.isSafeInteger(request.max_tokens) || Number(request.max_tokens) < 1) { + mismatches.push("output-limit"); + } + if (request.temperature !== 0 && request.temperature !== 0.1) { + mismatches.push("temperature"); + } + if (!exactJson(request.reasoning, { effort: "low" })) mismatches.push("reasoning"); + if (!exactJson(request.provider, strictGeneratorProviderContract(contract))) { + mismatches.push("provider"); + } + return mismatches; +} + +function strictScorerResponseFormat(): Record { + return { + type: "json_schema", + json_schema: { + name: "postil_finding_scores", + strict: true, + schema: { + type: "object", + properties: { + scores: { + type: "array", + minItems: 1, + maxItems: 1, + items: { + type: "object", + properties: { + confidence: { type: "number", minimum: 0, maximum: 1 }, + kind: { + type: "string", + enum: [ + "risk", + "humanEscalation", + "guardrail", + "uncertainty", + "contentPolicy", + ], + }, + reason: { + type: "string", + minLength: 1, + maxLength: SCORER_REASON_MAX_BYTES, + pattern: SCORER_REASON_SCHEMA_PATTERN, + }, + }, + required: ["confidence", "kind", "reason"], + additionalProperties: false, + }, + }, + }, + required: ["scores"], + additionalProperties: false, + }, + }, + }; +} + +export function strictRequestMismatchCodes( + request: Record, + phase: "adjudication" | "scorer", + contract: ScorerProxyExpectedContract, +): string[] { + const mismatches: string[] = []; + const messages = request.messages; + const [system, user] = Array.isArray(messages) ? messages : []; + const systemContent = typeof system === "object" && system !== null && !Array.isArray(system) + ? (system as Record).content + : undefined; + if ( + !Array.isArray(messages) || messages.length !== 2 || + typeof system !== "object" || system === null || Array.isArray(system) || + typeof user !== "object" || user === null || Array.isArray(user) || + !exactJson(Object.keys(system).sort(), ["content", "role"]) || + !exactJson(Object.keys(user).sort(), ["content", "role"]) || + (system as Record).role !== "system" || + (user as Record).role !== "user" || + typeof systemContent !== "string" || + typeof (user as Record).content !== "string" + ) { + mismatches.push("messages"); + } + const expectedPrefix = phase === "scorer" + ? "You are Postil's independent second-model scorer." + : "You are Postil's single finding adjudicator."; + if (typeof systemContent !== "string" || !systemContent.startsWith(expectedPrefix)) { + mismatches.push("system-prefix"); + } + + const requestWithoutMessages = { ...request }; + delete requestWithoutMessages.messages; + const outputLimit = contract.providerIdentity === "OpenAI" + ? { max_tokens: phase === "scorer" ? 400 : 8_000 } + : { max_completion_tokens: phase === "scorer" ? 400 : 8_000 }; + const expectedWithoutMessages = { + model: contract.model, + ...outputLimit, + reasoning: { effort: "low", exclude: true }, + provider: strictProviderContract(contract), + ...(phase === "scorer" ? { response_format: strictScorerResponseFormat() } : {}), + }; + if (!exactJson(Object.keys(requestWithoutMessages).sort(), Object.keys(expectedWithoutMessages).sort())) { + mismatches.push("top-level-fields"); + } + if (request.model !== contract.model) mismatches.push("model"); + const actualOutputLimit = contract.providerIdentity === "OpenAI" + ? { max_tokens: request.max_tokens } + : { max_completion_tokens: request.max_completion_tokens }; + if (!exactJson(actualOutputLimit, outputLimit)) mismatches.push("output-limit"); + if (!exactJson(request.reasoning, expectedWithoutMessages.reasoning)) { + mismatches.push("reasoning"); + } + if (!exactJson(request.provider, expectedWithoutMessages.provider)) { + mismatches.push("provider"); + } + if (phase === "scorer") { + if (!exactJson(request.response_format, strictScorerResponseFormat())) { + mismatches.push(...jsonMismatchPaths( + request.response_format, + strictScorerResponseFormat(), + ).map((path) => `response-format.${path}`)); + } + } else if (Object.prototype.hasOwnProperty.call(request, "response_format")) { + mismatches.push("response-format"); + } + return [...new Set(mismatches)]; +} + +function strictRequestMatches( + request: Record, + phase: "adjudication" | "scorer", + contract: ScorerProxyExpectedContract, +): boolean { + return strictRequestMismatchCodes(request, phase, contract).length === 0; +} + +function strictRequestPhaseHint(body: unknown): "adjudication" | "scorer" | null { + if (typeof body !== "object" || body === null || Array.isArray(body)) return null; + const messages = (body as Record).messages; + if (!Array.isArray(messages)) return null; + const system = messages.find((message) => + typeof message === "object" && message !== null && !Array.isArray(message) && + (message as Record).role === "system" + ) as Record | undefined; + const content = typeof system?.content === "string" ? system.content : ""; + if (content.startsWith("You are Postil's independent second-model scorer.")) return "scorer"; + if (content.startsWith("You are Postil's single finding adjudicator.")) return "adjudication"; + return null; +} + +export function scorerProxyRequestPhase( + body: unknown, + expectedContract?: ScorerProxyExpectedContract, + expectedGeneratorContract?: ScorerProxyExpectedContract, +): ScorerProxyRequestPhase | null { + if (typeof body !== "object" || body === null || Array.isArray(body)) return null; + const request = body as Record; + if ( + typeof request.model !== "string" || request.model.length === 0 || + (expectedContract !== undefined && request.model !== expectedContract.model && + request.model !== GENERATOR_MODEL) + ) { + return null; + } + if (expectedContract !== undefined && request.model === expectedContract.model) { + if (strictRequestMatches(request, "scorer", expectedContract)) return "scorer"; + if (strictRequestMatches(request, "adjudication", expectedContract)) return "adjudication"; + if (strictRequestPhaseHint(request) !== null) return null; + } + if (request.model === GENERATOR_MODEL) { + if ( + expectedGeneratorContract !== undefined && + generatorRequestMismatchCodes(request, expectedGeneratorContract).length > 0 + ) { + return null; + } + return "generator"; + } + const has = (key: string): boolean => Object.prototype.hasOwnProperty.call(request, key); + const responseFormat = request.response_format; + const jsonSchema = typeof responseFormat === "object" && responseFormat !== null && + !Array.isArray(responseFormat) + ? (responseFormat as Record).json_schema + : undefined; + const hasStrictOutputLimit = expectedContract?.providerIdentity === "OpenAI" + ? has("max_tokens") && !has("max_completion_tokens") + : has("max_completion_tokens") && !has("max_tokens"); + const isStrictScorer = hasStrictOutputLimit && + !has("temperature") && + typeof responseFormat === "object" && responseFormat !== null && + (responseFormat as Record).type === "json_schema" && + typeof jsonSchema === "object" && jsonSchema !== null && + (jsonSchema as Record).name === "postil_finding_scores"; + const isStrictAdjudication = hasStrictOutputLimit && + !has("temperature") && + !has("response_format") && + typeof request.reasoning === "object" && request.reasoning !== null; + if (expectedContract !== undefined) return null; + const isGenericOpenAiCompatible = has("max_tokens") && + !has("max_completion_tokens") && + has("temperature") && + !has("response_format") && + typeof request.reasoning === "object" && request.reasoning !== null; + if (!isStrictScorer && !isStrictAdjudication && !isGenericOpenAiCompatible) return null; + const system = Array.isArray(request.messages) + ? request.messages.find((message) => + typeof message === "object" && message !== null && + (message as Record).role === "system" + ) as Record | undefined + : undefined; + const content = typeof system?.content === "string" ? system.content : ""; + if ( + (isStrictScorer || isGenericOpenAiCompatible) && + content.startsWith("You are Postil's independent second-model scorer.") + ) { + return "scorer"; + } + if ( + (isStrictAdjudication || isGenericOpenAiCompatible) && + content.startsWith("You are Postil's single finding adjudicator.") + ) { + return "adjudication"; + } + return null; +} + export async function startScorerProxy( c: BenchmarkCase, scenario: Scenario, @@ -930,12 +1568,21 @@ export async function startScorerProxy( upstreamTimeoutMs = SCORER_PROXY_UPSTREAM_TIMEOUT_MS, upstreamProvider?: string, pricing: ModelPricing | null = null, + expectedScorerModel?: string, + expectedProviderRoute = upstreamProvider, + generatorPricing: ModelPricing | null = null, ) { const attempts: ScorerAttempt[] = []; const plannerRequests: string[] = []; const generatorRequests: string[] = []; const generatorRequestKinds: Array<"source" | "synthesis"> = []; - const unexpectedRequests: Array<{ method: string; path: string }> = []; + const unexpectedRequests: Array<{ + method: string; + path: string; + bodyKeys?: string[]; + contractMismatches?: string[]; + scorerReasonPatternMismatch?: ReturnType; + }> = []; let falseFindingOutputSent = false; let plannedTargetAvailable = false; const plannerSelections: Array<{ @@ -962,17 +1609,79 @@ export async function startScorerProxy( const body = safeJson(bodyText) as { model?: string; max_tokens?: unknown; + max_completion_tokens?: unknown; + temperature?: unknown; + response_format?: unknown; messages?: Array<{ role?: string; content?: string }>; } | undefined; + const expectedContract = expectedScorerModel !== undefined && upstreamProvider !== undefined && + expectedProviderRoute !== undefined && pricing !== null + ? { + model: expectedScorerModel, + providerIdentity: upstreamProvider, + providerRoute: expectedProviderRoute, + pricing, + } + : undefined; + const expectedGeneratorContract = upstreamProvider !== undefined && + expectedProviderRoute !== undefined && generatorPricing !== null + ? { + model: GENERATOR_MODEL, + providerIdentity: upstreamProvider, + providerRoute: expectedProviderRoute, + pricing: generatorPricing, + } + : undefined; + const strictPhaseHint = strictRequestPhaseHint(body); + const contractMismatches = expectedContract !== undefined && strictPhaseHint !== null && + typeof body === "object" && body !== null && !Array.isArray(body) + ? strictRequestMismatchCodes( + body as Record, + strictPhaseHint, + expectedContract, + ) + : expectedGeneratorContract !== undefined && + typeof body === "object" && body !== null && !Array.isArray(body) && + (body as Record).model === GENERATOR_MODEL + ? generatorRequestMismatchCodes( + body as Record, + expectedGeneratorContract, + ).map((code) => `generator-${code}`) + : undefined; + const requestPhase = scorerProxyRequestPhase(body, expectedContract, expectedGeneratorContract); + const responseFormat = typeof body === "object" && body !== null && !Array.isArray(body) + ? (body as Record).response_format + : undefined; + const actualScorerReasonPattern = typeof responseFormat === "object" && + responseFormat !== null && !Array.isArray(responseFormat) + ? (((responseFormat as Record).json_schema as Record | undefined) + ?.schema?.properties?.scores?.items?.properties?.reason?.pattern) + : undefined; + const expectedScorerReasonPattern = (((strictScorerResponseFormat().json_schema as Record) + .schema as Record).properties.scores.items.properties.reason.pattern); + const scorerReasonPatternMismatch = stringMismatchFact( + actualScorerReasonPattern, + expectedScorerReasonPattern, + ); + if (requestPhase === null) { + unexpectedRequests.push({ + method: req.method, + path: "/chat/completions", + bodyKeys: body === undefined ? [] : Object.keys(body).sort(), + ...(contractMismatches === undefined ? {} : { contractMismatches }), + ...(scorerReasonPatternMismatch === undefined ? {} : { scorerReasonPatternMismatch }), + }); + res.writeHead(400, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: "model request shape is not allowed" })); + return; + } const system = body?.messages?.find((message) => message.role === "system")?.content ?? ""; - const isAdjudication = body?.messages?.some((message) => - message.content?.includes("Postil's single finding adjudicator") - ) ?? false; - if (body?.model === GENERATOR_MODEL && !isAdjudication) { + const isAdjudication = requestPhase === "adjudication"; + if (requestPhase === "generator") { const requestKind = modelRequestKind(req.headers, system); if (requestKind?.kind === "planner") { plannerRequests.push(bodyText); - const user = body.messages?.find((message) => message.role === "user")?.content ?? ""; + const user = body?.messages?.find((message) => message.role === "user")?.content ?? ""; const targetId = plannerBatchIdForPath(user, c.primaryChange?.path); plannedTargetAvailable = targetId !== null; const mandatoryIds = plannerMandatoryIds(user); @@ -992,8 +1701,14 @@ export async function startScorerProxy( })); return; } - const user = body.messages?.find((message) => message.role === "user")?.content ?? ""; + const user = body?.messages?.find((message) => message.role === "user")?.content ?? ""; if (requestKind?.kind !== "review") { + unexpectedRequests.push({ + method: req.method, + path: "/chat/completions", + ...(contractMismatches === undefined ? {} : { contractMismatches }), + ...(scorerReasonPatternMismatch === undefined ? {} : { scorerReasonPatternMismatch }), + }); res.writeHead(400, { "content-type": "application/json" }); res.end(JSON.stringify({ error: "review request metadata is missing or invalid" })); return; @@ -1064,10 +1779,14 @@ export async function startScorerProxy( }); const text = await upstream.text(); const response = safeJson(text) as { + model?: unknown; + provider?: unknown; + error?: unknown; usage?: { prompt_tokens?: number; completion_tokens?: number; cost?: number }; } | undefined; const usageValid = isValidUsage(response?.usage); attempts.push({ + phase: isAdjudication ? "adjudication" : "scorer", outcome: "completed", durationMs: performance.now() - startedAt, promptTokens: Number(response?.usage?.prompt_tokens ?? 0), @@ -1078,11 +1797,17 @@ export async function startScorerProxy( : null, costProviderDecimal: providerCostDecimalFromResponse(text), usageValid, + httpStatus: upstream.status, + modelIdentityPresent: typeof response?.model === "string" && response.model.length > 0, + providerIdentityPresent: typeof response?.provider === "string" && response.provider.length > 0, + usagePresent: typeof response?.usage === "object" && response.usage !== null, + errorPresent: response?.error !== undefined, }); res.writeHead(upstream.status, { "content-type": upstream.headers.get("content-type") ?? "application/json" }); res.end(text); } catch { attempts.push({ + phase: isAdjudication ? "adjudication" : "scorer", outcome: closing ? "teardownAborted" : deadlineExceeded ? "timedOut" : "failed", durationMs: performance.now() - startedAt, promptTokens: 0, @@ -1090,6 +1815,11 @@ export async function startScorerProxy( costUsd: null, costProviderDecimal: null, usageValid: false, + httpStatus: null, + modelIdentityPresent: false, + providerIdentityPresent: false, + usagePresent: false, + errorPresent: false, }); if (!res.destroyed && !res.headersSent) { res.writeHead(closing ? 503 : 504, { "content-type": "application/json" }); @@ -1501,10 +2231,18 @@ export function aggregate( const structuredFailures = cases.filter((c) => isAdmissionFatalStructuralResult(c, model)).length; const trueCases = cases.filter((c) => c.scenario === "trueFinding"); const falseCases = cases.filter((c) => c.scenario === "falseFinding"); - const trueConf = trueCases.map((c) => c.scorerConfidence).filter((v): v is number => v !== null); - const falseConf = falseCases.map((c) => c.scorerConfidence).filter((v): v is number => v !== null); - const trueFindingHighConfidence = trueCases.filter((c) => c.passed).length; - const falseFindingDownscored = falseCases.filter((c) => c.passed).length; + const eligibleTrueCases = trueCases.filter((c) => + !isAdmissionFatalStructuralResult(c, model) + ); + const eligibleFalseCases = falseCases.filter((c) => + !isAdmissionFatalStructuralResult(c, model) + ); + const trueConf = eligibleTrueCases.map((c) => c.scorerConfidence) + .filter((v): v is number => v !== null); + const falseConf = eligibleFalseCases.map((c) => c.scorerConfidence) + .filter((v): v is number => v !== null); + const trueFindingHighConfidence = eligibleTrueCases.filter((c) => c.passed).length; + const falseFindingDownscored = eligibleFalseCases.filter((c) => c.passed).length; const durations = cases.map((c) => c.durationMs).filter((value): value is number => value !== null); const costs = cases.map((c) => c.costUsd).filter((value): value is number => value !== null); const reasonContractFailures = cases.filter((c) => !c.reasonContractValid).length; @@ -1523,34 +2261,41 @@ export function aggregate( } if (structuredFailures > 0) admissionFailures.push(`${structuredFailures} structured-output failure(s)`); if (timedOutCases > 0) admissionFailures.push(`${timedOutCases} case timeout(s)`); - if (trueFindingHighConfidence !== trueCases.length) { - admissionFailures.push( - `${trueCases.length - trueFindingHighConfidence} true risk(s) were not preserved as published, gate-failing risks`, - ); - } - const requiredFalseDownscores = Math.ceil(falseCases.length * SCORER_MIN_FALSE_DOWNSCORE_RATE); - if (falseFindingDownscored < requiredFalseDownscores) { - admissionFailures.push( - `only ${falseFindingDownscored}/${falseCases.length} false findings were down-scored; need ${requiredFalseDownscores}`, + const structuralPass = matrixComplete && structuredFailures === 0; + if (structuralPass) { + if (trueFindingHighConfidence !== eligibleTrueCases.length) { + admissionFailures.push( + `${eligibleTrueCases.length - trueFindingHighConfidence} true risk(s) were not preserved as published, gate-failing risks`, + ); + } + const requiredFalseDownscores = Math.ceil( + eligibleFalseCases.length * SCORER_MIN_FALSE_DOWNSCORE_RATE, ); - } - const perFixtureRequired = Math.ceil(repeats * SCORER_MIN_FALSE_DOWNSCORE_RATE); - for (const id of FALSE_FINDING_CASES) { - const fixtureCases = falseCases.filter((c) => c.id === id); - const downscored = fixtureCases.filter((c) => c.passed).length; - if (fixtureCases.length !== repeats || downscored < perFixtureRequired) { - admissionFailures.push(`${id} down-scored ${downscored}/${fixtureCases.length}; need ${perFixtureRequired}/${repeats}`); + if (falseFindingDownscored < requiredFalseDownscores) { + admissionFailures.push( + `only ${falseFindingDownscored}/${eligibleFalseCases.length} false findings were down-scored; need ${requiredFalseDownscores}`, + ); + } + const perFixtureRequired = Math.ceil(repeats * SCORER_MIN_FALSE_DOWNSCORE_RATE); + for (const id of FALSE_FINDING_CASES) { + const fixtureCases = eligibleFalseCases.filter((c) => c.id === id); + const downscored = fixtureCases.filter((c) => c.passed).length; + if (downscored < perFixtureRequired) { + admissionFailures.push( + `${id} down-scored ${downscored}/${fixtureCases.length}; need ${perFixtureRequired}/${repeats}`, + ); + } } } const pricingKnown = costs.length === cases.length && cases.length > 0; if (!pricingKnown) admissionFailures.push("pricing missing for one or more cases"); - if (p50DurationMs > SCORER_MAX_P50_MS) { + if (structuralPass && p50DurationMs > SCORER_MAX_P50_MS) { admissionFailures.push(`p50 latency ${p50DurationMs.toFixed(0)}ms exceeds ${SCORER_MAX_P50_MS}ms`); } - if (p95DurationMs > SCORER_MAX_P95_MS) { + if (structuralPass && p95DurationMs > SCORER_MAX_P95_MS) { admissionFailures.push(`p95 latency ${p95DurationMs.toFixed(0)}ms exceeds ${SCORER_MAX_P95_MS}ms`); } - if (maxDurationMs > SCORER_MAX_CASE_MS) { + if (structuralPass && maxDurationMs > SCORER_MAX_CASE_MS) { admissionFailures.push(`max latency ${maxDurationMs.toFixed(0)}ms exceeds ${SCORER_MAX_CASE_MS}ms`); } if (pricingKnown && meanCostUsd > SCORER_MAX_MEAN_COST_USD) { @@ -1566,9 +2311,9 @@ export function aggregate( timedOutCases, structuredFailures, trueFindingHighConfidence, - trueFindingCases: trueCases.length, + trueFindingCases: eligibleTrueCases.length, falseFindingDownscored, - falseFindingCases: falseCases.length, + falseFindingCases: eligibleFalseCases.length, meanTrueConfidence: mean(trueConf), meanFalseConfidence: mean(falseConf), reasonContractFailures, @@ -1680,7 +2425,12 @@ function readRequestBody(req: IncomingMessage): Promise { } if (import.meta.main) { - main().catch((err) => { + main().catch(async (err) => { + try { + await writeScorerEvalSetupFailureArtifact(process.argv.slice(2)); + } catch { + // Preserve the original setup failure when the diagnostic artifact cannot be written. + } console.error(err instanceof Error ? err.message : String(err)); process.exitCode = 1; }); diff --git a/bench/src/verify-admission.ts b/bench/src/verify-admission.ts index bb9fbc2..74e66fe 100644 --- a/bench/src/verify-admission.ts +++ b/bench/src/verify-admission.ts @@ -66,6 +66,7 @@ const modelDefaultsSchema = z.object({ const provisionalProfileSchema = z.object({ benchmarkProviderIdentity: z.literal("openrouter:managed-routing"), upstreamProviderIdentity: boundedIdentifierSchema, + upstreamProviderRoute: boundedIdentifierSchema, apiBase: z.literal("https://openrouter.ai:443/api/v1"), apiFormat: z.literal("openai-compatible"), generatorChain: z.array(boundedIdentifierSchema).min(1), diff --git a/provisional-models.json b/provisional-models.json index d7e248c..7782159 100644 --- a/provisional-models.json +++ b/provisional-models.json @@ -1,6 +1,7 @@ { "benchmarkProviderIdentity": "openrouter:managed-routing", "upstreamProviderIdentity": "Azure", + "upstreamProviderRoute": "azure/eu", "apiBase": "https://openrouter.ai:443/api/v1", "apiFormat": "openai-compatible", "generatorChain": ["openai/gpt-5.6-luna"], diff --git a/src/config.rs b/src/config.rs index 1570b5e..91b67cd 100644 --- a/src/config.rs +++ b/src/config.rs @@ -201,6 +201,8 @@ pub struct QualificationProfile { #[serde(default)] pub benchmark_provider_identity: Option, pub upstream_provider_identity: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub upstream_provider_route: Option, pub generator_chain: Vec, pub consensus: usize, pub scorer_chain: Vec, @@ -227,6 +229,7 @@ pub struct ModelPriceBound { pub(crate) struct QualificationCandidateProfile { pub benchmark_provider_identity: String, pub upstream_provider_identity: String, + pub upstream_provider_route: String, pub api_base: String, pub api_format: ApiFormat, pub generator_chain: Vec, @@ -242,6 +245,8 @@ struct QualificationProfileDigestMaterial<'a> { model_defaults_sha256: &'a str, benchmark_provider_identity: &'a Option, upstream_provider_identity: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + upstream_provider_route: &'a Option, api_base: &'a str, api_format: ApiFormat, generator_chain: &'a [String], @@ -628,6 +633,9 @@ fn parse_qualification_manifest(raw: &str) -> Result { !profile.upstream_provider_identity.trim().is_empty(), "qualification profile upstream provider identity must not be empty" ); + if let Some(route) = profile.upstream_provider_route.as_deref() { + validate_model_id("qualification profile upstreamProviderRoute", route)?; + } anyhow::ensure!( (1..=profile.generator_chain.len()).contains(&profile.consensus), "qualification profile consensus must fit its generator chain" @@ -901,6 +909,7 @@ fn qualification_profile_digest(profile: &QualificationProfile) -> String { model_defaults_sha256: &profile.model_defaults_sha256, benchmark_provider_identity: &profile.benchmark_provider_identity, upstream_provider_identity: &profile.upstream_provider_identity, + upstream_provider_route: &profile.upstream_provider_route, api_base: &profile.api_base, api_format: profile.api_format, generator_chain: &profile.generator_chain, @@ -1782,7 +1791,7 @@ fn validate_benchmark_bounded_selection_values( ) -> Result<()> { anyhow::ensure!( feature_enabled, - "benchmark bounded selection requires the non-default qualification-candidate feature" + "benchmark bounded selection requires the qualification-candidate build capability" ); anyhow::ensure!( ci == Some("true"), @@ -1882,19 +1891,51 @@ pub(crate) fn benchmark_screening_profile_for_config( let Some(profile) = benchmark_screening_profile()? else { return Ok(None); }; - let api_base = normalize_api_base(&config.api_base)?; + let mismatches = benchmark_screening_profile_mismatches(&profile, config)?; anyhow::ensure!( - profile.generator_chain == config.model_chain() - && profile.consensus == config.consensus - && profile.scorer_chain == config.scorer_chain() - && profile.api_base == api_base - && profile.api_format == config.api_format - && reasoning_efforts_match_embedded_defaults(config), - "benchmark screening profile does not exactly match the resolved review configuration" + mismatches.is_empty(), + "benchmark screening profile does not exactly match the resolved review configuration: {}", + mismatches.join(", ") ); Ok(Some(profile)) } +#[cfg(test)] +fn benchmark_screening_profile_matches_config( + profile: &QualificationCandidateProfile, + config: &Config, +) -> Result { + Ok(benchmark_screening_profile_mismatches(profile, config)?.is_empty()) +} + +fn benchmark_screening_profile_mismatches( + profile: &QualificationCandidateProfile, + config: &Config, +) -> Result> { + let active_scorer_chain = config.scorer_chain(); + let normalized_api_base = normalize_api_base(&config.api_base)?; + let mut mismatches = Vec::new(); + if profile.generator_chain != config.model_chain() { + mismatches.push("generator chain"); + } + if profile.consensus != config.consensus { + mismatches.push("consensus"); + } + if !active_scorer_chain.is_empty() && profile.scorer_chain != active_scorer_chain { + mismatches.push("scorer chain"); + } + if profile.api_base != normalized_api_base { + mismatches.push("API base"); + } + if profile.api_format != config.api_format { + mismatches.push("API format"); + } + if !reasoning_efforts_match_embedded_defaults(config) { + mismatches.push("reasoning effort"); + } + Ok(mismatches) +} + pub(crate) fn qualification_plan_only() -> bool { qualification_candidate_mode() && std::env::var("POSTIL_QUALIFICATION_PLAN_ONLY") @@ -1947,6 +1988,10 @@ fn qualification_candidate_profile() -> Result Option< }) } +fn qualified_provider_route(profile: &QualificationProfile) -> &str { + profile + .upstream_provider_route + .as_deref() + .unwrap_or(profile.upstream_provider_identity.as_str()) +} + +pub(crate) fn hosted_provider_route_for_config(config: &Config) -> Option<&'static str> { + if hosted_mode() + && provisional_hosted_roster_enabled() + && provisional_hosted_profile_for_config(config).is_some() + { + return Some( + provisional_hosted_profile() + .upstream_provider_route + .as_str(), + ); + } + let profile = hosted_mode() + .then(|| admitted_profile_for_config(config)) + .flatten()?; + Some(qualified_provider_route(profile)) +} + fn provisional_hosted_profile_for_config(config: &Config) -> Option { let profile = provisional_hosted_profile(); let generator_chain = config.model_chain(); @@ -2059,6 +2128,10 @@ fn parse_provisional_hosted_profile(raw: &str) -> Result, +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct RawUncertaintyResolution { @@ -614,6 +620,7 @@ struct RequestDecorations { require_openrouter_privacy: bool, hosted_price_bounds: Option>>, pinned_upstream_provider: Option, + pinned_upstream_provider_route: Option, } #[derive(Debug, Default)] @@ -696,6 +703,13 @@ pub(crate) const MAX_PROVIDER_REQUEST_BYTES: usize = 256 * 1024; // logical review call. Keep the output ceiling above the five-batch hosted // plan while the aggregate token and cost caps remain authoritative. pub(crate) const MAX_PROVIDER_OUTPUT_TOKEN_EXPOSURE: usize = 10_000_000; +// Admission prices every executable retry using serialized bytes as a safe +// upper bound on input tokens. That worst-case projection is distinct from +// the exact provider-reported usage cap: a bounded schedule can project above +// the latter without being able to exceed the independent input, output, +// attempt, or cost ceilings. +const MAX_PROVIDER_TOKEN_EXPOSURE: usize = + MAX_PROVIDER_INPUT_BYTES + MAX_PROVIDER_OUTPUT_TOKEN_EXPOSURE; // Hosted planning reserves one initial and one correction call for every // selected model request plus the maximum enabled resolution and compression // passes. The selected-batch limit is derived after all model fan-out is known. @@ -900,7 +914,7 @@ fn review_validation_retry_user(user: &str, previous: &str, reason: &str) -> Str fn scorer_repair_system(system: &str) -> String { format!( - "{system}\n\nYour previous response failed schema validation. Repair only the JSON schema. Kind is a category, so severity values such as info, warn, and error are invalid kinds. Every reason must be concise single-line text of at most {SCORER_REASON_PROMPT_MAX_BYTES} UTF-8 bytes ending in sentence punctuation. Return the complete array and nothing else." + "{system}\n\nYour previous response failed schema validation. Repair only the JSON schema. Kind is a category, so severity values such as info, warn, and error are invalid kinds. Every reason must be concise single-line text of at most {SCORER_REASON_PROMPT_MAX_BYTES} UTF-8 bytes ending in sentence punctuation. Return the complete object with exactly the `scores` array and nothing else." ) } @@ -1323,8 +1337,8 @@ impl RequestDecorations { .ok_or_else(|| anyhow!("qualification provider profile is unavailable"))? .upstream_provider_identity, ) - } else if let Some(profile) = screening_profile { - Some(profile.upstream_provider_identity) + } else if let Some(profile) = screening_profile.as_ref() { + Some(profile.upstream_provider_identity.clone()) } else if crate::config::hosted_mode() { Some( crate::config::admitted_profile_for_config(cfg) @@ -1337,6 +1351,20 @@ impl RequestDecorations { } else { None }; + let pinned_upstream_provider_route = + if let Some(route) = crate::config::hosted_provider_route_for_config(cfg) { + Some(route.to_string()) + } else if crate::config::qualification_candidate_mode() { + Some( + crate::config::qualification_candidate_profile_for_config(cfg)? + .ok_or_else(|| anyhow!("qualification provider profile is unavailable"))? + .upstream_provider_route, + ) + } else if let Some(profile) = screening_profile.as_ref() { + Some(profile.upstream_provider_route.clone()) + } else { + pinned_upstream_provider.clone() + }; Ok(Self { api_base: cfg.api_base.trim_end_matches('/').to_string(), api_format: cfg.api_format, @@ -1348,6 +1376,7 @@ impl RequestDecorations { || env_flag("POSTIL_BENCH_REQUIRE_HOSTED_PROVIDER_PRIVACY")), hosted_price_bounds, pinned_upstream_provider, + pinned_upstream_provider_route, }) } @@ -1378,15 +1407,30 @@ impl RequestDecorations { ); apply_openrouter_privacy(&mut body, self.require_openrouter_privacy); let canonical_openrouter = is_canonical_openrouter_base(&self.api_base); - if let Some(provider) = self.pinned_upstream_provider.as_deref() { + if let Some(provider) = self.pinned_upstream_provider_route.as_deref() { apply_openrouter_provider_pin(&mut body, provider); } - if canonical_openrouter && let LlmPhase::Scorer { expected_len } = phase { - apply_openrouter_scorer_contract(&mut body, expected_len); + if canonical_openrouter { + match phase { + LlmPhase::Scorer { expected_len } => { + apply_openrouter_scorer_contract( + &mut body, + expected_len, + self.pinned_upstream_provider.as_deref(), + ); + } + LlmPhase::Adjudication => { + apply_openrouter_adjudication_contract( + &mut body, + self.pinned_upstream_provider.as_deref(), + ); + } + _ => {} + } } #[cfg(feature = "qualification-candidate")] if matches!(phase, LlmPhase::Attribution) { - apply_openrouter_atomic_attribution_contract(&mut body, _expected_provider); + apply_openrouter_atomic_attribution_contract(&mut body); } if canonical_openrouter && let Some(bound) = self @@ -2237,8 +2281,8 @@ impl LlmClient { .checked_add(exposure.output_tokens) .context("planned token exposure overflowed")?; ensure!( - token_exposure <= MAX_REPORTED_TOKEN_SPEND, - "hosted {operation} admission needs {token_exposure} tokens of exposure, exceeding the {MAX_REPORTED_TOKEN_SPEND} token cap" + token_exposure <= MAX_PROVIDER_TOKEN_EXPOSURE, + "hosted {operation} admission needs {token_exposure} tokens of exposure, exceeding the {MAX_PROVIDER_TOKEN_EXPOSURE} token exposure cap" ); let model_costs = exposure .model_costs_micros @@ -4887,9 +4931,10 @@ impl LlmClient { )?; let output_tokens = body .get("max_tokens") + .or_else(|| body.get("max_completion_tokens")) .and_then(serde_json::Value::as_u64) .and_then(|value| usize::try_from(value).ok()) - .ok_or_else(|| anyhow!("model request is missing a bounded max_tokens value"))?; + .ok_or_else(|| anyhow!("model request is missing a bounded output token limit"))?; let projected_cost_micros = if let Some(bounds) = &self.request_decorations.hosted_price_bounds { @@ -4952,8 +4997,8 @@ impl LlmClient { "model provider output exposure hard cap ({MAX_PROVIDER_OUTPUT_TOKEN_EXPOSURE} tokens) exceeded" ); ensure!( - total_token_exposure <= MAX_REPORTED_TOKEN_SPEND, - "model token spend exposure exceeded the {MAX_REPORTED_TOKEN_SPEND} token hard cap" + total_token_exposure <= MAX_PROVIDER_TOKEN_EXPOSURE, + "model token exposure exceeded the {MAX_PROVIDER_TOKEN_EXPOSURE} token hard cap" ); ensure!( total_projected_cost <= HOSTED_OPERATION_COST_CAP_MICROS, @@ -5266,8 +5311,18 @@ fn apply_anthropic_reasoning(body: &mut serde_json::Value, effort: ReasoningEffo } } -fn apply_openrouter_scorer_contract(body: &mut serde_json::Value, expected_len: usize) { +fn apply_openrouter_scorer_contract( + body: &mut serde_json::Value, + expected_len: usize, + pinned_provider: Option<&str>, +) { debug_assert!(expected_len <= SCORER_MAX_FINDINGS); + // Strict scorer routing requires every request parameter to be supported + // by the selected endpoint. Reasoning endpoints expose their output limit + // through `reasoning.effort`. Azure names the output limit + // `max_completion_tokens`; OpenAI names the same limit `max_tokens`. + // A redundant temperature would disqualify both endpoint families. + apply_openrouter_strict_output_limit(body, "scorer", pinned_provider); let provider = body .as_object_mut() .expect("model request body is an object") @@ -5288,50 +5343,87 @@ fn apply_openrouter_scorer_contract(body: &mut serde_json::Value, expected_len: "name": "postil_finding_scores", "strict": true, "schema": { - "type": "array", - "minItems": expected_len, - "maxItems": expected_len, - "items": { - "type": "object", - "properties": { - "confidence": { - "type": "number", - "minimum": 0, - "maximum": 1, - }, - "kind": { - "type": "string", - "enum": [ - "risk", - "humanEscalation", - "guardrail", - "uncertainty", - "contentPolicy", - ], - }, - "reason": { - "type": "string", - "minLength": 1, - "maxLength": SCORER_REASON_SCHEMA_MAX_CHARS, - "pattern": SCORER_REASON_JSON_PATTERN, + "type": "object", + "properties": { + "scores": { + "type": "array", + "minItems": expected_len, + "maxItems": expected_len, + "items": { + "type": "object", + "properties": { + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1, + }, + "kind": { + "type": "string", + "enum": [ + "risk", + "humanEscalation", + "guardrail", + "uncertainty", + "contentPolicy", + ], + }, + "reason": { + "type": "string", + "minLength": 1, + "maxLength": SCORER_REASON_SCHEMA_MAX_CHARS, + "pattern": SCORER_REASON_JSON_PATTERN, + }, + }, + "required": ["confidence", "kind", "reason"], + "additionalProperties": false, }, }, - "required": ["confidence", "kind", "reason"], - "additionalProperties": false, }, + "required": ["scores"], + "additionalProperties": false, }, }, }); } -#[cfg(feature = "qualification-candidate")] -fn apply_openrouter_atomic_attribution_contract( +fn apply_openrouter_adjudication_contract( + body: &mut serde_json::Value, + pinned_provider: Option<&str>, +) { + apply_openrouter_strict_output_limit(body, "adjudication", pinned_provider); + let provider = body + .as_object_mut() + .expect("model request body is an object") + .entry("provider") + .or_insert_with(|| json!({})); + provider + .as_object_mut() + .expect("provider routing configuration is an object") + .insert("require_parameters".to_string(), json!(true)); + if body["reasoning"]["effort"] != "none" { + body["reasoning"]["exclude"] = json!(true); + } +} + +fn apply_openrouter_strict_output_limit( body: &mut serde_json::Value, - expected_provider: Option<&str>, + phase: &str, + pinned_provider: Option<&str>, ) { - if let Some(expected_provider) = expected_provider { - apply_openrouter_provider_pin(body, expected_provider); + let request = body + .as_object_mut() + .expect("model request body is an object"); + request.remove("temperature"); + if pinned_provider != Some("OpenAI") { + let max_tokens = request.remove("max_tokens").unwrap_or_else(|| { + panic!("OpenRouter {phase} request has a bounded output token limit") + }); + request.insert("max_completion_tokens".to_string(), max_tokens); } +} + +#[cfg(feature = "qualification-candidate")] +fn apply_openrouter_atomic_attribution_contract(body: &mut serde_json::Value) { let provider = body .as_object_mut() .expect("model request body is an object") @@ -5394,7 +5486,7 @@ fn is_canonical_openrouter_base(api_base: &str) -> bool { reqwest::Url::parse(api_base).is_ok_and(|url| { url.scheme() == "https" && url.host_str() == Some("openrouter.ai") - && url.port().is_none() + && url.port().is_none_or(|port| port == 443) && url.path().trim_end_matches('/') == "/api/v1" && url.query().is_none() && url.fragment().is_none() @@ -5962,8 +6054,10 @@ fn safe_review_output_shape(content: &str) -> String { } fn parse_scores(content: &str, expected_len: usize) -> Result, String> { - let json_str = extract_json_array(content).ok_or("no JSON array found")?; - let raw = serde_json::from_str::>(json_str).map_err(|e| e.to_string())?; + let json_str = extract_json_object(content).ok_or("no JSON object found")?; + let raw = serde_json::from_str::(json_str) + .map_err(|e| e.to_string())? + .scores; if raw.len() != expected_len { return Err(format!( "expected {expected_len} score(s), got {}", @@ -7059,6 +7153,7 @@ mod tests { .unwrap(); client.request_decorations.require_openrouter_privacy = true; client.request_decorations.pinned_upstream_provider = Some("p".repeat(12_000)); + client.request_decorations.pinned_upstream_provider_route = Some("p".repeat(12_000)); client.request_decorations.hosted_price_bounds = Some(Arc::new(HashMap::from([( config.model.clone(), ModelPriceBound { @@ -8223,8 +8318,10 @@ mod tests { output_micros_per_million_tokens: 1, }, )]))); - let users = - vec!["bounded candidate".to_string(); crate::review::MAX_HOSTED_SELECTED_BATCHES]; + let users = vec![ + "\"".repeat(crate::review::MAX_HOSTED_REVIEW_BATCH_BYTES); + crate::review::MAX_HOSTED_SELECTED_BATCHES + ]; let output_tokens = vec![REVIEW_MAX_TOKENS; crate::review::MAX_HOSTED_SELECTED_BATCHES]; let manifest = "m".repeat(96_000); let admission = client @@ -8304,8 +8401,10 @@ mod tests { .map(|bound| (bound.model.clone(), bound)) .collect(), )); - let users = - vec!["bounded candidate".to_string(); crate::review::MAX_HOSTED_SELECTED_BATCHES]; + let users = vec![ + "\"".repeat(crate::review::MAX_HOSTED_REVIEW_BATCH_BYTES); + crate::review::MAX_HOSTED_SELECTED_BATCHES + ]; let output_tokens = vec![REVIEW_MAX_TOKENS; crate::review::MAX_HOSTED_SELECTED_BATCHES]; let manifest = "m".repeat(96_000); let admission = client @@ -8332,6 +8431,11 @@ mod tests { admission.projected_cost_micros, HOSTED_ADMISSION_PROJECTION_CAP_MICROS ); + assert!( + admission.serialized_input_bytes + admission.output_tokens + > MAX_REPORTED_TOKEN_SPEND as u64, + "the regression fixture must exercise projection above the exact usage cap" + ); } #[test] @@ -8594,6 +8698,55 @@ mod tests { ))); } + #[test] + fn hosted_token_projection_is_distinct_from_reported_usage() { + let mut client = LlmClient::build( + &Config::default(), + "test-key".into(), + Duration::from_secs(1), + None, + None, + ) + .unwrap(); + client.request_decorations.hosted_price_bounds = Some(Arc::new(HashMap::new())); + + let planned = PlannedExposure { + attempts: 190, + input_bytes: 17_961_194, + output_tokens: 2_091_456, + projected_cost_micros: 0, + model_costs_micros: BTreeMap::new(), + }; + let admission = client + .validate_hosted_exposure("review", &planned) + .expect("the bounded hosted schedule must clear projection admission"); + assert_eq!(admission.provider_attempts, 190); + assert_eq!(admission.serialized_input_bytes, 17_961_194); + assert_eq!(admission.output_tokens, 2_091_456); + assert!( + admission.serialized_input_bytes + admission.output_tokens + > MAX_REPORTED_TOKEN_SPEND as u64 + ); + + client + .record_reported_usage(Usage { + prompt_tokens: MAX_REPORTED_TOKEN_SPEND as u64, + ..Usage::default() + }) + .expect("the exact reported-usage boundary is admitted"); + let error = client + .record_reported_usage(Usage { + prompt_tokens: 1, + ..Usage::default() + }) + .unwrap_err(); + assert!(error.to_string().contains("token hard cap")); + assert_eq!( + client.admission.lock().unwrap().reported_token_spend, + MAX_REPORTED_TOKEN_SPEND + ); + } + #[test] fn planned_exposure_aggregates_atomic_per_model_transport_costs() { let first = ModelPriceBound { @@ -8909,7 +9062,7 @@ mod tests { #[test] fn scorer_scores_use_array_order_and_validate_kind() { let scores = parse_scores( - r#"[{"confidence":0.8,"kind":"humanEscalation","reason":"This needs an owner decision."},{"confidence":0.6,"kind":"risk","reason":"This follows the second input."}]"#, + r#"{"scores":[{"confidence":0.8,"kind":"humanEscalation","reason":"This needs an owner decision."},{"confidence":0.6,"kind":"risk","reason":"This follows the second input."}]}"#, 2, ) .unwrap(); @@ -8924,15 +9077,30 @@ mod tests { #[test] fn scorer_rejects_unknown_fields_and_invalid_confidence() { let unknown = parse_scores( - r#"[{"index":0,"confidence":0.8,"kind":"risk","reason":"This field is not admitted."}]"#, + r#"{"scores":[{"index":0,"confidence":0.8,"kind":"risk","reason":"This field is not admitted."}]}"#, 1, ) .unwrap_err(); assert!(unknown.contains("unknown field `index`")); + let unknown_root = parse_scores( + r#"{"scores":[{"confidence":0.8,"kind":"risk","reason":"This score is valid."}],"extra":true}"#, + 1, + ) + .unwrap_err(); + assert!(unknown_root.contains("unknown field `extra`")); + assert!(parse_scores(r#"{"items":[]}"#, 0).is_err()); + assert!( + parse_scores( + r#"[{"confidence":0.8,"kind":"risk","reason":"Arrays are no longer the root."}]"#, + 1 + ) + .is_err() + ); + for confidence in ["-1", "5", "1e999", "\"NaN\""] { let input = format!( - r#"[{{"confidence":{confidence},"kind":"risk","reason":"This confidence is invalid."}}]"# + r#"{{"scores":[{{"confidence":{confidence},"kind":"risk","reason":"This confidence is invalid."}}]}}"# ); assert!( parse_scores(&input, 1).is_err(), @@ -8944,7 +9112,7 @@ mod tests { #[test] fn scorer_rejects_severity_label_as_kind() { let error = parse_scores( - r#"[{"confidence":0.7,"kind":"warn","reason":"The response used the wrong field."}]"#, + r#"{"scores":[{"confidence":0.7,"kind":"warn","reason":"The response used the wrong field."}]}"#, 1, ) .unwrap_err(); @@ -8955,7 +9123,7 @@ mod tests { fn scorer_rejects_missing_entries() { assert!( parse_scores( - r#"[{"confidence":0.5,"kind":"risk","reason":"Only one score is present."}]"#, + r#"{"scores":[{"confidence":0.5,"kind":"risk","reason":"Only one score is present."}]}"#, 2 ) .is_err() @@ -8973,17 +9141,18 @@ mod tests { #[test] fn scorer_rejects_incomplete_and_overlength_reasons() { let incomplete = parse_scores( - r#"[{"confidence":0.7,"kind":"risk","reason":"This has no sentence terminator"}]"#, + r#"{"scores":[{"confidence":0.7,"kind":"risk","reason":"This has no sentence terminator"}]}"#, 1, ) .unwrap_err(); assert!(incomplete.contains("sentence punctuation")); - let overlength = serde_json::json!([{ - "confidence": 0.7, - "kind": "risk", - "reason": format!("{}.", "x".repeat(SCORER_REASON_MAX_BYTES)), - }]); + let overlength = serde_json::json!({"scores": [{ + "confidence": 0.7, + "kind": "risk", + "reason": format!("{}.", "x".repeat(SCORER_REASON_MAX_BYTES)), + }] + }); let error = parse_scores(&overlength.to_string(), 1).unwrap_err(); assert!(error.contains("exceeds 240 UTF-8 bytes")); @@ -8993,11 +9162,12 @@ mod tests { "A tab\tis invalid.", "A line\u{2028}separator is invalid.", ] { - let input = serde_json::json!([{ - "confidence": 0.7, - "kind": "risk", - "reason": reason, - }]); + let input = serde_json::json!({"scores": [{ + "confidence": 0.7, + "kind": "risk", + "reason": reason, + }] + }); assert!( parse_scores(&input.to_string(), 1).is_err(), "accepted malformed reason {reason:?}" @@ -9008,20 +9178,22 @@ mod tests { #[test] fn scorer_reason_limits_match_json_schema_unicode_length() { let reason = format!("{}.", "x".repeat(SCORER_REASON_MAX_BYTES - 1)); - let input = serde_json::json!([{ - "confidence": 0.7, - "kind": "risk", - "reason": reason, - }]); + let input = serde_json::json!({"scores": [{ + "confidence": 0.7, + "kind": "risk", + "reason": reason, + }] + }); let scores = parse_scores(&input.to_string(), 1).unwrap(); assert_eq!(scores[0].reason.len(), SCORER_REASON_MAX_BYTES); let multibyte = format!("{}。", "界".repeat((SCORER_REASON_MAX_BYTES / 3) - 1)); - let input = serde_json::json!([{ - "confidence": 0.7, - "kind": "risk", - "reason": multibyte, - }]); + let input = serde_json::json!({"scores": [{ + "confidence": 0.7, + "kind": "risk", + "reason": multibyte, + }] + }); let scores = parse_scores(&input.to_string(), 1).unwrap(); assert_eq!(scores[0].reason.len(), SCORER_REASON_MAX_BYTES); } @@ -9029,14 +9201,14 @@ mod tests { #[test] fn scorer_reason_accepts_bounded_single_line_text() { let scores = parse_scores( - r#"[{"confidence":0.7,"kind":"risk","reason":"The U.S. retry path is not idempotent, e.g. on timeout."}]"#, + r#"{"scores":[{"confidence":0.7,"kind":"risk","reason":"The U.S. retry path is not idempotent, e.g. on timeout."}]}"#, 1, ) .unwrap(); assert_eq!(scores.len(), 1); let multiple_sentences = parse_scores( - r#"[{"confidence":0.7,"kind":"risk","reason":"The first condition fails. The second condition also fails."}]"#, + r#"{"scores":[{"confidence":0.7,"kind":"risk","reason":"The first condition fails. The second condition also fails."}]}"#, 1, ) .unwrap(); @@ -9045,29 +9217,30 @@ mod tests { let natural_long_reason = "The authorization check is bypassed when the cached administrator flag is stale."; assert!(natural_long_reason.chars().count() > 60); - let input = serde_json::json!([{ - "confidence": 0.7, - "kind": "risk", - "reason": natural_long_reason, - }]); + let input = serde_json::json!({"scores": [{ + "confidence": 0.7, + "kind": "risk", + "reason": natural_long_reason, + }] + }); assert_eq!(parse_scores(&input.to_string(), 1).unwrap().len(), 1); let lowercase = parse_scores( - r#"[{"confidence":0.7,"kind":"risk","reason":"The first condition fails. the second condition also fails."}]"#, + r#"{"scores":[{"confidence":0.7,"kind":"risk","reason":"The first condition fails. the second condition also fails."}]}"#, 1, ) .unwrap(); assert_eq!(lowercase.len(), 1); let no_space = parse_scores( - r#"[{"confidence":0.7,"kind":"risk","reason":"The first condition fails.The second condition also fails."}]"#, + r#"{"scores":[{"confidence":0.7,"kind":"risk","reason":"The first condition fails.The second condition also fails."}]}"#, 1, ) .unwrap(); assert_eq!(no_space.len(), 1); let file_and_version = parse_scores( - r#"[{"confidence":0.7,"kind":"risk","reason":"The src/lib.rs behavior changed in version 4.2."}]"#, + r#"{"scores":[{"confidence":0.7,"kind":"risk","reason":"The src/lib.rs behavior changed in version 4.2."}]}"#, 1, ) .unwrap(); @@ -9542,7 +9715,10 @@ mod tests { #[test] fn canonical_openrouter_uses_role_defaults_and_scorer_strict_schema() { let client = LlmClient::build( - &Config::default(), + &Config { + api_base: "https://openrouter.ai:443/api/v1".into(), + ..Config::default() + }, "test-key".into(), Duration::from_secs(1), None, @@ -9561,31 +9737,87 @@ mod tests { scorer["reasoning"], json!({"effort": "low", "exclude": true}) ); + assert!(scorer.get("temperature").is_none()); + assert!(scorer.get("max_tokens").is_none()); + assert_eq!(scorer["max_completion_tokens"], 400); assert!(scorer["reasoning"].get("enabled").is_none()); assert_eq!(scorer["provider"]["require_parameters"], true); assert_eq!(scorer["response_format"]["type"], "json_schema"); assert_eq!(scorer["response_format"]["json_schema"]["strict"], true); assert_eq!( - scorer["response_format"]["json_schema"]["schema"]["items"]["additionalProperties"], + scorer["response_format"]["json_schema"]["schema"]["additionalProperties"], false ); let schema = &scorer["response_format"]["json_schema"]["schema"]; - assert_eq!(schema["minItems"], 1); - assert_eq!(schema["maxItems"], 1); - assert!(schema["items"]["properties"].get("index").is_none()); + assert_eq!(schema["type"], "object"); + assert_eq!(schema["required"], json!(["scores"])); + let scores = &schema["properties"]["scores"]; + assert_eq!(scores["minItems"], 1); + assert_eq!(scores["maxItems"], 1); + assert!(scores["items"]["properties"].get("index").is_none()); + assert_eq!(scores["items"]["additionalProperties"], false); assert_eq!( - schema["items"]["required"], + scores["items"]["required"], json!(["confidence", "kind", "reason"]) ); - assert_eq!(schema["items"]["properties"]["reason"]["minLength"], 1); + assert_eq!(scores["items"]["properties"]["reason"]["minLength"], 1); assert_eq!( - schema["items"]["properties"]["reason"]["maxLength"], + scores["items"]["properties"]["reason"]["maxLength"], SCORER_REASON_SCHEMA_MAX_CHARS ); assert_eq!( - schema["items"]["properties"]["reason"]["pattern"], + scores["items"]["properties"]["reason"]["pattern"], SCORER_REASON_JSON_PATTERN ); + let adjudication = client.request_body( + "provider/scorer", + "system", + "user", + 8_000, + 0.0, + LlmPhase::Adjudication, + ); + assert!(adjudication.get("temperature").is_none()); + assert!(adjudication.get("max_tokens").is_none()); + assert_eq!(adjudication["max_completion_tokens"], 8_000); + assert_eq!(adjudication["provider"]["require_parameters"], true); + assert_eq!( + adjudication["reasoning"], + json!({"effort": "low", "exclude": true}) + ); + assert!(adjudication.get("response_format").is_none()); + + let mut openai_client = client.clone(); + openai_client.request_decorations.pinned_upstream_provider = Some("OpenAI".into()); + let openai_scorer = openai_client.request_body( + "provider/scorer", + "system", + "user", + 400, + 0.0, + LlmPhase::Scorer { expected_len: 1 }, + ); + assert_eq!(openai_scorer["max_tokens"], 400); + assert!(openai_scorer.get("max_completion_tokens").is_none()); + assert!(openai_scorer.get("temperature").is_none()); + assert_eq!(openai_scorer["provider"]["require_parameters"], true); + + let openai_adjudication = openai_client.request_body( + "provider/scorer", + "system", + "user", + 8_000, + 0.0, + LlmPhase::Adjudication, + ); + assert_eq!(openai_adjudication["max_tokens"], 8_000); + assert!(openai_adjudication.get("max_completion_tokens").is_none()); + assert!(openai_adjudication.get("temperature").is_none()); + assert_eq!(openai_adjudication["provider"]["require_parameters"], true); + assert_eq!( + openai_adjudication["reasoning"], + json!({"effort": "low", "exclude": true}) + ); let multiple = client.request_body( "provider/scorer", @@ -9596,10 +9828,11 @@ mod tests { LlmPhase::Scorer { expected_len: 7 }, ); let multiple_schema = &multiple["response_format"]["json_schema"]["schema"]; - assert_eq!(multiple_schema["minItems"], 7); - assert_eq!(multiple_schema["maxItems"], 7); + let multiple_scores = &multiple_schema["properties"]["scores"]; + assert_eq!(multiple_scores["minItems"], 7); + assert_eq!(multiple_scores["maxItems"], 7); assert!( - multiple_schema["items"]["properties"] + multiple_scores["items"]["properties"] .get("index") .is_none() ); @@ -10096,7 +10329,7 @@ mod tests { 0.0, LlmPhase::Review, ); - assert_eq!(body["provider"]["order"], json!(["Azure"])); + assert_eq!(body["provider"]["order"], json!(["azure/eu"])); assert_eq!(body["provider"]["allow_fallbacks"], false); assert_eq!(body["provider"]["data_collection"], "deny"); assert_eq!(body["provider"]["zdr"], true); @@ -10133,6 +10366,7 @@ mod tests { ) .unwrap(); client.request_decorations.pinned_upstream_provider = Some("PinnedProvider".into()); + client.request_decorations.pinned_upstream_provider_route = Some("pinned/route".into()); client.request_decorations.hosted_price_bounds = Some(Arc::new(HashMap::from([( "provider/model".into(), ModelPriceBound { @@ -10155,7 +10389,7 @@ mod tests { phase, matches!(phase, LlmPhase::Attribution).then_some("PinnedProvider"), ); - assert_eq!(body["provider"]["order"], json!(["PinnedProvider"])); + assert_eq!(body["provider"]["order"], json!(["pinned/route"])); assert_eq!(body["provider"]["allow_fallbacks"], false); assert_eq!( body["provider"]["max_price"], diff --git a/src/prompt.rs b/src/prompt.rs index 3cdcea2..7b9daf2 100644 --- a/src/prompt.rs +++ b/src/prompt.rs @@ -256,11 +256,12 @@ pub fn scorer_system_prompt(cfg: &Config, current_utc_date: Date) -> String { p.push_str(&format!( "--- END POSTIL REVIEW CONTRACT ---\n\ \n\ - Return ONLY a JSON array, no markdown fences, no prose. The array MUST contain \ - exactly one object per supplied finding, in the same order as the input:\n\ - [{{\"confidence\": <0..1>, \ + Return ONLY a JSON object, no markdown fences, no prose. The root object MUST \ + contain exactly one field, `scores`, whose array contains exactly one object per \ + supplied finding, in the same order as the input:\n\ + {{\"scores\": [{{\"confidence\": <0..1>, \ \"kind\": \"risk|humanEscalation|guardrail|uncertainty|contentPolicy\", \ - \"reason\": \"concise single-line text of at most {SCORER_REASON_PROMPT_MAX_BYTES} UTF-8 bytes\"}}]\n\ + \"reason\": \"concise single-line text of at most {SCORER_REASON_PROMPT_MAX_BYTES} UTF-8 bytes\"}}]}}\n\ \n\ Array position is the finding index. Do not emit an `index` field. The `kind` \ value is a finding category. `info`, `warn`, and `error` are \ @@ -567,6 +568,10 @@ mod tests { #[test] fn scorer_prompt_states_the_exact_reason_limits() { let prompt = scorer_system_prompt(&Config::default(), trusted_date()); + assert!(prompt.contains("Return ONLY a JSON object")); + assert!(prompt.contains("exactly one field, `scores`")); + assert!(prompt.contains("{\"scores\": [{\"confidence\": <0..1>")); + assert!(!prompt.contains("Return ONLY a JSON array")); assert!(prompt.contains(&format!( "at most {SCORER_REASON_PROMPT_MAX_BYTES} UTF-8 bytes" ))); diff --git a/tests/e2e.rs b/tests/e2e.rs index 47008d4..51b9472 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -382,7 +382,13 @@ fn request_system_contains(request: &Request, needle: &str) -> bool { } fn scorer_content(scores: Value) -> Value { - scorer_text(&scores.to_string()) + // Scorer responses use the strict root object contract; adjudication has + // a separate array contract and continues to use scorer_text directly. + scorer_text(&json!({"scores": scores}).to_string()) +} + +fn scorer_scores_text(scores: Value) -> String { + json!({"scores": scores}).to_string() } fn scorer_text(scores: &str) -> Value { @@ -412,6 +418,7 @@ fn write_atomic_attribution_inputs( json!({ "benchmarkProviderIdentity": postil_cli::config::MANAGED_OPENROUTER_PROVIDER_IDENTITY, "upstreamProviderIdentity": "test-provider", + "upstreamProviderRoute": "test-provider", "apiBase": postil_cli::config::MANAGED_OPENROUTER_API_BASE, "apiFormat": "openai-compatible", "generatorChain": ["openai/gpt-5-mini"], @@ -2100,18 +2107,15 @@ async fn native_anthropic_findings_use_explicit_native_scorer() { Mock::given(method("POST")) .and(path("/messages")) .and(body_string_contains("\"model\":\"claude-haiku-4-5\"")) - .respond_with( - ResponseTemplate::new(200).set_body_json(anthropic_text( - &json!([{ - "confidence": 0.82, - "kind": "risk", - "reason": "The changed line contains the reported flow." - }]) - .to_string(), - 5, - 3, - )), - ) + .respond_with(ResponseTemplate::new(200).set_body_json(anthropic_text( + &scorer_scores_text(json!([{ + "confidence": 0.82, + "kind": "risk", + "reason": "The changed line contains the reported flow." + }])), + 5, + 3, + ))) .expect(1) .mount(&server) .await; @@ -2198,11 +2202,11 @@ async fn openai_successful_scorer_with_zero_usage_marks_accounting_incomplete() ) .await; let scorer_response = json!({ - "choices": [{"finish_reason": "stop", "message": {"content": json!([{ + "choices": [{"finish_reason": "stop", "message": {"content": scorer_scores_text(json!([{ "confidence": 0.82, "kind": "risk", "reason": "The changed line contains the reported flow." - }]).to_string()}}], + }]))}}], "usage": {"prompt_tokens": 0, "completion_tokens": 0} }); Mock::given(method("POST")) @@ -2308,18 +2312,15 @@ async fn anthropic_successful_scorer_with_zero_usage_marks_accounting_incomplete Mock::given(method("POST")) .and(path("/messages")) .and(body_string_contains("\"model\":\"scorer-model\"")) - .respond_with( - ResponseTemplate::new(200).set_body_json(anthropic_text( - &json!([{ - "confidence": 0.82, - "kind": "risk", - "reason": "The changed line contains the reported flow." - }]) - .to_string(), - 0, - 0, - )), - ) + .respond_with(ResponseTemplate::new(200).set_body_json(anthropic_text( + &scorer_scores_text(json!([{ + "confidence": 0.82, + "kind": "risk", + "reason": "The changed line contains the reported flow." + }])), + 0, + 0, + ))) .expect(1) .mount(&server) .await; @@ -2552,6 +2553,7 @@ fn qualification_candidate_admits_semantically_complete_bounded_hosted_path_with serde_json::to_vec(&json!({ "benchmarkProviderIdentity": postil_cli::config::MANAGED_OPENROUTER_PROVIDER_IDENTITY, "upstreamProviderIdentity": "test-provider", + "upstreamProviderRoute": "test-provider", "apiBase": metadata.default_api_base, "apiFormat": metadata.default_api_format, "generatorChain": generator_chain, @@ -2616,6 +2618,7 @@ fn qualification_candidate_admits_complete_large_review_inside_watchdog_capacity serde_json::to_vec(&json!({ "benchmarkProviderIdentity": postil_cli::config::MANAGED_OPENROUTER_PROVIDER_IDENTITY, "upstreamProviderIdentity": "test-provider", + "upstreamProviderRoute": "test-provider", "apiBase": metadata.default_api_base, "apiFormat": metadata.default_api_format, "generatorChain": [model], @@ -2686,6 +2689,7 @@ fn qualification_candidate_splits_json_escaped_batches_within_model_context() { serde_json::to_vec(&json!({ "benchmarkProviderIdentity": postil_cli::config::MANAGED_OPENROUTER_PROVIDER_IDENTITY, "upstreamProviderIdentity": "test-provider", + "upstreamProviderRoute": "test-provider", "apiBase": metadata.default_api_base, "apiFormat": metadata.default_api_format, "generatorChain": ["openai/gpt-5-mini"], @@ -2843,6 +2847,7 @@ async fn qualification_candidate_covers_fixture_51_shape_before_plan_registratio serde_json::to_vec(&json!({ "benchmarkProviderIdentity": postil_cli::config::MANAGED_OPENROUTER_PROVIDER_IDENTITY, "upstreamProviderIdentity": "Fireworks", + "upstreamProviderRoute": "Fireworks", "apiBase": metadata.default_api_base, "apiFormat": metadata.default_api_format, "generatorChain": ["deepseek/deepseek-v4-pro"], @@ -2921,6 +2926,7 @@ async fn hosted_cost_rejection_precedes_durable_plan_registration() { serde_json::to_vec(&json!({ "benchmarkProviderIdentity": postil_cli::config::MANAGED_OPENROUTER_PROVIDER_IDENTITY, "upstreamProviderIdentity": "test-provider", + "upstreamProviderRoute": "test-provider", "apiBase": metadata.default_api_base, "apiFormat": metadata.default_api_format, "generatorChain": [model], @@ -6949,6 +6955,7 @@ async fn hidden_atomic_attribution_repairs_once_with_same_model_and_preserves_ra json!({ "benchmarkProviderIdentity": postil_cli::config::MANAGED_OPENROUTER_PROVIDER_IDENTITY, "upstreamProviderIdentity": "test-provider", + "upstreamProviderRoute": "test-provider", "apiBase": postil_cli::config::MANAGED_OPENROUTER_API_BASE, "apiFormat": "openai-compatible", "generatorChain": ["openai/gpt-5-mini"], @@ -7181,6 +7188,7 @@ async fn hidden_atomic_attribution_rejects_oversized_repair_before_second_provid json!({ "benchmarkProviderIdentity": postil_cli::config::MANAGED_OPENROUTER_PROVIDER_IDENTITY, "upstreamProviderIdentity": "test-provider", + "upstreamProviderRoute": "test-provider", "apiBase": postil_cli::config::MANAGED_OPENROUTER_API_BASE, "apiFormat": "openai-compatible", "generatorChain": ["openai/gpt-5-mini"], @@ -7613,49 +7621,48 @@ async fn generic_provider_repairs_each_malformed_ordered_scorer_shape() { "kind": "risk", "reason": "This is a concrete defect." }]); - let byte_overflow = json!([{ + let byte_overflow = scorer_scores_text(json!([{ "confidence": 0.75, "kind": "risk", "reason": format!("{}。", "界".repeat(80)) - }]) - .to_string(); + }])); let cases = [ ( "unknown-field", - r#"[{"index":0,"confidence":0.75,"kind":"risk","reason":"This is a concrete defect."}]"#.to_string(), + r#"{"scores":[{"index":0,"confidence":0.75,"kind":"risk","reason":"This is a concrete defect."}]}"#.to_string(), ), ( "negative-confidence", - r#"[{"confidence":-1,"kind":"risk","reason":"This is a concrete defect."}]"#.to_string(), + r#"{"scores":[{"confidence":-1,"kind":"risk","reason":"This is a concrete defect."}]}"#.to_string(), ), ( "high-confidence", - r#"[{"confidence":5,"kind":"risk","reason":"This is a concrete defect."}]"#.to_string(), + r#"{"scores":[{"confidence":5,"kind":"risk","reason":"This is a concrete defect."}]}"#.to_string(), ), ( "raw-nan", - r#"[{"confidence":NaN,"kind":"risk","reason":"This is a concrete defect."}]"#.to_string(), + r#"{"scores":[{"confidence":NaN,"kind":"risk","reason":"This is a concrete defect."}]}"#.to_string(), ), ( "string-nan", - r#"[{"confidence":"NaN","kind":"risk","reason":"This is a concrete defect."}]"#.to_string(), + r#"{"scores":[{"confidence":"NaN","kind":"risk","reason":"This is a concrete defect."}]}"#.to_string(), ), - ("missing-entry", "[]".to_string()), + ("missing-entry", r#"{"scores":[]}"#.to_string()), ( "duplicate-entry", - r#"[{"confidence":0.75,"kind":"risk","reason":"This is a concrete defect."},{"confidence":0.75,"kind":"risk","reason":"This repeats the same input."}]"#.to_string(), + r#"{"scores":[{"confidence":0.75,"kind":"risk","reason":"This is a concrete defect."},{"confidence":0.75,"kind":"risk","reason":"This repeats the same input."}]}"#.to_string(), ), ( "edge-whitespace", - r#"[{"confidence":0.75,"kind":"risk","reason":" Leading whitespace is invalid."}]"#.to_string(), + r#"{"scores":[{"confidence":0.75,"kind":"risk","reason":" Leading whitespace is invalid."}]}"#.to_string(), ), ( "control-character", - r#"[{"confidence":0.75,"kind":"risk","reason":"A control\u0000character is invalid."}]"#.to_string(), + r#"{"scores":[{"confidence":0.75,"kind":"risk","reason":"A control\u0000character is invalid."}]}"#.to_string(), ), ( "missing-punctuation", - r#"[{"confidence":0.75,"kind":"risk","reason":"This reason is incomplete"}]"#.to_string(), + r#"{"scores":[{"confidence":0.75,"kind":"risk","reason":"This reason is incomplete"}]}"#.to_string(), ), ("byte-overflow", byte_overflow), ]; @@ -8499,7 +8506,7 @@ async fn scorer_error_fails_open_and_preserves_generator_values() { Mock::given(method("POST")) .and(path("/chat/completions")) .and(body_string_contains("anthropic/claude-haiku-4.5")) - .respond_with(ResponseTemplate::new(200).set_body_json(llm_content(json!([])))) + .respond_with(ResponseTemplate::new(200).set_body_json(scorer_content(json!([])))) .mount(&server) .await; From 75104c39b7a411eea639859f8b5cf822841b4e8d Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Tue, 25 Aug 2026 17:14:44 +0000 Subject: [PATCH 2/2] Record the exact Luna release baseline --- bench/baseline.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bench/baseline.json b/bench/baseline.json index c4f2cfc..8d9824f 100644 --- a/bench/baseline.json +++ b/bench/baseline.json @@ -26,21 +26,21 @@ }, "openai/gpt-5.6-luna": { "populated": true, - "generatedAt": "2026-08-25T16:39:48.601Z", + "generatedAt": "2026-08-25T17:11:56.688Z", "reviewMode": "exhaustive", - "sourceRunAt": "2026-08-25T16:37:11.211Z", + "sourceRunAt": "2026-08-25T17:09:36.499Z", "providerContractEnforced": true, "screeningProfileSha256": "aea05c3f5622cebec480d2a8daf5bb53055bc0160206e6f22e889391ea71fa49", "upstreamProviderIdentity": "Azure", "totalCases": 70, "scoredCases": 70, - "detectionRate": 0.9122807017543859, + "detectionRate": 0.9473684210526315, "falsePositives": 0, - "gateVerdictCorrectness": 0.8, - "meanCostUsdPerCase": 0.0008069545000000001, + "gateVerdictCorrectness": 0.8428571428571429, + "meanCostUsdPerCase": 0.0008315965428571429, "latencyMs": { - "p50": 6066, - "p95": 10607 + "p50": 5659, + "p95": 9530 } } }