diff --git a/packages/headless/harbor/run-harness-ab.mjs b/packages/headless/harbor/run-harness-ab.mjs index f3e6d518f5..47442d3162 100644 --- a/packages/headless/harbor/run-harness-ab.mjs +++ b/packages/headless/harbor/run-harness-ab.mjs @@ -854,8 +854,8 @@ export function harnessMakaAgentEnv(benchmarkProfile, env = process.env) { }; } -function harnessMeasuredTransport(agentId, provider) { - const protocol = providerProxyUsageProtocol(agentId, provider); +function harnessMeasuredTransport(agentId, provider, model) { + const protocol = providerProxyUsageProtocol(agentId, provider, undefined, model); if (protocol === 'openai-chat-sse') return 'openai-chat'; if (protocol === 'openai-responses-sse') return 'openai-responses'; if (protocol === 'anthropic-sse') return 'anthropic-messages'; @@ -929,7 +929,7 @@ export function buildHarnessAbManifest({ config: { adapter: harnessAgentImportPath('maka'), ...(competitorProfiles.length > 1 - ? { transport: harnessMeasuredTransport('maka', execution.provider) } + ? { transport: harnessMeasuredTransport('maka', execution.provider, execution.model) } : {}), // The runner hands every arm MAKA_SYSTEM_PROMPT, but only the Maka // cell applies it; the native CLIs hash it into their execution @@ -964,7 +964,13 @@ export function buildHarnessAbManifest({ adapter: harnessAgentImportPath(profile.id), ...profile.config, ...(competitorProfiles.length > 1 - ? { transport: harnessMeasuredTransport(profile.id, execution.provider) } + ? { + transport: harnessMeasuredTransport( + profile.id, + execution.provider, + execution.model, + ), + } : {}), ...(profile.id === 'codex' && runtimeProfile.provider === 'deepseek' ? { modelCatalogFingerprint: CODEX_DEEPSEEK_MODEL_CATALOG_FINGERPRINT } diff --git a/packages/headless/src/__tests__/harbor-task-runner.test.ts b/packages/headless/src/__tests__/harbor-task-runner.test.ts index 09fc3bebbe..54a8f23410 100644 --- a/packages/headless/src/__tests__/harbor-task-runner.test.ts +++ b/packages/headless/src/__tests__/harbor-task-runner.test.ts @@ -126,6 +126,50 @@ test('DeepSeek routes each CLI through its native wire protocol', () => { assert.equal(harnessAgentImportPath('reasonix'), 'reasonix_agent:MakaReasonixAgent'); }); +test('the Maka arm measures the wire its own runtime resolves, not the adapter kind', () => { + // deepseek-v4-flash routes through the Responses API (`openAiAdapterApiProtocol`), + // so a proxy parsing it as Chat SSE never sees `[DONE]`, marks every request + // `interrupted`, and the runner throws the whole graded cell away as infra. + assert.equal( + providerProxyUsageProtocol('maka', 'deepseek', undefined, 'deepseek-v4-flash'), + 'openai-responses-sse', + ); + // Same provider, a model that stays on Chat Completions. + assert.equal( + providerProxyUsageProtocol('maka', 'deepseek', undefined, 'deepseek-v3.2'), + 'openai-chat-sse', + ); + // An advertised protocol wins only where the runtime's own connection carries + // one. For DeepSeek it does not: `connectionFromEnv` drops it, so the runtime + // dials Responses regardless and a proxy that honoured the override would be + // parsing a wire nobody is speaking — the same wrong number, reintroduced + // through this function's own input. + assert.equal( + providerProxyUsageProtocol('maka', 'deepseek', 'openai-chat', 'deepseek-v4-flash'), + 'openai-responses-sse', + ); + // The two providers whose connections do carry an advertised protocol have + // their own explicit branches above, asserted separately. + // Competitors run their own CLI, so the Maka runtime says nothing about them. + assert.equal( + providerProxyUsageProtocol('reasonix', 'deepseek', undefined, 'deepseek-v4-flash'), + 'openai-chat-sse', + ); + // The catalog spelling resolves to the same wire as the one the runtime dials. + // `resolveModelRuntime` does not recognize the prefixed id, so a caller that + // forwarded it raw would silently get the Chat guess back — this fix's own + // API re-entering the bug it exists to close. + assert.equal( + providerProxyUsageProtocol('maka', 'deepseek', undefined, 'deepseek/deepseek-v4-flash'), + 'openai-responses-sse', + ); + // And a caller with no model id at all gets an error, not the guess. + assert.throws( + () => providerProxyUsageProtocol('maka', 'deepseek'), + /the maka arm requires a model id/, + ); +}); + interface FakeOptions { reward?: string; cell?: HarborCellOutput | null; diff --git a/packages/headless/src/__tests__/harness-ab-cli.test.ts b/packages/headless/src/__tests__/harness-ab-cli.test.ts index a290d18a5d..f8cafea123 100644 --- a/packages/headless/src/__tests__/harness-ab-cli.test.ts +++ b/packages/headless/src/__tests__/harness-ab-cli.test.ts @@ -717,7 +717,11 @@ test('harness CLI freezes the synchronized DeepSeek three-way composition', asyn manifest.arms.map( (arm: { metadata?: { config?: { transport?: string } } }) => arm.metadata?.config?.transport, ), - ['openai-chat', 'openai-responses', 'anthropic-messages'], + // The Maka arm records the wire its runtime actually dials: deepseek-v4-flash + // resolves to Responses, so a manifest claiming Chat here would describe a + // run that never happened — and the proxy parsing it that way measured none + // of it. + ['openai-responses', 'openai-responses', 'anthropic-messages'], ); assert.equal(manifest.maxConcurrentAttempts, 6); assert.equal( diff --git a/packages/headless/src/__tests__/harness-ab-run.test.ts b/packages/headless/src/__tests__/harness-ab-run.test.ts index 7251fcf6da..7b3e56c542 100644 --- a/packages/headless/src/__tests__/harness-ab-run.test.ts +++ b/packages/headless/src/__tests__/harness-ab-run.test.ts @@ -221,6 +221,56 @@ describe('runHarnessAbComparison', () => { } }); + test('retries an infra-failed cell whose task id carries a dot', async () => { + const dir = await mkdtemp(join(tmpdir(), 'maka-harness-ab-dotted-retry-')); + try { + const promptPath = join(dir, 'empty-system-prompt.txt'); + await writeFile(promptPath, '', 'utf8'); + let failingAttempts = 0; + const calls: string[] = []; + const failingArm = harnessArm('maka', calls); + failingArm.harborRunner = async () => { + failingAttempts += 1; + throw new Error('container failed after launch'); + }; + const input = { + runId: 'glm-harness-ab', + runRoot: dir, + resultsJsonlPath: join(dir, 'results.jsonl'), + systemPromptPath: promptPath, + resumeFingerprint: 'sha256:manifest', + evaluationTasks: [ + { id: 'install-windows-3.11', path: '/tasks/install-windows-3.11' }, + { id: 'plain', path: '/tasks/plain' }, + ], + arms: [failingArm, harnessArm('opencode', calls)] as const, + }; + + await runHarnessAbComparison(input); + assert.equal(failingAttempts, 2); + + // The written round id normalizes the dot, so that is what an operator + // reads back out of results.jsonl and feeds to the retry. + await runHarnessAbComparison({ + ...input, + retryAdjudicatedInfraRoundIdsOnce: ['ab-maka-r0-install-windows-3-11'], + }); + assert.equal(failingAttempts, 3); + + // An operator recovering a sweep hands over a batch of ids. Naming only + // the first offender makes fixing a list of typos one round-trip each. + await assert.rejects( + runHarnessAbComparison({ + ...input, + retryAdjudicatedInfraRoundIdsOnce: ['ab-maka-r0-nope', 'ab-maka-r0-also-nope'], + }), + /unknown round ab-maka-r0-nope, ab-maka-r0-also-nope/, + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + test('continues the frozen schedule after a terminal cell infrastructure failure', async () => { const dir = await mkdtemp(join(tmpdir(), 'maka-harness-ab-resilient-schedule-')); try { diff --git a/packages/headless/src/harbor-task-runner.ts b/packages/headless/src/harbor-task-runner.ts index 68c5d5fd75..14621b99d3 100644 --- a/packages/headless/src/harbor-task-runner.ts +++ b/packages/headless/src/harbor-task-runner.ts @@ -49,8 +49,10 @@ import { type ProviderTokenUsage, type ProviderUpstreamCredentialResolver, } from './provider-auth-proxy.js'; +export { modelIdForProvider } from './harness-agent-registry.js'; import { harnessAgentImportPath, + modelIdForProvider, providerProxyClientAuthMode, providerProxyClientBaseUrl, providerProxyUpstreamAuthMode, @@ -1422,7 +1424,7 @@ async function hostSideProviderRuntime(options: HarborTaskRunnerOptions): Promis : { apiKeyFile: apiKeyFile! }), clientAuthMode: providerProxyClientAuthMode(agent, provider, apiProtocol), upstreamAuthMode: providerProxyUpstreamAuthMode(agent, provider, apiProtocol), - usageProtocol: providerProxyUsageProtocol(agent, provider, apiProtocol), + usageProtocol: providerProxyUsageProtocol(agent, provider, apiProtocol, options.model), }); return { env: @@ -1909,11 +1911,6 @@ export function isBudgetExhaustedError( * ("openai-compatible" routing "anthropic/claude-sonnet-4-5"). The cell's * parseModelSpec preserves whatever it receives when a provider is set, so the * stripping must happen here. */ -export function modelIdForProvider(model: string, provider: string): string { - const prefix = `${provider}/`; - return model.startsWith(prefix) ? model.slice(prefix.length) : model; -} - export function modelForOpenCode(model: string, provider: string): string { return model.includes('/') ? model : `${provider}/${model}`; } diff --git a/packages/headless/src/harness-ab-run.ts b/packages/headless/src/harness-ab-run.ts index 252519b517..755fb646da 100644 --- a/packages/headless/src/harness-ab-run.ts +++ b/packages/headless/src/harness-ab-run.ts @@ -1,5 +1,5 @@ import { buildRunManifestFingerprint } from './ab-manifest.js'; -import { runArmCohort } from './ab-run.js'; +import { buildAbRoundId, runArmCohort } from './ab-run.js'; import { withAbRunLock } from './ab-run-lock.js'; import type { AbComparisonSummary, ArmCohortResult } from './ab-types.js'; import { isEvaluatedOutcome, summarizeAbComparison } from './ab-summary.js'; @@ -142,13 +142,22 @@ async function executeHarnessArmCohort(input: RunHarnessArmCohortInput): Promise if (retryRoundIds.size !== (input.retryAdjudicatedInfraRoundIdsOnce?.length ?? 0)) { throw new Error('adjudicated infra retry round ids must be unique'); } + // Build the candidates with the same helper the cohort writes with. Spelling + // the id out here instead let the two drift: `buildAbRoundId` normalizes `.` + // to `-`, so a retry naming `install-windows-3.11` was rejected as unknown — + // and because the check throws rather than skips, one dotted task took the + // whole retry batch down with it. const validRoundIds = new Set( - input.evaluationTasks.flatMap((task) => input.arms.map((arm) => `ab-${arm.id}-r0-${task.id}`)), + input.evaluationTasks.flatMap((task) => + input.arms.map((arm) => buildAbRoundId(undefined, arm.id, 0, task.id)), + ), ); - for (const roundId of retryRoundIds) { - if (!validRoundIds.has(roundId)) { - throw new Error(`adjudicated infra retry names unknown round ${roundId}`); - } + // Report every unknown id at once. An operator recovering a sweep hands over + // a batch of them, and failing on the first turns one typo into as many + // round-trips as there are mistakes. + const unknownRoundIds = [...retryRoundIds].filter((roundId) => !validRoundIds.has(roundId)); + if (unknownRoundIds.length > 0) { + throw new Error(`adjudicated infra retry names unknown round ${unknownRoundIds.join(', ')}`); } const preexistingEventIds = new Set( (await readFixedPromptWal(input.resultsJsonlPath)) diff --git a/packages/headless/src/harness-agent-registry.ts b/packages/headless/src/harness-agent-registry.ts index a2daadd119..c347ee9939 100644 --- a/packages/headless/src/harness-agent-registry.ts +++ b/packages/headless/src/harness-agent-registry.ts @@ -1,5 +1,7 @@ -import { PROVIDER_DEFAULTS, type ProviderType } from '@maka/core/llm-connections'; +import { PROVIDER_DEFAULTS, type ModelInfo, type ProviderType } from '@maka/core/llm-connections'; +import { type ModelRuntimeWire, resolveModelRuntime } from '@maka/runtime/model-runtime'; import type { ProviderAuthProxyMode, ProviderUsageProtocol } from './provider-auth-proxy.js'; +import { modelApiProtocolFromEnv, selectedModelApiProtocol } from './provider-env.js'; export type HarnessAgentId = | 'maka' @@ -68,10 +70,21 @@ export function providerProxyUpstreamAuthMode( : 'bearer'; } +/** + * The model id as the provider itself spells it, with the catalog's `provider/` + * prefix removed. This is what the runtime dials with, so it is also the only + * spelling any provider-facing decision may be made on. + */ +export function modelIdForProvider(model: string, provider: string): string { + const prefix = `${provider}/`; + return model.startsWith(prefix) ? model.slice(prefix.length) : model; +} + export function providerProxyUsageProtocol( agent: HarnessAgentId, provider: string, apiProtocol?: string, + modelId?: string, ): ProviderUsageProtocol | undefined { if (agent === 'codex') return 'openai-responses-sse'; if (agent === 'claude-code') return 'anthropic-sse'; @@ -79,12 +92,61 @@ export function providerProxyUsageProtocol( if (provider === 'kimi-coding-plan' && apiProtocol === 'openai-chat') return 'openai-chat-sse'; if (provider === 'kimi-coding-plan' && apiProtocol === 'anthropic-messages') return 'anthropic-sse'; + // The Maka arm dials whatever wire its own runtime resolves, so that runtime + // is the authority here — the adapter kind is a guess, and a guess that + // drifts is not a gap but a wrong number: when deepseek-v4-flash moved to + // Responses, a proxy still parsing Chat SSE never saw `[DONE]`, recorded + // every request `interrupted` with no usage, and the runner threw each + // graded cell away as an infra failure. + // + // Which model is not optional information for the Maka arm: without it there + // is nothing to ask the runtime and the answer silently degrades back to the + // guess. Say so rather than returning a plausible wrong number. The id is + // normalized here so a caller passing the catalog spelling cannot resolve a + // different wire than the one the runtime dials — `resolveModelRuntime` + // does not recognize `deepseek/deepseek-v4-flash` and falls back to Chat. + if (agent === 'maka') { + if (!modelId) throw new Error('providerProxyUsageProtocol: the maka arm requires a model id'); + const wire = makaRuntimeWire(provider, modelIdForProvider(modelId, provider), apiProtocol); + if (wire) return usageProtocolForWire(wire); + } const definition = providerDefinition(provider); if (definition?.runtimeAdapter.kind === 'anthropic') return 'anthropic-sse'; if (definition?.runtimeAdapter.kind === 'openai-compatible') return 'openai-chat-sse'; return undefined; } +function makaRuntimeWire( + provider: string, + modelId: string, + apiProtocol?: string, +): ModelRuntimeWire | null { + if (!providerDefinition(provider)) return null; + // The connection the runtime will actually build, not one this side invents: + // an advertised protocol only reaches the model entry for the providers whose + // connections carry it, and `selectedModelApiProtocol` is where that is + // decided. Honouring it here where the runtime drops it would resolve a wire + // nothing dials — the wrong-number failure this whole path exists to remove. + const advertised = selectedModelApiProtocol( + provider as ProviderType, + modelApiProtocolFromEnv(apiProtocol), + ); + return resolveModelRuntime( + { + providerType: provider as ProviderType, + ...(advertised ? { models: [{ id: modelId, apiProtocol: advertised }] } : {}), + }, + modelId, + ).wire; +} + +function usageProtocolForWire(wire: ModelRuntimeWire): ProviderUsageProtocol | undefined { + if (wire === 'anthropic-messages') return 'anthropic-sse'; + if (wire === 'openai-chat') return 'openai-chat-sse'; + if (wire === 'openai-responses') return 'openai-responses-sse'; + return undefined; +} + function providerDefinition(provider: string) { return (PROVIDER_DEFAULTS as Partial>)[ provider diff --git a/packages/headless/src/pier-task-runner.ts b/packages/headless/src/pier-task-runner.ts index 58d971346f..e2a05a3081 100644 --- a/packages/headless/src/pier-task-runner.ts +++ b/packages/headless/src/pier-task-runner.ts @@ -915,7 +915,7 @@ async function pierProviderRuntime( : { apiKeyFile: options.apiKeyFile! }), clientAuthMode: providerProxyClientAuthMode(agent, provider, apiProtocol), upstreamAuthMode: providerProxyUpstreamAuthMode(agent, provider, apiProtocol), - usageProtocol: providerProxyUsageProtocol(agent, provider, apiProtocol), + usageProtocol: providerProxyUsageProtocol(agent, provider, apiProtocol, options.model), }; const proxy = options.providerProxyHub && proxyPort !== undefined diff --git a/packages/headless/src/provider-env.ts b/packages/headless/src/provider-env.ts index a2779ac124..6a5504338e 100644 --- a/packages/headless/src/provider-env.ts +++ b/packages/headless/src/provider-env.ts @@ -9,6 +9,28 @@ import { } from '@maka/core'; import type { RunHarborCellEnv } from './headless-run-env.js'; +/** + * The protocol a connection actually carries, given what was advertised. + * + * Only two providers let an advertised protocol pick the wire: GitHub Copilot, + * where the account discovers it, and the Kimi Coding Plan, which is sold as + * two protocols against one account. Everywhere else the model's own catalog + * entry decides, and an advertised value is dropped. + * + * Anything deciding what wire the Maka runtime will dial has to apply this same + * rule, or it decides about a connection the runtime will never build. The + * proxy resolves its SSE parser that way (`providerProxyUsageProtocol`), and + * honouring the override where the runtime drops it is the same wrong-wire + * failure — a proxy parsing Chat while the runtime dials Responses records + * every request interrupted with no usage. + */ +export function selectedModelApiProtocol( + provider: ProviderType, + advertised: ModelInfo['apiProtocol'], +): ModelInfo['apiProtocol'] { + return provider === 'github-copilot' || provider === 'kimi-coding-plan' ? advertised : undefined; +} + export interface ProviderCredentialEnv { apiKeys: readonly string[]; apiKeyFile: string; @@ -218,8 +240,7 @@ function connectionFromEnv( if (provider === 'github-copilot' && !modelApiProtocol) { throw new Error('GitHub Copilot requires an account-discovered model protocol'); } - const selectedApiProtocol = - provider === 'github-copilot' || provider === 'kimi-coding-plan' ? modelApiProtocol : undefined; + const selectedApiProtocol = selectedModelApiProtocol(provider, modelApiProtocol); return { slug: values.MAKA_LLM_CONNECTION_SLUG ?? provider, name: defaults.label, @@ -237,7 +258,7 @@ function connectionFromEnv( }; } -function modelApiProtocolFromEnv(value: string | undefined): ModelInfo['apiProtocol'] { +export function modelApiProtocolFromEnv(value: string | undefined): ModelInfo['apiProtocol'] { if (value === 'openai-chat' || value === 'openai-responses' || value === 'anthropic-messages') return value; return undefined; diff --git a/packages/runtime/package.json b/packages/runtime/package.json index dd52472096..4a6000bf01 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -20,6 +20,7 @@ "./tool-output-delta": "./dist/tool-output-delta.js", "./stream-watchdog": "./dist/stream-watchdog.js", "./model-factory": "./dist/model-factory.js", + "./model-runtime": "./dist/model-runtime.js", "./context-budget": "./dist/context-budget.js", "./active-full-compact": "./dist/active-full-compact.js", "./semantic-compact": "./dist/semantic-compact.js",