From 035b7ef36b810d78ad039c3103390a6a603e8f47 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 6 Aug 2026 02:39:51 +0800 Subject: [PATCH 1/5] fix(headless): measure the wire the Maka runtime actually dials The provider proxy inferred its usage protocol from the adapter kind, so every `openai-compatible` provider was parsed as Chat SSE. That inference silently went wrong when deepseek-v4-flash moved to the Responses API: the proxy never saw `data: [DONE]`, recorded all 20 requests of a cell as `interrupted` with zero usage and no reasoning tokens, and the runner's terminal-request check then threw the whole graded cell away as an infra failure -- a cell that had in fact passed with reward 1.0. The Maka arm runs the Maka runtime, so `resolveModelRuntime` is the authority on which wire it dials; ask it instead of guessing. Competitors run their own CLI and keep the adapter-kind path, which is what describes them. The A/B manifest's per-arm transport comes from the same call, so a run now records the protocol it measured rather than the one it assumed. Verified on the 5-task pilot the guess had been failing. --- packages/headless/harbor/run-harness-ab.mjs | 14 +++++-- .../src/__tests__/harbor-task-runner.test.ts | 25 +++++++++++ .../src/__tests__/harness-ab-cli.test.ts | 6 ++- packages/headless/src/harbor-task-runner.ts | 7 +++- .../headless/src/harness-agent-registry.ts | 41 ++++++++++++++++++- packages/headless/src/pier-task-runner.ts | 7 +++- packages/runtime/package.json | 1 + 7 files changed, 93 insertions(+), 8 deletions(-) diff --git a/packages/headless/harbor/run-harness-ab.mjs b/packages/headless/harbor/run-harness-ab.mjs index f3e6d518f5..47442d3162 100644 --- a/packages/headless/harbor/run-harness-ab.mjs +++ b/packages/headless/harbor/run-harness-ab.mjs @@ -854,8 +854,8 @@ export function harnessMakaAgentEnv(benchmarkProfile, env = process.env) { }; } -function harnessMeasuredTransport(agentId, provider) { - const protocol = providerProxyUsageProtocol(agentId, provider); +function harnessMeasuredTransport(agentId, provider, model) { + const protocol = providerProxyUsageProtocol(agentId, provider, undefined, model); if (protocol === 'openai-chat-sse') return 'openai-chat'; if (protocol === 'openai-responses-sse') return 'openai-responses'; if (protocol === 'anthropic-sse') return 'anthropic-messages'; @@ -929,7 +929,7 @@ export function buildHarnessAbManifest({ config: { adapter: harnessAgentImportPath('maka'), ...(competitorProfiles.length > 1 - ? { transport: harnessMeasuredTransport('maka', execution.provider) } + ? { transport: harnessMeasuredTransport('maka', execution.provider, execution.model) } : {}), // The runner hands every arm MAKA_SYSTEM_PROMPT, but only the Maka // cell applies it; the native CLIs hash it into their execution @@ -964,7 +964,13 @@ export function buildHarnessAbManifest({ adapter: harnessAgentImportPath(profile.id), ...profile.config, ...(competitorProfiles.length > 1 - ? { transport: harnessMeasuredTransport(profile.id, execution.provider) } + ? { + transport: harnessMeasuredTransport( + profile.id, + execution.provider, + execution.model, + ), + } : {}), ...(profile.id === 'codex' && runtimeProfile.provider === 'deepseek' ? { modelCatalogFingerprint: CODEX_DEEPSEEK_MODEL_CATALOG_FINGERPRINT } diff --git a/packages/headless/src/__tests__/harbor-task-runner.test.ts b/packages/headless/src/__tests__/harbor-task-runner.test.ts index 09fc3bebbe..23522ab25d 100644 --- a/packages/headless/src/__tests__/harbor-task-runner.test.ts +++ b/packages/headless/src/__tests__/harbor-task-runner.test.ts @@ -126,6 +126,31 @@ test('DeepSeek routes each CLI through its native wire protocol', () => { assert.equal(harnessAgentImportPath('reasonix'), 'reasonix_agent:MakaReasonixAgent'); }); +test('the Maka arm measures the wire its own runtime resolves, not the adapter kind', () => { + // deepseek-v4-flash routes through the Responses API (`openAiAdapterApiProtocol`), + // so a proxy parsing it as Chat SSE never sees `[DONE]`, marks every request + // `interrupted`, and the runner throws the whole graded cell away as infra. + assert.equal( + providerProxyUsageProtocol('maka', 'deepseek', undefined, 'deepseek-v4-flash'), + 'openai-responses-sse', + ); + // Same provider, a model that stays on Chat Completions. + assert.equal( + providerProxyUsageProtocol('maka', 'deepseek', undefined, 'deepseek-v3.2'), + 'openai-chat-sse', + ); + // An account-advertised protocol still wins over the model default. + assert.equal( + providerProxyUsageProtocol('maka', 'deepseek', 'openai-chat', 'deepseek-v4-flash'), + 'openai-chat-sse', + ); + // Competitors run their own CLI, so the Maka runtime says nothing about them. + assert.equal( + providerProxyUsageProtocol('reasonix', 'deepseek', undefined, 'deepseek-v4-flash'), + 'openai-chat-sse', + ); +}); + interface FakeOptions { reward?: string; cell?: HarborCellOutput | null; diff --git a/packages/headless/src/__tests__/harness-ab-cli.test.ts b/packages/headless/src/__tests__/harness-ab-cli.test.ts index a290d18a5d..f8cafea123 100644 --- a/packages/headless/src/__tests__/harness-ab-cli.test.ts +++ b/packages/headless/src/__tests__/harness-ab-cli.test.ts @@ -717,7 +717,11 @@ test('harness CLI freezes the synchronized DeepSeek three-way composition', asyn manifest.arms.map( (arm: { metadata?: { config?: { transport?: string } } }) => arm.metadata?.config?.transport, ), - ['openai-chat', 'openai-responses', 'anthropic-messages'], + // The Maka arm records the wire its runtime actually dials: deepseek-v4-flash + // resolves to Responses, so a manifest claiming Chat here would describe a + // run that never happened — and the proxy parsing it that way measured none + // of it. + ['openai-responses', 'openai-responses', 'anthropic-messages'], ); assert.equal(manifest.maxConcurrentAttempts, 6); assert.equal( diff --git a/packages/headless/src/harbor-task-runner.ts b/packages/headless/src/harbor-task-runner.ts index 68c5d5fd75..a3c2f8e14a 100644 --- a/packages/headless/src/harbor-task-runner.ts +++ b/packages/headless/src/harbor-task-runner.ts @@ -1422,7 +1422,12 @@ async function hostSideProviderRuntime(options: HarborTaskRunnerOptions): Promis : { apiKeyFile: apiKeyFile! }), clientAuthMode: providerProxyClientAuthMode(agent, provider, apiProtocol), upstreamAuthMode: providerProxyUpstreamAuthMode(agent, provider, apiProtocol), - usageProtocol: providerProxyUsageProtocol(agent, provider, apiProtocol), + usageProtocol: providerProxyUsageProtocol( + agent, + provider, + apiProtocol, + modelIdForProvider(options.model, provider), + ), }); return { env: diff --git a/packages/headless/src/harness-agent-registry.ts b/packages/headless/src/harness-agent-registry.ts index a2daadd119..8831973c1b 100644 --- a/packages/headless/src/harness-agent-registry.ts +++ b/packages/headless/src/harness-agent-registry.ts @@ -1,4 +1,5 @@ -import { PROVIDER_DEFAULTS, type ProviderType } from '@maka/core/llm-connections'; +import { PROVIDER_DEFAULTS, type ModelInfo, type ProviderType } from '@maka/core/llm-connections'; +import { type ModelRuntimeWire, resolveModelRuntime } from '@maka/runtime/model-runtime'; import type { ProviderAuthProxyMode, ProviderUsageProtocol } from './provider-auth-proxy.js'; export type HarnessAgentId = @@ -72,6 +73,7 @@ export function providerProxyUsageProtocol( agent: HarnessAgentId, provider: string, apiProtocol?: string, + modelId?: string, ): ProviderUsageProtocol | undefined { if (agent === 'codex') return 'openai-responses-sse'; if (agent === 'claude-code') return 'anthropic-sse'; @@ -79,12 +81,49 @@ export function providerProxyUsageProtocol( if (provider === 'kimi-coding-plan' && apiProtocol === 'openai-chat') return 'openai-chat-sse'; if (provider === 'kimi-coding-plan' && apiProtocol === 'anthropic-messages') return 'anthropic-sse'; + // The Maka arm dials whatever wire its own runtime resolves, so that runtime + // is the authority here — the adapter kind is a guess, and a guess that + // drifts is not a gap but a wrong number: when deepseek-v4-flash moved to + // Responses, a proxy still parsing Chat SSE never saw `[DONE]`, recorded + // every request `interrupted` with no usage, and the runner threw each + // graded cell away as an infra failure. + const wire = agent === 'maka' && modelId ? makaRuntimeWire(provider, modelId, apiProtocol) : null; + if (wire) return usageProtocolForWire(wire); const definition = providerDefinition(provider); if (definition?.runtimeAdapter.kind === 'anthropic') return 'anthropic-sse'; if (definition?.runtimeAdapter.kind === 'openai-compatible') return 'openai-chat-sse'; return undefined; } +function makaRuntimeWire( + provider: string, + modelId: string, + apiProtocol?: string, +): ModelRuntimeWire | null { + if (!providerDefinition(provider)) return null; + const advertised = modelApiProtocol(apiProtocol); + return resolveModelRuntime( + { + providerType: provider as ProviderType, + ...(advertised ? { models: [{ id: modelId, apiProtocol: advertised }] } : {}), + }, + modelId, + ).wire; +} + +function modelApiProtocol(value: string | undefined): ModelInfo['apiProtocol'] { + return value === 'openai-chat' || value === 'openai-responses' || value === 'anthropic-messages' + ? value + : undefined; +} + +function usageProtocolForWire(wire: ModelRuntimeWire): ProviderUsageProtocol | undefined { + if (wire === 'anthropic-messages') return 'anthropic-sse'; + if (wire === 'openai-chat') return 'openai-chat-sse'; + if (wire === 'openai-responses') return 'openai-responses-sse'; + return undefined; +} + function providerDefinition(provider: string) { return (PROVIDER_DEFAULTS as Partial>)[ provider diff --git a/packages/headless/src/pier-task-runner.ts b/packages/headless/src/pier-task-runner.ts index 58d971346f..714318dca3 100644 --- a/packages/headless/src/pier-task-runner.ts +++ b/packages/headless/src/pier-task-runner.ts @@ -915,7 +915,12 @@ async function pierProviderRuntime( : { apiKeyFile: options.apiKeyFile! }), clientAuthMode: providerProxyClientAuthMode(agent, provider, apiProtocol), upstreamAuthMode: providerProxyUpstreamAuthMode(agent, provider, apiProtocol), - usageProtocol: providerProxyUsageProtocol(agent, provider, apiProtocol), + usageProtocol: providerProxyUsageProtocol( + agent, + provider, + apiProtocol, + modelIdForProvider(options.model, provider), + ), }; const proxy = options.providerProxyHub && proxyPort !== undefined diff --git a/packages/runtime/package.json b/packages/runtime/package.json index dd52472096..4a6000bf01 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -20,6 +20,7 @@ "./tool-output-delta": "./dist/tool-output-delta.js", "./stream-watchdog": "./dist/stream-watchdog.js", "./model-factory": "./dist/model-factory.js", + "./model-runtime": "./dist/model-runtime.js", "./context-budget": "./dist/context-budget.js", "./active-full-compact": "./dist/active-full-compact.js", "./semantic-compact": "./dist/semantic-compact.js", From ed147affb69d7ce98d0f670bb66a308ed91bf0ab Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 6 Aug 2026 08:59:32 +0800 Subject: [PATCH 2/5] fix(headless): build infra-retry round ids with the helper that writes them The retry allowlist spelled `ab-${arm}-r0-${task.id}` by hand while `buildAbRoundId` normalizes `.` to `-`, so a retry naming a dotted task never matched. Because the check throws instead of skipping, one such task took the whole batch down: a rerun of three infra-failed cells exited in twelve seconds without running any of them. --- .../src/__tests__/harness-ab-run.test.ts | 40 +++++++++++++++++++ packages/headless/src/harness-ab-run.ts | 11 ++++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/packages/headless/src/__tests__/harness-ab-run.test.ts b/packages/headless/src/__tests__/harness-ab-run.test.ts index 7251fcf6da..bf833c2f9c 100644 --- a/packages/headless/src/__tests__/harness-ab-run.test.ts +++ b/packages/headless/src/__tests__/harness-ab-run.test.ts @@ -221,6 +221,46 @@ describe('runHarnessAbComparison', () => { } }); + test('retries an infra-failed cell whose task id carries a dot', async () => { + const dir = await mkdtemp(join(tmpdir(), 'maka-harness-ab-dotted-retry-')); + try { + const promptPath = join(dir, 'empty-system-prompt.txt'); + await writeFile(promptPath, '', 'utf8'); + let failingAttempts = 0; + const calls: string[] = []; + const failingArm = harnessArm('maka', calls); + failingArm.harborRunner = async () => { + failingAttempts += 1; + throw new Error('container failed after launch'); + }; + const input = { + runId: 'glm-harness-ab', + runRoot: dir, + resultsJsonlPath: join(dir, 'results.jsonl'), + systemPromptPath: promptPath, + resumeFingerprint: 'sha256:manifest', + evaluationTasks: [ + { id: 'install-windows-3.11', path: '/tasks/install-windows-3.11' }, + { id: 'plain', path: '/tasks/plain' }, + ], + arms: [failingArm, harnessArm('opencode', calls)] as const, + }; + + await runHarnessAbComparison(input); + assert.equal(failingAttempts, 2); + + // The written round id normalizes the dot, so that is what an operator + // reads back out of results.jsonl and feeds to the retry. + await runHarnessAbComparison({ + ...input, + retryAdjudicatedInfraRoundIdsOnce: ['ab-maka-r0-install-windows-3-11'], + }); + assert.equal(failingAttempts, 3); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + test('continues the frozen schedule after a terminal cell infrastructure failure', async () => { const dir = await mkdtemp(join(tmpdir(), 'maka-harness-ab-resilient-schedule-')); try { diff --git a/packages/headless/src/harness-ab-run.ts b/packages/headless/src/harness-ab-run.ts index 252519b517..71473d6bb1 100644 --- a/packages/headless/src/harness-ab-run.ts +++ b/packages/headless/src/harness-ab-run.ts @@ -1,5 +1,5 @@ import { buildRunManifestFingerprint } from './ab-manifest.js'; -import { runArmCohort } from './ab-run.js'; +import { buildAbRoundId, runArmCohort } from './ab-run.js'; import { withAbRunLock } from './ab-run-lock.js'; import type { AbComparisonSummary, ArmCohortResult } from './ab-types.js'; import { isEvaluatedOutcome, summarizeAbComparison } from './ab-summary.js'; @@ -142,8 +142,15 @@ async function executeHarnessArmCohort(input: RunHarnessArmCohortInput): Promise if (retryRoundIds.size !== (input.retryAdjudicatedInfraRoundIdsOnce?.length ?? 0)) { throw new Error('adjudicated infra retry round ids must be unique'); } + // Build the candidates with the same helper the cohort writes with. Spelling + // the id out here instead let the two drift: `buildAbRoundId` normalizes `.` + // to `-`, so a retry naming `install-windows-3.11` was rejected as unknown — + // and because the check throws rather than skips, one dotted task took the + // whole retry batch down with it. const validRoundIds = new Set( - input.evaluationTasks.flatMap((task) => input.arms.map((arm) => `ab-${arm.id}-r0-${task.id}`)), + input.evaluationTasks.flatMap((task) => + input.arms.map((arm) => buildAbRoundId(undefined, arm.id, 0, task.id)), + ), ); for (const roundId of retryRoundIds) { if (!validRoundIds.has(roundId)) { From f064a2940445799a63f84e663e2bbca0ef517ca7 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 6 Aug 2026 09:44:21 +0800 Subject: [PATCH 3/5] fix(headless): make the proxy wire decision independent of how a caller spells the model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review found the fix's own API could re-admit the bug it closes: the model id was prefix-sensitive and optional, so a caller passing the catalog spelling `deepseek/deepseek-v4-flash` — which `resolveModelRuntime` does not recognize — silently got the Chat guess back, and a caller passing nothing at all got it too. Normalize inside the function and refuse to guess for the maka arm, which deletes the caller-side discipline at both runners rather than testing it. Also report every unknown retry round id at once. --- .../src/__tests__/harbor-task-runner.test.ts | 13 ++++++++++ .../src/__tests__/harness-ab-run.test.ts | 10 ++++++++ packages/headless/src/harbor-task-runner.ts | 14 +++-------- packages/headless/src/harness-ab-run.ts | 10 ++++---- .../headless/src/harness-agent-registry.ts | 24 +++++++++++++++++-- packages/headless/src/pier-task-runner.ts | 7 +----- 6 files changed, 55 insertions(+), 23 deletions(-) diff --git a/packages/headless/src/__tests__/harbor-task-runner.test.ts b/packages/headless/src/__tests__/harbor-task-runner.test.ts index 23522ab25d..2db8c8ff32 100644 --- a/packages/headless/src/__tests__/harbor-task-runner.test.ts +++ b/packages/headless/src/__tests__/harbor-task-runner.test.ts @@ -149,6 +149,19 @@ test('the Maka arm measures the wire its own runtime resolves, not the adapter k providerProxyUsageProtocol('reasonix', 'deepseek', undefined, 'deepseek-v4-flash'), 'openai-chat-sse', ); + // The catalog spelling resolves to the same wire as the one the runtime dials. + // `resolveModelRuntime` does not recognize the prefixed id, so a caller that + // forwarded it raw would silently get the Chat guess back — this fix's own + // API re-entering the bug it exists to close. + assert.equal( + providerProxyUsageProtocol('maka', 'deepseek', undefined, 'deepseek/deepseek-v4-flash'), + 'openai-responses-sse', + ); + // And a caller with no model id at all gets an error, not the guess. + assert.throws( + () => providerProxyUsageProtocol('maka', 'deepseek'), + /the maka arm requires a model id/, + ); }); interface FakeOptions { diff --git a/packages/headless/src/__tests__/harness-ab-run.test.ts b/packages/headless/src/__tests__/harness-ab-run.test.ts index bf833c2f9c..7b3e56c542 100644 --- a/packages/headless/src/__tests__/harness-ab-run.test.ts +++ b/packages/headless/src/__tests__/harness-ab-run.test.ts @@ -256,6 +256,16 @@ describe('runHarnessAbComparison', () => { retryAdjudicatedInfraRoundIdsOnce: ['ab-maka-r0-install-windows-3-11'], }); assert.equal(failingAttempts, 3); + + // An operator recovering a sweep hands over a batch of ids. Naming only + // the first offender makes fixing a list of typos one round-trip each. + await assert.rejects( + runHarnessAbComparison({ + ...input, + retryAdjudicatedInfraRoundIdsOnce: ['ab-maka-r0-nope', 'ab-maka-r0-also-nope'], + }), + /unknown round ab-maka-r0-nope, ab-maka-r0-also-nope/, + ); } finally { await rm(dir, { recursive: true, force: true }); } diff --git a/packages/headless/src/harbor-task-runner.ts b/packages/headless/src/harbor-task-runner.ts index a3c2f8e14a..14621b99d3 100644 --- a/packages/headless/src/harbor-task-runner.ts +++ b/packages/headless/src/harbor-task-runner.ts @@ -49,8 +49,10 @@ import { type ProviderTokenUsage, type ProviderUpstreamCredentialResolver, } from './provider-auth-proxy.js'; +export { modelIdForProvider } from './harness-agent-registry.js'; import { harnessAgentImportPath, + modelIdForProvider, providerProxyClientAuthMode, providerProxyClientBaseUrl, providerProxyUpstreamAuthMode, @@ -1422,12 +1424,7 @@ async function hostSideProviderRuntime(options: HarborTaskRunnerOptions): Promis : { apiKeyFile: apiKeyFile! }), clientAuthMode: providerProxyClientAuthMode(agent, provider, apiProtocol), upstreamAuthMode: providerProxyUpstreamAuthMode(agent, provider, apiProtocol), - usageProtocol: providerProxyUsageProtocol( - agent, - provider, - apiProtocol, - modelIdForProvider(options.model, provider), - ), + usageProtocol: providerProxyUsageProtocol(agent, provider, apiProtocol, options.model), }); return { env: @@ -1914,11 +1911,6 @@ export function isBudgetExhaustedError( * ("openai-compatible" routing "anthropic/claude-sonnet-4-5"). The cell's * parseModelSpec preserves whatever it receives when a provider is set, so the * stripping must happen here. */ -export function modelIdForProvider(model: string, provider: string): string { - const prefix = `${provider}/`; - return model.startsWith(prefix) ? model.slice(prefix.length) : model; -} - export function modelForOpenCode(model: string, provider: string): string { return model.includes('/') ? model : `${provider}/${model}`; } diff --git a/packages/headless/src/harness-ab-run.ts b/packages/headless/src/harness-ab-run.ts index 71473d6bb1..755fb646da 100644 --- a/packages/headless/src/harness-ab-run.ts +++ b/packages/headless/src/harness-ab-run.ts @@ -152,10 +152,12 @@ async function executeHarnessArmCohort(input: RunHarnessArmCohortInput): Promise input.arms.map((arm) => buildAbRoundId(undefined, arm.id, 0, task.id)), ), ); - for (const roundId of retryRoundIds) { - if (!validRoundIds.has(roundId)) { - throw new Error(`adjudicated infra retry names unknown round ${roundId}`); - } + // Report every unknown id at once. An operator recovering a sweep hands over + // a batch of them, and failing on the first turns one typo into as many + // round-trips as there are mistakes. + const unknownRoundIds = [...retryRoundIds].filter((roundId) => !validRoundIds.has(roundId)); + if (unknownRoundIds.length > 0) { + throw new Error(`adjudicated infra retry names unknown round ${unknownRoundIds.join(', ')}`); } const preexistingEventIds = new Set( (await readFixedPromptWal(input.resultsJsonlPath)) diff --git a/packages/headless/src/harness-agent-registry.ts b/packages/headless/src/harness-agent-registry.ts index 8831973c1b..c171275d40 100644 --- a/packages/headless/src/harness-agent-registry.ts +++ b/packages/headless/src/harness-agent-registry.ts @@ -69,6 +69,16 @@ export function providerProxyUpstreamAuthMode( : 'bearer'; } +/** + * The model id as the provider itself spells it, with the catalog's `provider/` + * prefix removed. This is what the runtime dials with, so it is also the only + * spelling any provider-facing decision may be made on. + */ +export function modelIdForProvider(model: string, provider: string): string { + const prefix = `${provider}/`; + return model.startsWith(prefix) ? model.slice(prefix.length) : model; +} + export function providerProxyUsageProtocol( agent: HarnessAgentId, provider: string, @@ -87,8 +97,18 @@ export function providerProxyUsageProtocol( // Responses, a proxy still parsing Chat SSE never saw `[DONE]`, recorded // every request `interrupted` with no usage, and the runner threw each // graded cell away as an infra failure. - const wire = agent === 'maka' && modelId ? makaRuntimeWire(provider, modelId, apiProtocol) : null; - if (wire) return usageProtocolForWire(wire); + // + // Which model is not optional information for the Maka arm: without it there + // is nothing to ask the runtime and the answer silently degrades back to the + // guess. Say so rather than returning a plausible wrong number. The id is + // normalized here so a caller passing the catalog spelling cannot resolve a + // different wire than the one the runtime dials — `resolveModelRuntime` + // does not recognize `deepseek/deepseek-v4-flash` and falls back to Chat. + if (agent === 'maka') { + if (!modelId) throw new Error('providerProxyUsageProtocol: the maka arm requires a model id'); + const wire = makaRuntimeWire(provider, modelIdForProvider(modelId, provider), apiProtocol); + if (wire) return usageProtocolForWire(wire); + } const definition = providerDefinition(provider); if (definition?.runtimeAdapter.kind === 'anthropic') return 'anthropic-sse'; if (definition?.runtimeAdapter.kind === 'openai-compatible') return 'openai-chat-sse'; diff --git a/packages/headless/src/pier-task-runner.ts b/packages/headless/src/pier-task-runner.ts index 714318dca3..e2a05a3081 100644 --- a/packages/headless/src/pier-task-runner.ts +++ b/packages/headless/src/pier-task-runner.ts @@ -915,12 +915,7 @@ async function pierProviderRuntime( : { apiKeyFile: options.apiKeyFile! }), clientAuthMode: providerProxyClientAuthMode(agent, provider, apiProtocol), upstreamAuthMode: providerProxyUpstreamAuthMode(agent, provider, apiProtocol), - usageProtocol: providerProxyUsageProtocol( - agent, - provider, - apiProtocol, - modelIdForProvider(options.model, provider), - ), + usageProtocol: providerProxyUsageProtocol(agent, provider, apiProtocol, options.model), }; const proxy = options.providerProxyHub && proxyPort !== undefined From 2520873763383c30fdaad09373e86245b2f36375 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 6 Aug 2026 10:08:40 +0800 Subject: [PATCH 4/5] fix(headless): resolve the proxy wire from the connection the runtime will build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proxy asked the runtime which wire it would dial, but asked about a connection the runtime never builds: an advertised protocol only reaches a model entry for GitHub Copilot and the Kimi Coding Plan, and `connectionFromEnv` drops it everywhere else. With MAKA_MODEL_API_PROTOCOL=openai-chat set against DeepSeek, the proxy resolved Chat SSE while the runtime dialled Responses — the wrong-wire failure this path exists to remove, readmitted through its own input. The gating rule is now one exported function that both sides read, and the test that pinned the divergent answer asserts the runtime's. --- .../src/__tests__/harbor-task-runner.test.ts | 10 ++++++-- .../headless/src/harness-agent-registry.ts | 11 +++++++- packages/headless/src/provider-env.ts | 25 +++++++++++++++++-- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/packages/headless/src/__tests__/harbor-task-runner.test.ts b/packages/headless/src/__tests__/harbor-task-runner.test.ts index 2db8c8ff32..54a8f23410 100644 --- a/packages/headless/src/__tests__/harbor-task-runner.test.ts +++ b/packages/headless/src/__tests__/harbor-task-runner.test.ts @@ -139,11 +139,17 @@ test('the Maka arm measures the wire its own runtime resolves, not the adapter k providerProxyUsageProtocol('maka', 'deepseek', undefined, 'deepseek-v3.2'), 'openai-chat-sse', ); - // An account-advertised protocol still wins over the model default. + // An advertised protocol wins only where the runtime's own connection carries + // one. For DeepSeek it does not: `connectionFromEnv` drops it, so the runtime + // dials Responses regardless and a proxy that honoured the override would be + // parsing a wire nobody is speaking — the same wrong number, reintroduced + // through this function's own input. assert.equal( providerProxyUsageProtocol('maka', 'deepseek', 'openai-chat', 'deepseek-v4-flash'), - 'openai-chat-sse', + 'openai-responses-sse', ); + // The two providers whose connections do carry an advertised protocol have + // their own explicit branches above, asserted separately. // Competitors run their own CLI, so the Maka runtime says nothing about them. assert.equal( providerProxyUsageProtocol('reasonix', 'deepseek', undefined, 'deepseek-v4-flash'), diff --git a/packages/headless/src/harness-agent-registry.ts b/packages/headless/src/harness-agent-registry.ts index c171275d40..769335d4a4 100644 --- a/packages/headless/src/harness-agent-registry.ts +++ b/packages/headless/src/harness-agent-registry.ts @@ -1,6 +1,7 @@ import { PROVIDER_DEFAULTS, type ModelInfo, type ProviderType } from '@maka/core/llm-connections'; import { type ModelRuntimeWire, resolveModelRuntime } from '@maka/runtime/model-runtime'; import type { ProviderAuthProxyMode, ProviderUsageProtocol } from './provider-auth-proxy.js'; +import { selectedModelApiProtocol } from './provider-env.js'; export type HarnessAgentId = | 'maka' @@ -121,7 +122,15 @@ function makaRuntimeWire( apiProtocol?: string, ): ModelRuntimeWire | null { if (!providerDefinition(provider)) return null; - const advertised = modelApiProtocol(apiProtocol); + // The connection the runtime will actually build, not one this side invents: + // an advertised protocol only reaches the model entry for the providers whose + // connections carry it, and `selectedModelApiProtocol` is where that is + // decided. Honouring it here where the runtime drops it would resolve a wire + // nothing dials — the wrong-number failure this whole path exists to remove. + const advertised = selectedModelApiProtocol( + provider as ProviderType, + modelApiProtocol(apiProtocol), + ); return resolveModelRuntime( { providerType: provider as ProviderType, diff --git a/packages/headless/src/provider-env.ts b/packages/headless/src/provider-env.ts index a2779ac124..877892c5c3 100644 --- a/packages/headless/src/provider-env.ts +++ b/packages/headless/src/provider-env.ts @@ -9,6 +9,28 @@ import { } from '@maka/core'; import type { RunHarborCellEnv } from './headless-run-env.js'; +/** + * The protocol a connection actually carries, given what was advertised. + * + * Only two providers let an advertised protocol pick the wire: GitHub Copilot, + * where the account discovers it, and the Kimi Coding Plan, which is sold as + * two protocols against one account. Everywhere else the model's own catalog + * entry decides, and an advertised value is dropped. + * + * Anything deciding what wire the Maka runtime will dial has to apply this same + * rule, or it decides about a connection the runtime will never build. The + * proxy resolves its SSE parser that way (`providerProxyUsageProtocol`), and + * honouring the override where the runtime drops it is the same wrong-wire + * failure — a proxy parsing Chat while the runtime dials Responses records + * every request interrupted with no usage. + */ +export function selectedModelApiProtocol( + provider: ProviderType, + advertised: ModelInfo['apiProtocol'], +): ModelInfo['apiProtocol'] { + return provider === 'github-copilot' || provider === 'kimi-coding-plan' ? advertised : undefined; +} + export interface ProviderCredentialEnv { apiKeys: readonly string[]; apiKeyFile: string; @@ -218,8 +240,7 @@ function connectionFromEnv( if (provider === 'github-copilot' && !modelApiProtocol) { throw new Error('GitHub Copilot requires an account-discovered model protocol'); } - const selectedApiProtocol = - provider === 'github-copilot' || provider === 'kimi-coding-plan' ? modelApiProtocol : undefined; + const selectedApiProtocol = selectedModelApiProtocol(provider, modelApiProtocol); return { slug: values.MAKA_LLM_CONNECTION_SLUG ?? provider, name: defaults.label, From 2d62951645cde2728a7c0cf4fd4515b1fe99549f Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 6 Aug 2026 10:23:12 +0800 Subject: [PATCH 5/5] refactor(headless): one whitelist for which protocol names are real The proxy path parsed the three protocol names with its own copy of the check `provider-env` already owned, next to the gating rule it just started sharing with it. Two copies of a value whitelist is how one of them ends up accepting a name the other rejects. --- packages/headless/src/harness-agent-registry.ts | 10 ++-------- packages/headless/src/provider-env.ts | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/packages/headless/src/harness-agent-registry.ts b/packages/headless/src/harness-agent-registry.ts index 769335d4a4..c347ee9939 100644 --- a/packages/headless/src/harness-agent-registry.ts +++ b/packages/headless/src/harness-agent-registry.ts @@ -1,7 +1,7 @@ import { PROVIDER_DEFAULTS, type ModelInfo, type ProviderType } from '@maka/core/llm-connections'; import { type ModelRuntimeWire, resolveModelRuntime } from '@maka/runtime/model-runtime'; import type { ProviderAuthProxyMode, ProviderUsageProtocol } from './provider-auth-proxy.js'; -import { selectedModelApiProtocol } from './provider-env.js'; +import { modelApiProtocolFromEnv, selectedModelApiProtocol } from './provider-env.js'; export type HarnessAgentId = | 'maka' @@ -129,7 +129,7 @@ function makaRuntimeWire( // nothing dials — the wrong-number failure this whole path exists to remove. const advertised = selectedModelApiProtocol( provider as ProviderType, - modelApiProtocol(apiProtocol), + modelApiProtocolFromEnv(apiProtocol), ); return resolveModelRuntime( { @@ -140,12 +140,6 @@ function makaRuntimeWire( ).wire; } -function modelApiProtocol(value: string | undefined): ModelInfo['apiProtocol'] { - return value === 'openai-chat' || value === 'openai-responses' || value === 'anthropic-messages' - ? value - : undefined; -} - function usageProtocolForWire(wire: ModelRuntimeWire): ProviderUsageProtocol | undefined { if (wire === 'anthropic-messages') return 'anthropic-sse'; if (wire === 'openai-chat') return 'openai-chat-sse'; diff --git a/packages/headless/src/provider-env.ts b/packages/headless/src/provider-env.ts index 877892c5c3..6a5504338e 100644 --- a/packages/headless/src/provider-env.ts +++ b/packages/headless/src/provider-env.ts @@ -258,7 +258,7 @@ function connectionFromEnv( }; } -function modelApiProtocolFromEnv(value: string | undefined): ModelInfo['apiProtocol'] { +export function modelApiProtocolFromEnv(value: string | undefined): ModelInfo['apiProtocol'] { if (value === 'openai-chat' || value === 'openai-responses' || value === 'anthropic-messages') return value; return undefined;