diff --git a/packages/core/src/events.ts b/packages/core/src/events.ts index 0ae8cc240d..cfcf4d2eba 100644 --- a/packages/core/src/events.ts +++ b/packages/core/src/events.ts @@ -363,6 +363,8 @@ export interface TokenUsageEvent extends BaseEvent { reasoning?: number; total?: number; rawFinishReason?: string; + /** Number of provider runtime/tool-loop steps represented by this usage. */ + runtimeSteps?: number; /** Backward-compatible alias for cacheHitInput. */ cacheRead?: number; /** Backward-compatible alias for cacheWriteInput. */ diff --git a/packages/core/src/runtime-event.ts b/packages/core/src/runtime-event.ts index 6974d9ac31..25c9a5dd6e 100644 --- a/packages/core/src/runtime-event.ts +++ b/packages/core/src/runtime-event.ts @@ -178,6 +178,8 @@ export interface RuntimeEventTokenUsage { reasoning?: number; total?: number; rawFinishReason?: string; + /** Number of provider runtime/tool-loop steps represented by this usage. */ + runtimeSteps?: number; /** Backward-compatible alias for cacheHitInput. */ cacheRead?: number; /** Backward-compatible alias for cacheWriteInput. */ diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts index 2370cd48e2..d1bc7a6a85 100644 --- a/packages/core/src/session.ts +++ b/packages/core/src/session.ts @@ -242,6 +242,8 @@ export interface TokenUsageMessage { reasoning?: number; total?: number; rawFinishReason?: string; + /** Number of provider runtime/tool-loop steps represented by this usage. */ + runtimeSteps?: number; /** Backward-compatible alias for cacheHitInput. */ cacheRead?: number; /** Backward-compatible alias for cacheWriteInput. */ diff --git a/packages/headless/harbor/maka_agent.py b/packages/headless/harbor/maka_agent.py index d73f4c626e..1cf8c4ac7f 100644 --- a/packages/headless/harbor/maka_agent.py +++ b/packages/headless/harbor/maka_agent.py @@ -279,6 +279,12 @@ def _cell_env(self, instruction_path: Any) -> dict[str, str]: # Default per-command timeout floor for the in-container Bash tool, so # long builds/tests do not hit a hard-coded 2-minute ceiling. "MAKA_CELL_COMMAND_TIMEOUT_MS", + # Benchmark-safe deterministic continuation. These are consumed by + # run-host-cell.mjs/run-cell.mjs, not by the provider backend. + "MAKA_HARBOR_CONTINUATION", + "MAKA_HARBOR_CONTINUATION_MAX_TURNS", + "MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS", + "MAKA_HARBOR_CONTINUATION_PROMPT", ): value = self._get_env(key) if value: diff --git a/packages/headless/harbor/run-host-cell.mjs b/packages/headless/harbor/run-host-cell.mjs index 474bac1c6c..1264cae3b6 100644 --- a/packages/headless/harbor/run-host-cell.mjs +++ b/packages/headless/harbor/run-host-cell.mjs @@ -5,6 +5,7 @@ import { join } from 'node:path'; import { buildAiSdkCellBackendRegistration, buildHarborCellContextBudgetPolicySnapshot, + buildHarborCellContinuationPolicy, normalizeHarborCellContextEnv, runHarborCell, } from '#harbor-cell'; @@ -29,6 +30,7 @@ export async function main() { const storageRoot = env.MAKA_STORAGE_ROOT || join(outputDir, 'maka-storage'); const contextEnv = normalizeHarborCellContextEnv(env); const contextBudgetPolicy = buildHarborCellContextBudgetPolicySnapshot(contextEnv); + const continuationPolicy = buildHarborCellContinuationPolicy(env); const now = Date.now; const newId = randomId; @@ -45,6 +47,7 @@ export async function main() { outputDir, storageRoot, ...(contextBudgetPolicy ? { contextBudgetPolicy } : {}), + ...(continuationPolicy ? { continuationPolicy } : {}), registerBackends: buildAiSdkCellBackendRegistration({ provider, model, diff --git a/packages/headless/src/__tests__/cell-output.test.ts b/packages/headless/src/__tests__/cell-output.test.ts index a9554118b9..7ee0f6ebf5 100644 --- a/packages/headless/src/__tests__/cell-output.test.ts +++ b/packages/headless/src/__tests__/cell-output.test.ts @@ -194,11 +194,14 @@ describe('Harbor cell output contract', () => { activeEstimatedTokensSaved: 450, activeArchiveFailures: 1, archivePlaceholders: 2, + archivePlaceholderReasonCounts: { active_prune: 1, stale_prune: 1 }, archiveWriteFailures: 1, retrievedArchiveToolResults: 1, retrievedArchiveEstimatedTokens: 120, archiveRetrievalSkipped: 3, + archiveRetrievalSkippedReasonCounts: { max_bytes: 2, max_results: 1 }, archiveRetrievalFailures: 1, + archiveRetrievalFailureReasonCounts: { corrupt: 1 }, }, }, }, @@ -242,11 +245,14 @@ describe('Harbor cell output contract', () => { activeEstimatedTokensSaved: 450, activeArchiveFailures: 1, archivePlaceholders: 2, + archivePlaceholderReasonCounts: { active_prune: 1, stale_prune: 1 }, archiveWriteFailures: 1, retrievedArchiveToolResults: 1, retrievedArchiveEstimatedTokens: 120, archiveRetrievalSkipped: 3, + archiveRetrievalSkippedReasonCounts: { max_bytes: 2, max_results: 1 }, archiveRetrievalFailures: 1, + archiveRetrievalFailureReasonCounts: { corrupt: 1 }, }); assert.deepEqual(validateHarborCellOutput(output), output); }); diff --git a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts index 1b8e0e01e9..a4e474d362 100644 --- a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts +++ b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts @@ -916,11 +916,14 @@ describe('fixed prompt controller', () => { activeEstimatedTokensSaved: 0, activeArchiveFailures: 0, archivePlaceholders: 2, + archivePlaceholderReasonCounts: {}, archiveWriteFailures: 0, retrievedArchiveToolResults: 1, retrievedArchiveEstimatedTokens: 120, archiveRetrievalSkipped: 0, + archiveRetrievalSkippedReasonCounts: {}, archiveRetrievalFailures: 0, + archiveRetrievalFailureReasonCounts: {}, }; const contextBudgetPolicy = { enabled: true as const, @@ -953,6 +956,48 @@ describe('fixed prompt controller', () => { }); }); + test('records continuation summary in completed task WAL events', async () => { + await withDir(async (dir) => { + const systemPromptPath = join(dir, 'system_prompt.md'); + const resultsJsonlPath = join(dir, 'results.jsonl'); + await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); + const continuationSummary = { + enabled: true, + maxTurns: 3, + maxTotalRuntimeSteps: 150, + turnsUsed: 2, + continuedTurns: 1, + stepCapHits: 1, + capExhausted: false, + totalRuntimeSteps: 42, + turns: [ + { turnIndex: 0, status: 'failed' as const, stepCapHit: true, runtimeSteps: 42 }, + { turnIndex: 1, status: 'completed' as const, stepCapHit: false, runtimeSteps: 0 }, + ], + }; + + const result = await runFixedPromptController({ + runId: 'run-1', + roundId: 'round-1', + config, + systemPromptPath, + resultsJsonlPath, + resultsTsvPath: join(dir, 'results.tsv'), + tasks: [{ id: 'task-a', path: '/bench/task-a' }], + harborRunner: async () => harborOutput({ taskId: 'task-a', continuationSummary }), + now: () => 100, + newId: idFactory(), + }); + + assert.equal(result.events[0]?.type, 'task_completed'); + if (result.events[0]?.type === 'task_completed') { + assert.deepEqual(result.events[0].continuationSummary, continuationSummary); + } + const event = JSON.parse((await readFile(resultsJsonlPath, 'utf8')).trimEnd()); + assert.deepEqual(event.continuationSummary, continuationSummary); + }); + }); + test('classifies completed Harbor reward failures as benchmark failures', async () => { await withDir(async (dir) => { const systemPromptPath = join(dir, 'system_prompt.md'); @@ -1115,6 +1160,7 @@ function harborOutput(input: { tokenSummary?: HarborTaskRunOutput['cell']['tokenSummary']; contextBudgetPolicy?: HarborTaskRunOutput['cell']['contextBudgetPolicy']; contextBudgetSummary?: HarborTaskRunOutput['cell']['contextBudgetSummary']; + continuationSummary?: HarborTaskRunOutput['cell']['continuationSummary']; }): HarborTaskRunOutput { return { harbor: { reward: input.reward ?? 1 }, @@ -1127,6 +1173,7 @@ function harborOutput(input: { tokenSummary: input.tokenSummary ?? tokenSummary({ input: 1, output: 2, reasoning: 0, total: 3, costUsd: 0.02 }), ...(input.contextBudgetPolicy ? { contextBudgetPolicy: input.contextBudgetPolicy } : {}), ...(input.contextBudgetSummary ? { contextBudgetSummary: input.contextBudgetSummary } : {}), + ...(input.continuationSummary ? { continuationSummary: input.continuationSummary } : {}), toolSummary: { providerVisibleToolCount: 0, actualToolCalls: 0, diff --git a/packages/headless/src/__tests__/harbor-adapter.test.ts b/packages/headless/src/__tests__/harbor-adapter.test.ts index 82980e84bd..3a53d664b9 100644 --- a/packages/headless/src/__tests__/harbor-adapter.test.ts +++ b/packages/headless/src/__tests__/harbor-adapter.test.ts @@ -707,6 +707,10 @@ with tempfile.TemporaryDirectory() as tmp: "MAKA_TRIAL_OUTPUT_USD_PER_1M": "0.29", "MAKA_TRIAL_CACHE_READ_USD_PER_1M": "0.0029", "MAKA_TRIAL_PRICING_SOURCE": "deepseek-v4-flash", + "MAKA_HARBOR_CONTINUATION": "on", + "MAKA_HARBOR_CONTINUATION_MAX_TURNS": "3", + "MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS": "150", + "MAKA_HARBOR_CONTINUATION_PROMPT": "Continue neutrally.", **runtime_policy_context_env, "MAKA_CONTEXT_FOO": "1", }) @@ -719,6 +723,10 @@ with tempfile.TemporaryDirectory() as tmp: assert gateway_env["MAKA_TRIAL_OUTPUT_USD_PER_1M"] == "0.29", gateway_env assert gateway_env["MAKA_TRIAL_CACHE_READ_USD_PER_1M"] == "0.0029", gateway_env assert gateway_env["MAKA_TRIAL_PRICING_SOURCE"] == "deepseek-v4-flash", gateway_env + assert gateway_env["MAKA_HARBOR_CONTINUATION"] == "on", gateway_env + assert gateway_env["MAKA_HARBOR_CONTINUATION_MAX_TURNS"] == "3", gateway_env + assert gateway_env["MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS"] == "150", gateway_env + assert gateway_env["MAKA_HARBOR_CONTINUATION_PROMPT"] == "Continue neutrally.", gateway_env for key, value in runtime_policy_context_env.items(): assert gateway_env[key] == value, gateway_env assert gateway_env["MAKA_CONTEXT_FOO"] == "1", gateway_env diff --git a/packages/headless/src/__tests__/harbor-cell.test.ts b/packages/headless/src/__tests__/harbor-cell.test.ts index b6ae800846..86c1fadcb1 100644 --- a/packages/headless/src/__tests__/harbor-cell.test.ts +++ b/packages/headless/src/__tests__/harbor-cell.test.ts @@ -23,6 +23,7 @@ import type { HeadlessBackendContext, IsolatedToolExecutor } from '../isolation. import { buildAiSdkCellBackendRegistration, buildHarborCellContextBudgetBackendOptions, + buildHarborCellContextBudgetPolicySnapshot, buildHarborCellAiSdkTools, createHarborCellLocalToolExecutor, HARBOR_CELL_OUTPUT_FILENAME, @@ -114,6 +115,181 @@ const registerThrowingBackend = (registry: BackendRegistry): void => { registry.register('fake', (ctx) => new ThrowingBackend({ sessionId: ctx.sessionId })); }; +class StepCapThenCompleteBackend implements AgentBackend { + readonly kind: BackendKind = 'fake'; + readonly sessionId: string; + readonly prompts: string[] = []; + readonly cwds: string[] = []; + + constructor(protected readonly ctx: { sessionId: string; header: SessionHeader }) { + this.sessionId = ctx.sessionId; + } + + async *send(input: BackendSendInput): AsyncIterable { + const ts = Date.now(); + this.prompts.push(input.text); + this.cwds.push(this.ctx.header.cwd); + if (this.prompts.length === 1) { + yield { + type: 'token_usage', + id: 'usage-step-cap', + turnId: input.turnId, + ts, + input: 10, + output: 1, + total: 11, + costUsd: 0.01, + rawFinishReason: 'tool-calls', + runtimeSteps: 50, + }; + yield { type: 'complete', id: 'complete-step-cap', turnId: input.turnId, ts, stopReason: 'end_turn' }; + return; + } + await writeFile(join(this.ctx.header.cwd, 'continued-proof.txt'), input.text, 'utf8'); + yield { + type: 'token_usage', + id: 'usage-done', + turnId: input.turnId, + ts, + input: 3, + output: 2, + total: 5, + costUsd: 0.02, + rawFinishReason: 'stop', + }; + yield { type: 'complete', id: 'complete-done', turnId: input.turnId, ts, stopReason: 'end_turn' }; + } + + async stop(): Promise {} + async respondToPermission(_decision: PermissionDecision): Promise {} + async dispose(): Promise {} +} + +function registerStepCapThenCompleteBackend(seen: { backend?: StepCapThenCompleteBackend }) { + return (registry: BackendRegistry): void => { + registry.register('fake', (ctx) => { + const backend = new StepCapThenCompleteBackend({ sessionId: ctx.sessionId, header: ctx.header }); + seen.backend = backend; + return backend; + }); + }; +} + +class StepCapThenThrowBackend extends StepCapThenCompleteBackend { + async *send(input: BackendSendInput): AsyncIterable { + if (this.prompts.length > 0) { + this.prompts.push(input.text); + this.cwds.push(this.ctx.header.cwd); + throw new Error('continuation turn crashed'); + } + yield* super.send(input); + } +} + +function registerStepCapThenThrowBackend(seen: { backend?: StepCapThenThrowBackend }) { + return (registry: BackendRegistry): void => { + registry.register('fake', (ctx) => { + const backend = new StepCapThenThrowBackend({ sessionId: ctx.sessionId, header: ctx.header }); + seen.backend = backend; + return backend; + }); + }; +} + +class NoisyStepCapThenCompleteBackend extends StepCapThenCompleteBackend { + async *send(input: BackendSendInput): AsyncIterable { + const ts = Date.now(); + if (this.prompts.length > 0) { + yield* super.send(input); + return; + } + + this.prompts.push(input.text); + this.cwds.push(this.ctx.header.cwd); + for (let index = 0; index < 60; index += 1) { + yield { + type: 'token_usage', + id: `noise-${index}`, + turnId: input.turnId, + ts, + input: 0, + output: 0, + total: 0, + }; + } + yield { + type: 'token_usage', + id: 'usage-step-cap-noisy', + turnId: input.turnId, + ts, + input: 10, + output: 1, + total: 11, + costUsd: 0.01, + rawFinishReason: 'tool-calls', + runtimeSteps: 50, + }; + yield { type: 'complete', id: 'complete-step-cap-noisy', turnId: input.turnId, ts, stopReason: 'end_turn' }; + } +} + +function registerNoisyStepCapThenCompleteBackend(seen: { backend?: NoisyStepCapThenCompleteBackend }) { + return (registry: BackendRegistry): void => { + registry.register('fake', (ctx) => { + const backend = new NoisyStepCapThenCompleteBackend({ sessionId: ctx.sessionId, header: ctx.header }); + seen.backend = backend; + return backend; + }); + }; +} + +class StepCapTwiceThenCompleteBackend extends StepCapThenCompleteBackend { + async *send(input: BackendSendInput): AsyncIterable { + const ts = Date.now(); + this.prompts.push(input.text); + this.cwds.push(this.ctx.header.cwd); + if (this.prompts.length <= 2) { + yield { + type: 'token_usage', + id: `usage-step-cap-${this.prompts.length}`, + turnId: input.turnId, + ts, + input: 10, + output: 1, + total: 11, + costUsd: 0.01, + rawFinishReason: 'tool-calls', + runtimeSteps: 50, + }; + yield { type: 'complete', id: `complete-step-cap-${this.prompts.length}`, turnId: input.turnId, ts, stopReason: 'end_turn' }; + return; + } + await writeFile(join(this.ctx.header.cwd, 'continued-proof.txt'), input.text, 'utf8'); + yield { + type: 'token_usage', + id: 'usage-done', + turnId: input.turnId, + ts, + input: 3, + output: 2, + total: 5, + costUsd: 0.02, + rawFinishReason: 'stop', + }; + yield { type: 'complete', id: 'complete-done', turnId: input.turnId, ts, stopReason: 'end_turn' }; + } +} + +function registerStepCapTwiceThenCompleteBackend(seen: { backend?: StepCapTwiceThenCompleteBackend }) { + return (registry: BackendRegistry): void => { + registry.register('fake', (ctx) => { + const backend = new StepCapTwiceThenCompleteBackend({ sessionId: ctx.sessionId, header: ctx.header }); + seen.backend = backend; + return backend; + }); + }; +} + describe('runHarborCell', () => { test('runs in the provided workspace and writes the shared cell artifacts', async () => { await withDirs(async ({ workspaceDir, outputDir, storageRoot }) => { @@ -165,6 +341,251 @@ describe('runHarborCell', () => { }); }); + test('continues after a tool-call step cap without verifier feedback', async () => { + await withDirs(async ({ workspaceDir, outputDir, storageRoot }) => { + const seen: { backend?: StepCapThenCompleteBackend } = {}; + const result = await runHarborCell({ + config, + instruction: 'solve the benchmark task', + cwd: workspaceDir, + outputDir, + storageRoot, + registerBackends: registerStepCapThenCompleteBackend(seen), + continuationPolicy: { + enabled: true, + maxTurns: 3, + maxTotalRuntimeSteps: 150, + prompt: 'Continue neutrally from current workspace.', + }, + }); + + assert.equal(result.output.status, 'completed'); + assert.deepEqual(seen.backend?.prompts, [ + 'solve the benchmark task', + 'Continue neutrally from current workspace.', + ]); + assert.deepEqual(seen.backend?.cwds, [workspaceDir, workspaceDir]); + assert.equal(await readFile(join(workspaceDir, 'continued-proof.txt'), 'utf8'), 'Continue neutrally from current workspace.'); + assert.deepEqual(result.output.continuationSummary, { + enabled: true, + maxTurns: 3, + maxTotalRuntimeSteps: 150, + turnsUsed: 2, + continuedTurns: 1, + stepCapHits: 1, + capExhausted: false, + totalRuntimeSteps: 50, + turns: [ + { turnIndex: 0, status: 'failed', stepCapHit: true, runtimeSteps: 50 }, + { turnIndex: 1, status: 'completed', stepCapHit: false, runtimeSteps: 0 }, + ], + }); + assert.equal(result.output.tokenSummary.input, 13); + assert.equal(result.output.tokenSummary.costUsd, 0.03); + const runtimeEvents = await readFile(join(outputDir, HARBOR_CELL_RUNTIME_EVENTS_FILENAME), 'utf8'); + assert.match(runtimeEvents, /usage-step-cap/); + assert.match(runtimeEvents, /usage-done/); + assert.doesNotMatch(seen.backend?.prompts[1] ?? '', /verifier|verification|failed|taxonomy|retry/i); + }); + }); + + test('fails the cell when a continuation turn throws after a step cap', async () => { + await withDirs(async ({ workspaceDir, outputDir, storageRoot }) => { + const seen: { backend?: StepCapThenThrowBackend } = {}; + const result = await runHarborCell({ + config, + instruction: 'solve the benchmark task', + cwd: workspaceDir, + outputDir, + storageRoot, + registerBackends: registerStepCapThenThrowBackend(seen), + continuationPolicy: { + enabled: true, + maxTurns: 3, + maxTotalRuntimeSteps: 150, + prompt: 'Continue neutrally from current workspace.', + }, + }); + + assert.equal(result.output.status, 'failed'); + assert.equal(result.output.errorClass, 'Error'); + assert.match(result.invocation.failure?.message ?? '', /continuation turn crashed/); + assert.deepEqual(seen.backend?.prompts, [ + 'solve the benchmark task', + 'Continue neutrally from current workspace.', + ]); + assert.deepEqual(result.output.continuationSummary, { + enabled: true, + maxTurns: 3, + maxTotalRuntimeSteps: 150, + turnsUsed: 2, + continuedTurns: 1, + stepCapHits: 1, + capExhausted: false, + totalRuntimeSteps: 50, + turns: [ + { turnIndex: 0, status: 'failed', stepCapHit: true, runtimeSteps: 50 }, + { turnIndex: 1, status: 'failed', stepCapHit: false, runtimeSteps: 0 }, + ], + }); + }); + }); + + test('stops continuation when the total runtime step budget is exhausted', async () => { + await withDirs(async ({ workspaceDir, outputDir, storageRoot }) => { + const seen: { backend?: StepCapThenCompleteBackend } = {}; + const result = await runHarborCell({ + config, + instruction: 'solve the benchmark task', + cwd: workspaceDir, + outputDir, + storageRoot, + registerBackends: registerStepCapThenCompleteBackend(seen), + continuationPolicy: { + enabled: true, + maxTurns: 3, + maxTotalRuntimeSteps: 3, + prompt: 'Continue neutrally from current workspace.', + }, + }); + + assert.equal(result.output.status, 'failed'); + assert.equal(result.output.errorClass, 'tool_step_cap_reached'); + assert.deepEqual(seen.backend?.prompts, ['solve the benchmark task']); + assert.deepEqual(result.output.continuationSummary, { + enabled: true, + maxTurns: 3, + maxTotalRuntimeSteps: 3, + turnsUsed: 1, + continuedTurns: 0, + stepCapHits: 1, + capExhausted: true, + totalRuntimeSteps: 50, + turns: [ + { turnIndex: 0, status: 'failed', stepCapHit: true, runtimeSteps: 50 }, + ], + }); + }); + }); + + test('does not spend continuation step budget from diagnostic event count', async () => { + await withDirs(async ({ workspaceDir, outputDir, storageRoot }) => { + const seen: { backend?: NoisyStepCapThenCompleteBackend } = {}; + const result = await runHarborCell({ + config, + instruction: 'solve the benchmark task', + cwd: workspaceDir, + outputDir, + storageRoot, + registerBackends: registerNoisyStepCapThenCompleteBackend(seen), + continuationPolicy: { + enabled: true, + maxTurns: 3, + maxTotalRuntimeSteps: 51, + prompt: 'Continue neutrally from current workspace.', + }, + }); + + assert.equal(result.output.status, 'completed'); + assert.deepEqual(seen.backend?.prompts, [ + 'solve the benchmark task', + 'Continue neutrally from current workspace.', + ]); + assert.deepEqual(result.output.continuationSummary, { + enabled: true, + maxTurns: 3, + maxTotalRuntimeSteps: 51, + turnsUsed: 2, + continuedTurns: 1, + stepCapHits: 1, + capExhausted: false, + totalRuntimeSteps: 50, + turns: [ + { turnIndex: 0, status: 'failed', stepCapHit: true, runtimeSteps: 50 }, + { turnIndex: 1, status: 'completed', stepCapHit: false, runtimeSteps: 0 }, + ], + }); + }); + }); + + test('records per-turn step-cap hits across continuation turns', async () => { + await withDirs(async ({ workspaceDir, outputDir, storageRoot }) => { + const seen: { backend?: StepCapTwiceThenCompleteBackend } = {}; + const result = await runHarborCell({ + config, + instruction: 'solve the benchmark task', + cwd: workspaceDir, + outputDir, + storageRoot, + registerBackends: registerStepCapTwiceThenCompleteBackend(seen), + continuationPolicy: { + enabled: true, + maxTurns: 3, + maxTotalRuntimeSteps: 150, + prompt: 'Continue neutrally from current workspace.', + }, + }); + + assert.equal(result.output.status, 'completed'); + assert.deepEqual(seen.backend?.prompts, [ + 'solve the benchmark task', + 'Continue neutrally from current workspace.', + 'Continue neutrally from current workspace.', + ]); + assert.deepEqual(result.output.continuationSummary?.turns.map((turn) => turn.stepCapHit), [true, true, false]); + assert.deepEqual(result.output.continuationSummary, { + enabled: true, + maxTurns: 3, + maxTotalRuntimeSteps: 150, + turnsUsed: 3, + continuedTurns: 2, + stepCapHits: 2, + capExhausted: false, + totalRuntimeSteps: 100, + turns: [ + { turnIndex: 0, status: 'failed', stepCapHit: true, runtimeSteps: 50 }, + { turnIndex: 1, status: 'failed', stepCapHit: true, runtimeSteps: 50 }, + { turnIndex: 2, status: 'completed', stepCapHit: false, runtimeSteps: 0 }, + ], + }); + }); + }); + + test('env entrypoint wires continuation policy from env', async () => { + await withDirs(async ({ workspaceDir, outputDir, storageRoot }) => { + const seen: { backend?: StepCapThenCompleteBackend } = {}; + const result = await runHarborCellFromEnv({ + MAKA_BACKEND: 'fake', + MAKA_INSTRUCTION: 'solve the benchmark task', + MAKA_WORKDIR: workspaceDir, + MAKA_OUTPUT_DIR: outputDir, + MAKA_STORAGE_ROOT: storageRoot, + MAKA_HARBOR_CONTINUATION: 'on', + MAKA_HARBOR_CONTINUATION_MAX_TURNS: '3', + MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS: '3', + MAKA_HARBOR_CONTINUATION_PROMPT: 'Continue neutrally from current workspace.', + }, { + registerBackends: registerStepCapThenCompleteBackend(seen), + }); + + assert.equal(result.output.status, 'failed'); + assert.deepEqual(seen.backend?.prompts, ['solve the benchmark task']); + assert.deepEqual(result.output.continuationSummary, { + enabled: true, + maxTurns: 3, + maxTotalRuntimeSteps: 3, + turnsUsed: 1, + continuedTurns: 0, + stepCapHits: 1, + capExhausted: true, + totalRuntimeSteps: 50, + turns: [ + { turnIndex: 0, status: 'failed', stepCapHit: true, runtimeSteps: 50 }, + ], + }); + }); + }); + test('env entrypoint records a context budget policy snapshot', async () => { await withDirs(async ({ workspaceDir, outputDir, storageRoot }) => { const off = await runHarborCellFromEnv({ @@ -762,6 +1183,16 @@ describe('runHarborCell', () => { ); }); + test('Harbor active tool result prune defaults to the measured 2048-token threshold in policy snapshots', () => { + const snapshot = buildHarborCellContextBudgetPolicySnapshot({ + MAKA_CONTEXT_ACTIVE_TOOL_RESULT_PRUNE: 'on', + }); + + assert.equal(snapshot?.enabled, true); + if (!snapshot?.enabled) throw new Error('expected context budget snapshot to be enabled'); + assert.equal(snapshot.activeToolResultPrune?.maxCurrentResultEstimatedTokens, 2048); + }); + test('Harbor tool builder keeps the six container-native tools non-interactive', () => { const tools = buildHarborCellAiSdkTools(fakeToolExecutor()); const names = tools.map((tool) => tool.name); diff --git a/packages/headless/src/__tests__/heavy-task-finalization.test.ts b/packages/headless/src/__tests__/heavy-task-finalization.test.ts index 9c63660451..84d9d981a9 100644 --- a/packages/headless/src/__tests__/heavy-task-finalization.test.ts +++ b/packages/headless/src/__tests__/heavy-task-finalization.test.ts @@ -163,7 +163,8 @@ describe('heavy-task finalization status', () => { { name: 'runtime step cap', status: 'failed', errorClass: 'max_steps', message: 'runtime step cap reached', capKind: 'runtime_step_cap' }, { name: 'wall time cap', status: 'failed', message: 'wall time cap reached', capKind: 'wall_time_cap' }, { name: 'max attempts', status: 'failed', reason: 'max attempts exhausted', capKind: 'max_attempts' }, - { name: 'tool calls', status: 'incomplete', errorClass: 'incomplete_tool_calls', capKind: 'tool_call_step_cap' }, + { name: 'legacy tool calls', status: 'incomplete', errorClass: 'incomplete_tool_calls', capKind: 'tool_call_step_cap' }, + { name: 'tool step cap', status: 'incomplete', errorClass: 'tool_step_cap_reached', capKind: 'tool_call_step_cap' }, { name: 'max tokens', status: 'incomplete', errorClass: 'max_tokens', capKind: 'token_cap' }, { name: 'timeout', status: 'failed', errorClass: 'timeout', capKind: 'timeout' }, ]; diff --git a/packages/headless/src/__tests__/prompt-ab-run.test.ts b/packages/headless/src/__tests__/prompt-ab-run.test.ts index bcead51ff3..34ff42d55f 100644 --- a/packages/headless/src/__tests__/prompt-ab-run.test.ts +++ b/packages/headless/src/__tests__/prompt-ab-run.test.ts @@ -269,6 +269,48 @@ describe('runAbComparison', () => { 'ab-tools-on-r1-t2:tools-on:t2', ]); }); + + test('starts both arms for the same task-rep pair before waiting for either arm to finish', async () => { + const calls: string[] = []; + const waiters: Array<() => void> = []; + let released = false; + const releaseAll = () => { + released = true; + for (const resolve of waiters.splice(0)) resolve(); + }; + + const runPromise = runAbComparison({ + runId: 'ab-run', + arms: [ + { id: 'tools-off', kind: 'tools', fingerprint: sha256('tools-off') }, + { id: 'tools-on', kind: 'tools', fingerprint: sha256('tools-on') }, + ], + evaluationTasks: [{ id: 't1', path: '/tasks/t1' }], + reps: 1, + maxConcurrency: 1, + runArm: async ({ roundId, task }) => { + calls.push(roundId); + if (!released) { + await new Promise((resolve) => waiters.push(resolve)); + } + return completed(task.id, true); + }, + }); + + while (calls.length === 0) { + await new Promise((resolve) => setImmediate(resolve)); + } + await new Promise((resolve) => setImmediate(resolve)); + const callsStartedBeforeFirstFinish = calls.length; + releaseAll(); + await runPromise; + + assert.equal(callsStartedBeforeFirstFinish, 2); + assert.deepEqual(calls, [ + 'ab-tools-off-r0-t1', + 'ab-tools-on-r0-t1', + ]); + }); }); describe('prompt A/B source fingerprints', () => { @@ -538,8 +580,13 @@ describe('summarizePromptAbComparison', () => { activeEstimatedTokensSaved: 450, activeArchiveFailures: 1, archivePlaceholders: 2, + archivePlaceholderReasonCounts: { active_prune: 2 }, retrievedArchiveToolResults: 1, retrievedArchiveEstimatedTokens: 120, + archiveRetrievalSkipped: 3, + archiveRetrievalSkippedReasonCounts: { max_bytes: 2, max_results: 1 }, + archiveRetrievalFailures: 1, + archiveRetrievalFailureReasonCounts: { not_found: 1 }, }); const candidateInactive = contextBudgetSummary({ prunedToolResults: 0 }); const result = summarizePromptAbComparison({ @@ -588,11 +635,14 @@ describe('summarizePromptAbComparison', () => { activeEstimatedTokensSaved: 450, activeArchiveFailures: 1, archivePlaceholders: 2, + archivePlaceholderReasonCounts: { active_prune: 2 }, archiveWriteFailures: 0, retrievedArchiveToolResults: 1, retrievedArchiveEstimatedTokens: 120, - archiveRetrievalSkipped: 0, - archiveRetrievalFailures: 0, + archiveRetrievalSkipped: 3, + archiveRetrievalSkippedReasonCounts: { max_bytes: 2, max_results: 1 }, + archiveRetrievalFailures: 1, + archiveRetrievalFailureReasonCounts: { not_found: 1 }, }); assert.deepEqual(result.candidate.activePruneSubset, { taskCount: 1, @@ -631,11 +681,14 @@ describe('summarizePromptAbComparison', () => { activeEstimatedTokensSaved: 450, activeArchiveFailures: 1, archivePlaceholders: 2, + archivePlaceholderReasonCounts: { active_prune: 2 }, archiveWriteFailures: 0, retrievedArchiveToolResults: 1, retrievedArchiveEstimatedTokens: 120, - archiveRetrievalSkipped: 0, - archiveRetrievalFailures: 0, + archiveRetrievalSkipped: 3, + archiveRetrievalSkippedReasonCounts: { max_bytes: 2, max_results: 1 }, + archiveRetrievalFailures: 1, + archiveRetrievalFailureReasonCounts: { not_found: 1 }, }, }); assert.deepEqual(result.baseline.activePruneSubset, { @@ -675,20 +728,23 @@ describe('summarizePromptAbComparison', () => { activeEstimatedTokensSaved: 0, activeArchiveFailures: 0, archivePlaceholders: 0, + archivePlaceholderReasonCounts: {}, archiveWriteFailures: 0, retrievedArchiveToolResults: 0, retrievedArchiveEstimatedTokens: 0, archiveRetrievalSkipped: 0, + archiveRetrievalSkippedReasonCounts: {}, archiveRetrievalFailures: 0, + archiveRetrievalFailureReasonCounts: {}, }, }); assert.match( renderPromptAbComparisonMarkdown(result), - /Context budget: A activated=0\/2 stale_pruned=0 active_pruned=0 active_tokens_saved=0 active_archive_failures=0 archive_placeholders=0 archive_write_failures=0 retrieved=0 retrieved_tokens=0 retrieval_skipped=0 retrieval_failures=0, B activated=1\/2 stale_pruned=2 active_pruned=3 active_tokens_saved=450 active_archive_failures=1 archive_placeholders=2 archive_write_failures=0 retrieved=1 retrieved_tokens=120 retrieval_skipped=0 retrieval_failures=0/, + /Context budget: A activated=0\/2 stale_pruned=0 active_pruned=0 active_tokens_saved=0 active_archive_failures=0 archive_placeholders=0 archive_placeholder_reasons=\{\} archive_write_failures=0 retrieved=0 retrieved_tokens=0 retrieval_skipped=0 retrieval_skipped_reasons=\{\} retrieval_failures=0 retrieval_failure_reasons=\{\}, B activated=1\/2 stale_pruned=2 active_pruned=3 active_tokens_saved=450 active_archive_failures=1 archive_placeholders=2 archive_placeholder_reasons=\{"active_prune":2\} archive_write_failures=0 retrieved=1 retrieved_tokens=120 retrieval_skipped=3 retrieval_skipped_reasons=\{"max_bytes":2,"max_results":1\} retrieval_failures=1 retrieval_failure_reasons=\{"not_found":1\}/, ); assert.match( renderPromptAbComparisonMarkdown(result), - /Active prune subset: A tasks=1 attempts=1 observed=1 missing=0 coverage=1 pass_rate=1 passed=1\/1 completed=1 timed_out=0 infra_failed=0 plumbing_failed=0 input=1 cache_hit=0 cache_miss=1 cache_write=0 output=1 total=2 cost_usd=0\.01 mean_duration_ms=100 activated=0\/1 stale_pruned=0 active_pruned=0 active_tokens_saved=0 active_archive_failures=0 archive_placeholders=0 archive_write_failures=0 retrieved=0 retrieved_tokens=0 retrieval_skipped=0 retrieval_failures=0, B tasks=1 attempts=1 observed=1 missing=0 coverage=1 pass_rate=1 passed=1\/1 completed=1 timed_out=0 infra_failed=0 plumbing_failed=0 input=1 cache_hit=0 cache_miss=1 cache_write=0 output=1 total=2 cost_usd=0\.01 mean_duration_ms=100 activated=1\/1 stale_pruned=2 active_pruned=3 active_tokens_saved=450 active_archive_failures=1 archive_placeholders=2 archive_write_failures=0 retrieved=1 retrieved_tokens=120 retrieval_skipped=0 retrieval_failures=0/, + /Active prune subset: A tasks=1 attempts=1 observed=1 missing=0 coverage=1 pass_rate=1 passed=1\/1 completed=1 timed_out=0 infra_failed=0 plumbing_failed=0 input=1 cache_hit=0 cache_miss=1 cache_write=0 output=1 total=2 cost_usd=0\.01 mean_duration_ms=100 activated=0\/1 stale_pruned=0 active_pruned=0 active_tokens_saved=0 active_archive_failures=0 archive_placeholders=0 archive_placeholder_reasons=\{\} archive_write_failures=0 retrieved=0 retrieved_tokens=0 retrieval_skipped=0 retrieval_skipped_reasons=\{\} retrieval_failures=0 retrieval_failure_reasons=\{\}, B tasks=1 attempts=1 observed=1 missing=0 coverage=1 pass_rate=1 passed=1\/1 completed=1 timed_out=0 infra_failed=0 plumbing_failed=0 input=1 cache_hit=0 cache_miss=1 cache_write=0 output=1 total=2 cost_usd=0\.01 mean_duration_ms=100 activated=1\/1 stale_pruned=2 active_pruned=3 active_tokens_saved=450 active_archive_failures=1 archive_placeholders=2 archive_placeholder_reasons=\{"active_prune":2\} archive_write_failures=0 retrieved=1 retrieved_tokens=120 retrieval_skipped=3 retrieval_skipped_reasons=\{"max_bytes":2,"max_results":1\} retrieval_failures=1 retrieval_failure_reasons=\{"not_found":1\}/, ); assert.match( renderPromptAbComparisonMarkdown(result), @@ -724,7 +780,7 @@ describe('summarizePromptAbComparison', () => { assert.match( renderPromptAbComparisonMarkdown(result), - /Active prune subset: A tasks=1 attempts=1 observed=0 missing=1 coverage=0 pass_rate=null passed=0\/0 completed=0 timed_out=0 infra_failed=0 plumbing_failed=0 input=0 cache_hit=0 cache_miss=0 cache_write=0 output=0 total=0 cost_usd=0 mean_duration_ms=null activated=0\/0 stale_pruned=0 active_pruned=0 active_tokens_saved=0 active_archive_failures=0 archive_placeholders=0 archive_write_failures=0 retrieved=0 retrieved_tokens=0 retrieval_skipped=0 retrieval_failures=0, B tasks=1 attempts=1 observed=1 missing=0 coverage=1 pass_rate=1 passed=1\/1 completed=1 timed_out=0 infra_failed=0 plumbing_failed=0 input=10 cache_hit=3 cache_miss=4 cache_write=2 output=5 total=16 cost_usd=0\.02 mean_duration_ms=250 activated=1\/1 stale_pruned=0 active_pruned=1 active_tokens_saved=0 active_archive_failures=0 archive_placeholders=0 archive_write_failures=0 retrieved=0 retrieved_tokens=0 retrieval_skipped=0 retrieval_failures=0/, + /Active prune subset: A tasks=1 attempts=1 observed=0 missing=1 coverage=0 pass_rate=null passed=0\/0 completed=0 timed_out=0 infra_failed=0 plumbing_failed=0 input=0 cache_hit=0 cache_miss=0 cache_write=0 output=0 total=0 cost_usd=0 mean_duration_ms=null activated=0\/0 stale_pruned=0 active_pruned=0 active_tokens_saved=0 active_archive_failures=0 archive_placeholders=0 archive_placeholder_reasons=\{\} archive_write_failures=0 retrieved=0 retrieved_tokens=0 retrieval_skipped=0 retrieval_skipped_reasons=\{\} retrieval_failures=0 retrieval_failure_reasons=\{\}, B tasks=1 attempts=1 observed=1 missing=0 coverage=1 pass_rate=1 passed=1\/1 completed=1 timed_out=0 infra_failed=0 plumbing_failed=0 input=10 cache_hit=3 cache_miss=4 cache_write=2 output=5 total=16 cost_usd=0\.02 mean_duration_ms=250 activated=1\/1 stale_pruned=0 active_pruned=1 active_tokens_saved=0 active_archive_failures=0 archive_placeholders=0 archive_placeholder_reasons=\{\} archive_write_failures=0 retrieved=0 retrieved_tokens=0 retrieval_skipped=0 retrieval_skipped_reasons=\{\} retrieval_failures=0 retrieval_failure_reasons=\{\}/, ); }); @@ -777,6 +833,65 @@ describe('summarizePromptAbComparison', () => { assert.match(markdown, /B input=60000 cache_hit=15000 cache_miss=40000 cache_write=5000 output=25000 total=90000 cost_usd=2000 mean_duration_ms=800/); }); + test('summarizes continuation cap diagnostics for A/B validity review', () => { + const result = summarizePromptAbComparison({ + runId: 'ab-run', + roundId: 'ab-summary', + baselinePromptId: 'prune-off', + candidatePromptId: 'prune-on', + evaluationTaskIds: ['t1', 't2'], + budgetMs: 600_000, + baselineRuns: [[ + { ...completed('t1', true), continuationSummary: continuationSummary({ turnsUsed: 2, continuedTurns: 1, stepCapHits: 1, totalRuntimeSteps: 42, turns: [ + { turnIndex: 0, status: 'failed', stepCapHit: true, runtimeSteps: 42 }, + { turnIndex: 1, status: 'completed', stepCapHit: false, runtimeSteps: 0 }, + ] }) }, + { ...completed('t2', false), continuationSummary: continuationSummary({ capExhausted: true, turnsUsed: 3, continuedTurns: 2, stepCapHits: 3, totalRuntimeSteps: 60, turns: [ + { turnIndex: 0, status: 'failed', stepCapHit: true, runtimeSteps: 20 }, + { turnIndex: 1, status: 'failed', stepCapHit: true, runtimeSteps: 20 }, + { turnIndex: 2, status: 'failed', stepCapHit: true, runtimeSteps: 20 }, + ] }) }, + ]], + candidateRuns: [[ + { ...completed('t1', true), continuationSummary: continuationSummary({ turnsUsed: 1, totalRuntimeSteps: 20 }) }, + { ...completed('t2', true), continuationSummary: continuationSummary({ turnsUsed: 2, continuedTurns: 1, stepCapHits: 1, totalRuntimeSteps: 44, turns: [ + { turnIndex: 0, status: 'failed', stepCapHit: true, runtimeSteps: 44 }, + { turnIndex: 1, status: 'completed', stepCapHit: false, runtimeSteps: 0 }, + ] }) }, + ]], + }); + + assert.deepEqual(result.baseline.continuation, { + attempts: 2, + enabledAttempts: 2, + wallTimeoutMs: 600_000, + turnsUsed: 5, + continuedTurns: 3, + stepCapHits: 4, + capExhaustedAttempts: 1, + totalRuntimeSteps: 102, + perTurnStepCapHits: [true, false, true, true, true], + maxTurns: 3, + maxTotalRuntimeSteps: 150, + }); + assert.deepEqual(result.candidate.continuation, { + attempts: 2, + enabledAttempts: 2, + wallTimeoutMs: 600_000, + turnsUsed: 3, + continuedTurns: 1, + stepCapHits: 1, + capExhaustedAttempts: 0, + totalRuntimeSteps: 64, + perTurnStepCapHits: [false, true, false], + maxTurns: 3, + maxTotalRuntimeSteps: 150, + }); + + const markdown = renderPromptAbComparisonMarkdown(result); + assert.match(markdown, /Continuation: A enabled=2\/2 wall_timeout=600000ms turns=5 continued=3 step_cap_hits=4 per_turn_step_cap_hits=\[true,false,true,true,true\] cap_exhausted=1 runtime_steps=102 max_turns=3 max_total_steps=150, B enabled=2\/2 wall_timeout=600000ms turns=3 continued=1 step_cap_hits=1 per_turn_step_cap_hits=\[false,true,false\] cap_exhausted=0 runtime_steps=64 max_turns=3 max_total_steps=150/); + }); + test('records activated attempts and investigation refs for follow-up', () => { const activatedSummary = contextBudgetSummary({ activePrunedToolResults: 1, activeEstimatedTokensSaved: 50 }); const staleOnlySummary = contextBudgetSummary({ prunedToolResults: 1, archivePlaceholders: 1 }); @@ -1101,13 +1216,20 @@ describe('runPromptAbComparison', () => { assert.equal(result.candidatePromptId, 'maka-improved-v1'); assert.equal(result.decision, 'non_inferior'); assert.equal(result.taskLevel.wins, 2); - assert.deepEqual(calls, [ + assert.equal(calls.length, 8); + assert.deepEqual(calls.slice(0, 2).sort(), [ 'ab-baseline-r0-t1:t1', 'ab-candidate-r0-t1:t1', - 'ab-candidate-r0-t2:t2', + ]); + assert.deepEqual(calls.slice(2, 4).sort(), [ 'ab-baseline-r0-t2:t2', - 'ab-candidate-r1-t1:t1', + 'ab-candidate-r0-t2:t2', + ]); + assert.deepEqual(calls.slice(4, 6).sort(), [ 'ab-baseline-r1-t1:t1', + 'ab-candidate-r1-t1:t1', + ]); + assert.deepEqual(calls.slice(6, 8).sort(), [ 'ab-baseline-r1-t2:t2', 'ab-candidate-r1-t2:t2', ]); @@ -1319,11 +1441,31 @@ function contextBudgetSummary( activeEstimatedTokensSaved: 0, activeArchiveFailures: 0, archivePlaceholders: 0, + archivePlaceholderReasonCounts: {}, archiveWriteFailures: 0, retrievedArchiveToolResults: 0, retrievedArchiveEstimatedTokens: 0, archiveRetrievalSkipped: 0, + archiveRetrievalSkippedReasonCounts: {}, archiveRetrievalFailures: 0, + archiveRetrievalFailureReasonCounts: {}, + ...input, + }; +} + +function continuationSummary( + input: Partial>, +): NonNullable { + return { + enabled: true, + maxTurns: 3, + maxTotalRuntimeSteps: 150, + turnsUsed: 1, + continuedTurns: 0, + stepCapHits: 0, + capExhausted: false, + totalRuntimeSteps: 1, + turns: [{ turnIndex: 0, status: 'completed', stepCapHit: false, runtimeSteps: 1 }], ...input, }; } diff --git a/packages/headless/src/__tests__/runtime-policy-ab-run.test.ts b/packages/headless/src/__tests__/runtime-policy-ab-run.test.ts index e6d3574dd1..7be666fe25 100644 --- a/packages/headless/src/__tests__/runtime-policy-ab-run.test.ts +++ b/packages/headless/src/__tests__/runtime-policy-ab-run.test.ts @@ -40,6 +40,11 @@ describe('runRuntimePolicyAbComparison', () => { candidateLimit: null, maxConcurrency: 4, nonInferiorityMargin: 0.1, + sharedAgentEnv: { + MAKA_HARBOR_CONTINUATION: 'on', + MAKA_HARBOR_CONTINUATION_MAX_TURNS: '3', + MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS: '150', + }, }); assert.equal(manifest.experimentKind, 'runtime'); @@ -50,6 +55,10 @@ describe('runRuntimePolicyAbComparison', () => { 'deepseek/deepseek-v4-flash', 'deepseek/deepseek-v4-flash', ]); + assert.deepEqual(manifest.arms.map((arm) => arm.metadata?.sharedAgentEnv), [ + { MAKA_HARBOR_CONTINUATION: 'on', MAKA_HARBOR_CONTINUATION_MAX_TURNS: '3', MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS: '150' }, + { MAKA_HARBOR_CONTINUATION: 'on', MAKA_HARBOR_CONTINUATION_MAX_TURNS: '3', MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS: '150' }, + ]); assert.notEqual(manifest.arms[0].fingerprint, manifest.arms[1].fingerprint); assert.deepEqual(manifest.arms.map((arm) => arm.metadata?.contextEnv), [ { MAKA_CONTEXT_BUDGET: 'off' }, @@ -82,6 +91,32 @@ describe('runRuntimePolicyAbComparison', () => { ); }); + test('rejects unsupported shared agent env keys before fingerprinting runtime-policy arms', () => { + assert.throws( + () => buildRuntimePolicyAbRunManifest({ + arms: [ + { id: 'prune-off', contextEnv: { MAKA_CONTEXT_BUDGET: 'off' } }, + { id: 'prune-on', contextEnv: { MAKA_CONTEXT_STALE_TOOL_RESULT_PRUNE: 'on' } }, + ], + sharedAgentEnv: { MAKA_CONTEXT_STALE_TOOL_RESULT_PRUNE: 'on' } as never, + promptHash: sha256('p'), + provider: 'deepseek', + baseUrl: 'https://api.deepseek.com', + model: 'deepseek/deepseek-v4-flash', + taskBudgetSec: 1800, + harborTimeoutMs: 2_100_000, + subjectFingerprint: sha256('s'), + taskSourceFingerprint: sha256('t'), + toolchainFingerprint: sha256('c'), + evaluationTaskIds: ['t1'], + reps: 1, + candidateLimit: null, + maxConcurrency: 1, + }), + /unsupported runtime policy shared agent env key: MAKA_CONTEXT_STALE_TOOL_RESULT_PRUNE/, + ); + }); + test('runs prune off against prune on with only arm-local context env changed', async () => { await withDir(async (dir) => { const promptPath = join(dir, 'system-prompt.md'); @@ -109,6 +144,11 @@ describe('runRuntimePolicyAbComparison', () => { calls.push(input); return harborOutput(input); }, + sharedAgentEnv: { + MAKA_HARBOR_CONTINUATION: 'on', + MAKA_HARBOR_CONTINUATION_MAX_TURNS: '3', + MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS: '150', + }, now: () => 100, newId: idFactory(), }); @@ -133,13 +173,20 @@ describe('runRuntimePolicyAbComparison', () => { assert.deepEqual(calls.map((call) => call.systemPrompt), ['shared prompt\n', 'shared prompt\n']); assert.deepEqual(calls.map((call) => call.config.model), [config.model, config.model]); assert.deepEqual(calls.map((call) => call.task.id), ['t1', 't1']); - assert.deepEqual(calls.map((call) => call.agentEnv), [ - { MAKA_CONTEXT_BUDGET: 'off' }, - { - MAKA_CONTEXT_STALE_TOOL_RESULT_PRUNE: 'on', - MAKA_CONTEXT_ARCHIVE_RETRIEVAL: 'on', - }, - ]); + const agentEnvByRoundId = new Map(calls.map((call) => [call.roundId, call.agentEnv])); + assert.deepEqual(agentEnvByRoundId.get('ab-prune-off-r0-t1'), { + MAKA_HARBOR_CONTINUATION: 'on', + MAKA_HARBOR_CONTINUATION_MAX_TURNS: '3', + MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS: '150', + MAKA_CONTEXT_BUDGET: 'off', + }); + assert.deepEqual(agentEnvByRoundId.get('ab-prune-on-r0-t1'), { + MAKA_HARBOR_CONTINUATION: 'on', + MAKA_HARBOR_CONTINUATION_MAX_TURNS: '3', + MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS: '150', + MAKA_CONTEXT_STALE_TOOL_RESULT_PRUNE: 'on', + MAKA_CONTEXT_ARCHIVE_RETRIEVAL: 'on', + }); }); }); @@ -161,7 +208,7 @@ describe('runRuntimePolicyAbComparison', () => { MAKA_CONTEXT_STALE_TOOL_RESULT_MAX_TOKENS: '4096', }, } as const; - const calls: string[] = []; + let calls: string[] = []; const run = async (arms: Parameters[0]['arms']) => { await runRuntimePolicyAbComparison({ runId: 'runtime-ab-run', @@ -171,6 +218,7 @@ describe('runRuntimePolicyAbComparison', () => { evaluationTasks: [task], reps: 1, arms, + sharedAgentEnv: { MAKA_HARBOR_CONTINUATION: 'on' }, resumeFingerprint: 'caller-salt', harborRunner: async (input) => { calls.push(`${input.roundId}:${JSON.stringify(input.agentEnv ?? {})}`); @@ -184,15 +232,42 @@ describe('runRuntimePolicyAbComparison', () => { await run([pruneOff, pruneOn]); assert.equal(calls.length, 2); - calls.length = 0; + calls = []; await run([pruneOff, pruneOnChanged]); assert.deepEqual(calls, [ - 'ab-prune-on-r0-t1:{"MAKA_CONTEXT_STALE_TOOL_RESULT_PRUNE":"on","MAKA_CONTEXT_STALE_TOOL_RESULT_MAX_TOKENS":"4096"}', + 'ab-prune-on-r0-t1:{"MAKA_HARBOR_CONTINUATION":"on","MAKA_CONTEXT_STALE_TOOL_RESULT_PRUNE":"on","MAKA_CONTEXT_STALE_TOOL_RESULT_MAX_TOKENS":"4096"}', ]); - calls.length = 0; + calls = []; await run([pruneOff, pruneOnChanged]); assert.deepEqual(calls, []); + + calls = []; + await runRuntimePolicyAbComparison({ + runId: 'runtime-ab-run', + config, + systemPromptPath: promptPath, + resultsJsonlPath, + evaluationTasks: [task], + reps: 1, + arms: [pruneOff, pruneOnChanged], + sharedAgentEnv: { + MAKA_HARBOR_CONTINUATION: 'on', + MAKA_HARBOR_CONTINUATION_MAX_TURNS: '3', + MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS: '150', + }, + resumeFingerprint: 'caller-salt', + harborRunner: async (input) => { + calls.push(`${input.roundId}:${JSON.stringify(input.agentEnv ?? {})}`); + return harborOutput(input); + }, + now: () => 100, + newId: idFactory(), + }); + assert.deepEqual(calls, [ + 'ab-prune-off-r0-t1:{"MAKA_HARBOR_CONTINUATION":"on","MAKA_HARBOR_CONTINUATION_MAX_TURNS":"3","MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS":"150","MAKA_CONTEXT_BUDGET":"off"}', + 'ab-prune-on-r0-t1:{"MAKA_HARBOR_CONTINUATION":"on","MAKA_HARBOR_CONTINUATION_MAX_TURNS":"3","MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS":"150","MAKA_CONTEXT_STALE_TOOL_RESULT_PRUNE":"on","MAKA_CONTEXT_STALE_TOOL_RESULT_MAX_TOKENS":"4096"}', + ]); }); }); }); diff --git a/packages/headless/src/__tests__/task-run-adapter.test.ts b/packages/headless/src/__tests__/task-run-adapter.test.ts index 8b5eceb247..4a2d85c125 100644 --- a/packages/headless/src/__tests__/task-run-adapter.test.ts +++ b/packages/headless/src/__tests__/task-run-adapter.test.ts @@ -127,6 +127,7 @@ describe('taskEventsFromResultRecord', () => { test('maps incomplete, policy, budget, aborted, blocked, and infra failures explicitly', () => { const cases = [ ['incomplete_tool_calls', 'agent_incomplete', 'incomplete'], + ['tool_step_cap_reached', 'agent_incomplete', 'incomplete'], ['permission_denied', 'policy_denied', 'policy_denied'], ['max_steps_exceeded', 'budget_exhausted', 'budget_exhausted'], ['user_aborted', 'aborted', 'aborted'], diff --git a/packages/headless/src/ab-render.ts b/packages/headless/src/ab-render.ts index b279b95ffb..f21fbc8cde 100644 --- a/packages/headless/src/ab-render.ts +++ b/packages/headless/src/ab-render.ts @@ -1,6 +1,7 @@ import type { AbAttemptRef, AbComparisonSummary, + AbContinuationSummary, AbContextBudgetSummary, AbDecision, AbPairInvestigationRef, @@ -11,6 +12,7 @@ export function renderAbComparisonMarkdown(summary: AbComparisonSummary): string const contextBudgetLine = renderContextBudgetLine(summary); const activePruneSubsetLine = renderActivePruneSubsetLine(summary); const contextBudgetPolicyLine = renderContextBudgetPolicyLine(summary); + const continuationLine = renderContinuationLine(summary); const investigationRefLines = renderInvestigationRefLines(summary); const lines = [ '# A/B Comparison', @@ -31,6 +33,7 @@ export function renderAbComparisonMarkdown(summary: AbComparisonSummary): string `- Budget outcomes: A timed_out=${summary.baseline.budgetExhausted}, B timed_out=${summary.candidate.budgetExhausted}`, `- Infra outcomes: A infra_failed=${summary.baseline.infraFailed}, B infra_failed=${summary.candidate.infraFailed}; A plumbing_failed=${summary.baseline.plumbingFailed}, B plumbing_failed=${summary.candidate.plumbingFailed}`, ...(contextBudgetPolicyLine ? [contextBudgetPolicyLine] : []), + ...(continuationLine ? [continuationLine] : []), ...(contextBudgetLine ? [contextBudgetLine] : []), ...(activePruneSubsetLine ? [activePruneSubsetLine] : []), '', @@ -88,6 +91,42 @@ function renderContextBudgetPolicyLine(summary: AbComparisonSummary): string | u return `- Context budget policy: A enabled=${baseline?.enabledAttempts ?? 0}/${baseline?.attempts ?? 0} snapshots=${JSON.stringify(baseline?.snapshots ?? [])}, B enabled=${candidate?.enabledAttempts ?? 0}/${candidate?.attempts ?? 0} snapshots=${JSON.stringify(candidate?.snapshots ?? [])}`; } +function renderContinuationLine(summary: AbComparisonSummary): string | undefined { + if (!summary.baseline.continuation && !summary.candidate.continuation) return undefined; + return `- Continuation: A ${renderContinuationMetrics(continuationOrZero(summary.baseline.continuation))}, B ${renderContinuationMetrics(continuationOrZero(summary.candidate.continuation))}`; +} + +function renderContinuationMetrics(summary: AbContinuationSummary): string { + return [ + `enabled=${summary.enabledAttempts}/${summary.attempts}`, + `wall_timeout=${summary.wallTimeoutMs !== null ? `${summary.wallTimeoutMs}ms` : 'null'}`, + `turns=${summary.turnsUsed}`, + `continued=${summary.continuedTurns}`, + `step_cap_hits=${summary.stepCapHits}`, + `per_turn_step_cap_hits=${JSON.stringify(summary.perTurnStepCapHits)}`, + `cap_exhausted=${summary.capExhaustedAttempts}`, + `runtime_steps=${summary.totalRuntimeSteps}`, + `max_turns=${summary.maxTurns ?? 'null'}`, + `max_total_steps=${summary.maxTotalRuntimeSteps ?? 'null'}`, + ].join(' '); +} + +function continuationOrZero(summary: AbContinuationSummary | undefined): AbContinuationSummary { + return summary ?? { + attempts: 0, + enabledAttempts: 0, + wallTimeoutMs: null, + turnsUsed: 0, + continuedTurns: 0, + stepCapHits: 0, + perTurnStepCapHits: [], + capExhaustedAttempts: 0, + totalRuntimeSteps: 0, + maxTurns: null, + maxTotalRuntimeSteps: null, + }; +} + function decisionLabel(decision: AbDecision): string { switch (decision) { case 'non_inferior': @@ -126,14 +165,21 @@ function renderContextBudgetMetrics(summary: AbContextBudgetSummary): string { `active_tokens_saved=${summary.activeEstimatedTokensSaved}`, `active_archive_failures=${summary.activeArchiveFailures}`, `archive_placeholders=${summary.archivePlaceholders}`, + `archive_placeholder_reasons=${renderCountRecord(summary.archivePlaceholderReasonCounts)}`, `archive_write_failures=${summary.archiveWriteFailures}`, `retrieved=${summary.retrievedArchiveToolResults}`, `retrieved_tokens=${summary.retrievedArchiveEstimatedTokens}`, `retrieval_skipped=${summary.archiveRetrievalSkipped}`, + `retrieval_skipped_reasons=${renderCountRecord(summary.archiveRetrievalSkippedReasonCounts)}`, `retrieval_failures=${summary.archiveRetrievalFailures}`, + `retrieval_failure_reasons=${renderCountRecord(summary.archiveRetrievalFailureReasonCounts)}`, ].join(' '); } +function renderCountRecord(record: Record): string { + return JSON.stringify(Object.fromEntries(Object.entries(record).sort(([left], [right]) => left.localeCompare(right)))); +} + function renderActivePruneSubsetMetrics(summary: NonNullable): string { const contextBudget = contextBudgetOrZero(summary.contextBudget); return [ @@ -164,11 +210,14 @@ function contextBudgetOrZero(summary: AbContextBudgetSummary | undefined): AbCon activeEstimatedTokensSaved: 0, activeArchiveFailures: 0, archivePlaceholders: 0, + archivePlaceholderReasonCounts: {}, archiveWriteFailures: 0, retrievedArchiveToolResults: 0, retrievedArchiveEstimatedTokens: 0, archiveRetrievalSkipped: 0, + archiveRetrievalSkippedReasonCounts: {}, archiveRetrievalFailures: 0, + archiveRetrievalFailureReasonCounts: {}, }; } diff --git a/packages/headless/src/ab-run.ts b/packages/headless/src/ab-run.ts index f82e62fbba..d710d898ad 100644 --- a/packages/headless/src/ab-run.ts +++ b/packages/headless/src/ab-run.ts @@ -67,22 +67,17 @@ async function runComparisonPair( input: RunAbComparisonInput, pair: { rep: number; taskIndex: number; task: FixedPromptTask }, ): Promise<{ rep: number; baseline: FixedPromptTaskWalEvent; candidate: FixedPromptTaskWalEvent }> { - let baseline: FixedPromptTaskWalEvent | undefined; - let candidate: FixedPromptTaskWalEvent | undefined; - const runBaseline = async () => { - baseline = await runComparisonTaskArm(input, input.arms[0], pair); - }; - const runCandidate = async () => { - candidate = await runComparisonTaskArm(input, input.arms[1], pair); - }; if ((pair.rep + pair.taskIndex) % 2 === 0) { - await runBaseline(); - await runCandidate(); - } else { - await runCandidate(); - await runBaseline(); + const [baseline, candidate] = await Promise.all([ + runComparisonTaskArm(input, input.arms[0], pair), + runComparisonTaskArm(input, input.arms[1], pair), + ]); + return { rep: pair.rep, baseline, candidate }; } - if (!baseline || !candidate) throw new Error(`A/B pair did not produce both arms for ${pair.task.id} rep ${pair.rep}`); + const [candidate, baseline] = await Promise.all([ + runComparisonTaskArm(input, input.arms[1], pair), + runComparisonTaskArm(input, input.arms[0], pair), + ]); return { rep: pair.rep, baseline, candidate }; } diff --git a/packages/headless/src/ab-summary.ts b/packages/headless/src/ab-summary.ts index f2a4132e4d..4c1f3d7b49 100644 --- a/packages/headless/src/ab-summary.ts +++ b/packages/headless/src/ab-summary.ts @@ -6,6 +6,7 @@ import type { AbAttemptRef, AbAttemptPairSummary, AbComparisonSummary, + AbContinuationSummary, AbContextBudgetSummary, AbContextBudgetPolicySummary, AbDecision, @@ -32,8 +33,8 @@ export function summarizeAbComparison(input: SummarizeAbComparisonInput): AbComp const reps = input.baselineRuns.length; const taskIds = [...input.evaluationTaskIds]; const activePrunePairIds = candidateActivePrunePairIds(observedArmAttempts(input.candidateRuns, taskIds, 'B')); - const baseline = summarizeArm(input.baselineRuns, taskIds, reps, 'A', activePrunePairIds); - const candidate = summarizeArm(input.candidateRuns, taskIds, reps, 'B', activePrunePairIds); + const baseline = summarizeArm(input.baselineRuns, taskIds, reps, 'A', activePrunePairIds, input.budgetMs); + const candidate = summarizeArm(input.candidateRuns, taskIds, reps, 'B', activePrunePairIds, input.budgetMs); const taskLevel = summarizeTasks(input.baselineRuns, input.candidateRuns, taskIds, reps); const pairedAttempts = summarizeAttemptPairs(input.baselineRuns, input.candidateRuns, taskIds); const investigationRefs = summarizeInvestigationRefs(input.baselineRuns, input.candidateRuns, taskIds); @@ -77,6 +78,7 @@ function summarizeArm( reps: number, arm: AbArmLabel, activePrunePairIds: ReadonlySet, + wallTimeoutMs: number | undefined, ): AbArmSummary { const attempts = taskIds.length * reps; const observedAttempts = observedArmAttempts(runs, taskIds, arm); @@ -86,6 +88,7 @@ function summarizeArm( const passed = valid.filter((event) => event.passed).length; const durations = budgetedRuns.map((event) => event.durationMs); const contextBudget = summarizeContextBudget(observedAttempts); + const continuation = summarizeContinuation(observed, wallTimeoutMs); const activePruneSubset = summarizeActivePruneSubset(observedAttempts, activePrunePairIds); const contextBudgetPolicy = summarizeContextBudgetPolicy(observed); const tokenCostSummary = summarizeTokenCost(budgetedRuns); @@ -106,6 +109,7 @@ function summarizeArm( tokenCostSummary, ...(contextBudgetPolicy ? { contextBudgetPolicy } : {}), ...(contextBudget ? { contextBudget } : {}), + ...(continuation ? { continuation } : {}), ...(activePruneSubset ? { activePruneSubset } : {}), }; } @@ -226,14 +230,50 @@ function summarizeContextBudget(attempts: readonly ObservedAttempt[]): AbContext activeEstimatedTokensSaved: sum(summaries.map((summary) => summary.activeEstimatedTokensSaved)), activeArchiveFailures: sum(summaries.map((summary) => summary.activeArchiveFailures)), archivePlaceholders: sum(summaries.map((summary) => summary.archivePlaceholders)), + archivePlaceholderReasonCounts: sumCountRecords(summaries.map((summary) => summary.archivePlaceholderReasonCounts)), archiveWriteFailures: sum(summaries.map((summary) => summary.archiveWriteFailures)), retrievedArchiveToolResults: sum(summaries.map((summary) => summary.retrievedArchiveToolResults)), retrievedArchiveEstimatedTokens: sum(summaries.map((summary) => summary.retrievedArchiveEstimatedTokens)), archiveRetrievalSkipped: sum(summaries.map((summary) => summary.archiveRetrievalSkipped)), + archiveRetrievalSkippedReasonCounts: sumCountRecords(summaries.map((summary) => summary.archiveRetrievalSkippedReasonCounts)), archiveRetrievalFailures: sum(summaries.map((summary) => summary.archiveRetrievalFailures)), + archiveRetrievalFailureReasonCounts: sumCountRecords(summaries.map((summary) => summary.archiveRetrievalFailureReasonCounts)), }; } +function summarizeContinuation( + events: readonly FixedPromptTaskWalEvent[], + wallTimeoutMs: number | undefined, +): AbContinuationSummary | undefined { + const summaries = events + .map((event) => ('continuationSummary' in event ? event.continuationSummary : undefined)) + .filter((summary): summary is NonNullable => summary !== undefined); + if (summaries.length === 0) return undefined; + return { + attempts: summaries.length, + enabledAttempts: summaries.filter((summary) => summary.enabled).length, + wallTimeoutMs: wallTimeoutMs ?? null, + turnsUsed: sum(summaries.map((summary) => summary.turnsUsed)), + continuedTurns: sum(summaries.map((summary) => summary.continuedTurns)), + stepCapHits: sum(summaries.map((summary) => summary.stepCapHits)), + capExhaustedAttempts: summaries.filter((summary) => summary.capExhausted).length, + totalRuntimeSteps: sum(summaries.map((summary) => summary.totalRuntimeSteps)), + perTurnStepCapHits: summaries.flatMap((summary) => summary.turns.map((turn) => turn.stepCapHit)), + maxTurns: summaries.length > 0 ? Math.max(...summaries.map((summary) => summary.maxTurns)) : null, + maxTotalRuntimeSteps: summaries.length > 0 ? Math.max(...summaries.map((summary) => summary.maxTotalRuntimeSteps)) : null, + }; +} + +function sumCountRecords(records: readonly Record[]): Record { + const result: Record = {}; + for (const record of records) { + for (const [key, value] of Object.entries(record)) { + result[key] = (result[key] ?? 0) + value; + } + } + return Object.fromEntries(Object.entries(result).sort(([left], [right]) => left.localeCompare(right))); +} + function isActivePruneActivated(summary: HarborCellContextBudgetSummary | undefined): boolean { return (summary?.activePrunedToolResults ?? 0) > 0; } diff --git a/packages/headless/src/ab-types.ts b/packages/headless/src/ab-types.ts index 2981636be4..8627f5c7e5 100644 --- a/packages/headless/src/ab-types.ts +++ b/packages/headless/src/ab-types.ts @@ -65,6 +65,7 @@ export interface AbArmSummary { tokenCostSummary: AbTokenCostSummary; contextBudgetPolicy?: AbContextBudgetPolicySummary; contextBudget?: AbContextBudgetSummary; + continuation?: AbContinuationSummary; activePruneSubset?: AbActivePruneSubsetSummary; } @@ -116,11 +117,28 @@ export interface AbContextBudgetSummary { activeEstimatedTokensSaved: number; activeArchiveFailures: number; archivePlaceholders: number; + archivePlaceholderReasonCounts: Record; archiveWriteFailures: number; retrievedArchiveToolResults: number; retrievedArchiveEstimatedTokens: number; archiveRetrievalSkipped: number; + archiveRetrievalSkippedReasonCounts: Record; archiveRetrievalFailures: number; + archiveRetrievalFailureReasonCounts: Record; +} + +export interface AbContinuationSummary { + attempts: number; + enabledAttempts: number; + wallTimeoutMs: number | null; + turnsUsed: number; + continuedTurns: number; + stepCapHits: number; + capExhaustedAttempts: number; + totalRuntimeSteps: number; + perTurnStepCapHits: boolean[]; + maxTurns: number | null; + maxTotalRuntimeSteps: number | null; } export interface AbTaskArmSummary { diff --git a/packages/headless/src/cell-output.ts b/packages/headless/src/cell-output.ts index aafc730c80..830118374f 100644 --- a/packages/headless/src/cell-output.ts +++ b/packages/headless/src/cell-output.ts @@ -31,11 +31,14 @@ export interface HarborCellContextBudgetSummary { activeEstimatedTokensSaved: number; activeArchiveFailures: number; archivePlaceholders: number; + archivePlaceholderReasonCounts: Record; archiveWriteFailures: number; retrievedArchiveToolResults: number; retrievedArchiveEstimatedTokens: number; archiveRetrievalSkipped: number; + archiveRetrievalSkippedReasonCounts: Record; archiveRetrievalFailures: number; + archiveRetrievalFailureReasonCounts: Record; } export type HarborCellContextBudgetPolicySnapshot = ({ enabled: false } | ({ enabled: true } & ContextBudgetPolicy)); @@ -47,6 +50,25 @@ export interface HarborCellRuntimeRefs { turnId: string; } +export interface HarborCellContinuationSummary { + enabled: boolean; + maxTurns: number; + maxTotalRuntimeSteps: number; + turnsUsed: number; + continuedTurns: number; + stepCapHits: number; + capExhausted: boolean; + totalRuntimeSteps: number; + turns: HarborCellContinuationTurnSummary[]; +} + +export interface HarborCellContinuationTurnSummary { + turnIndex: number; + status: 'completed' | 'failed'; + stepCapHit: boolean; + runtimeSteps: number; +} + export interface HarborCellToolSummary { providerVisibleToolCount: number; actualToolCalls: number; @@ -63,6 +85,7 @@ export interface HarborCellOutput { tokenSummary: HarborCellTokenSummary; contextBudgetPolicy?: HarborCellContextBudgetPolicySnapshot; contextBudgetSummary?: HarborCellContextBudgetSummary; + continuationSummary?: HarborCellContinuationSummary; toolSummary: HarborCellToolSummary; steps: number; durationMs: number; @@ -75,6 +98,7 @@ export function buildHarborCellOutput(input: { invocation: InvocationResult; runtimeEventsPath: string; contextBudgetPolicy?: HarborCellContextBudgetPolicySnapshot; + continuationSummary?: HarborCellContinuationSummary; }): HarborCellOutput { const { invocation } = input; return { @@ -86,6 +110,7 @@ export function buildHarborCellOutput(input: { tokenSummary: summarizeCellTokens(invocation.events), ...(input.contextBudgetPolicy ? { contextBudgetPolicy: input.contextBudgetPolicy } : {}), ...contextBudgetSummaryField(invocation.events), + ...(input.continuationSummary ? { continuationSummary: input.continuationSummary } : {}), toolSummary: summarizeCellTools(invocation.events), steps: invocation.events.length, durationMs: invocation.finishedAt - invocation.startedAt, @@ -119,6 +144,9 @@ export function validateHarborCellOutput(value: unknown): HarborCellOutput { const contextBudgetSummary = 'contextBudgetSummary' in value ? validateContextBudgetSummary(value.contextBudgetSummary) : undefined; + const continuationSummary = 'continuationSummary' in value + ? validateContinuationSummary(value.continuationSummary) + : undefined; const toolSummary = validateToolSummary(value.toolSummary); const steps = requireNumber(value.steps, 'steps'); const durationMs = requireNumber(value.durationMs, 'durationMs'); @@ -134,6 +162,7 @@ export function validateHarborCellOutput(value: unknown): HarborCellOutput { tokenSummary, ...(contextBudgetPolicy !== undefined ? { contextBudgetPolicy } : {}), ...(contextBudgetSummary !== undefined ? { contextBudgetSummary } : {}), + ...(continuationSummary !== undefined ? { continuationSummary } : {}), toolSummary, steps, durationMs, @@ -144,6 +173,34 @@ export function validateHarborCellOutput(value: unknown): HarborCellOutput { return output; } +function validateContinuationSummary(value: unknown): HarborCellContinuationSummary { + if (!isRecord(value)) throw new Error('continuationSummary must be a JSON object'); + return { + enabled: requireBoolean(value.enabled, 'continuationSummary.enabled'), + maxTurns: requireNumber(value.maxTurns, 'continuationSummary.maxTurns'), + maxTotalRuntimeSteps: requireNumber(value.maxTotalRuntimeSteps, 'continuationSummary.maxTotalRuntimeSteps'), + turnsUsed: requireNumber(value.turnsUsed, 'continuationSummary.turnsUsed'), + continuedTurns: requireNumber(value.continuedTurns, 'continuationSummary.continuedTurns'), + stepCapHits: requireNumber(value.stepCapHits, 'continuationSummary.stepCapHits'), + capExhausted: requireBoolean(value.capExhausted, 'continuationSummary.capExhausted'), + totalRuntimeSteps: requireNumber(value.totalRuntimeSteps, 'continuationSummary.totalRuntimeSteps'), + turns: requireContinuationTurns(value.turns), + }; +} + +function requireContinuationTurns(value: unknown): HarborCellContinuationTurnSummary[] { + if (!Array.isArray(value)) throw new Error('continuationSummary.turns must be a JSON array'); + return value.map((turn, index) => { + if (!isRecord(turn)) throw new Error(`continuationSummary.turns[${index}] must be a JSON object`); + return { + turnIndex: requireNumber(turn.turnIndex, `continuationSummary.turns[${index}].turnIndex`), + status: requireStringUnion(turn.status, `continuationSummary.turns[${index}].status`, ['completed', 'failed'] as const), + stepCapHit: requireBoolean(turn.stepCapHit, `continuationSummary.turns[${index}].stepCapHit`), + runtimeSteps: requireNumber(turn.runtimeSteps, `continuationSummary.turns[${index}].runtimeSteps`), + }; + }); +} + export function summarizeCellTokens(events: readonly RuntimeEvent[]): HarborCellTokenSummary { const summary: HarborCellTokenSummary = { input: 0, @@ -236,11 +293,14 @@ export function summarizeCellContextBudget( activeEstimatedTokensSaved: 0, activeArchiveFailures: 0, archivePlaceholders: 0, + archivePlaceholderReasonCounts: {}, archiveWriteFailures: 0, retrievedArchiveToolResults: 0, retrievedArchiveEstimatedTokens: 0, archiveRetrievalSkipped: 0, + archiveRetrievalSkippedReasonCounts: {}, archiveRetrievalFailures: 0, + archiveRetrievalFailureReasonCounts: {}, }; for (const event of events) { @@ -259,11 +319,14 @@ export function summarizeCellContextBudget( summary.activeEstimatedTokensSaved += diagnostic.activeEstimatedTokensSaved ?? 0; summary.activeArchiveFailures += diagnostic.activeArchiveFailures ?? 0; summary.archivePlaceholders += diagnostic.archivePlaceholders ?? 0; + mergeCountRecord(summary.archivePlaceholderReasonCounts, diagnostic.archivePlaceholderReasonCounts); summary.archiveWriteFailures += diagnostic.archiveWriteFailures ?? 0; summary.retrievedArchiveToolResults += diagnostic.retrievedArchiveToolResults ?? 0; summary.retrievedArchiveEstimatedTokens += diagnostic.retrievedArchiveEstimatedTokens ?? 0; summary.archiveRetrievalSkipped += diagnostic.archiveRetrievalSkipped ?? 0; + mergeCountRecord(summary.archiveRetrievalSkippedReasonCounts, diagnostic.archiveRetrievalSkippedReasonCounts); summary.archiveRetrievalFailures += diagnostic.archiveRetrievalFailures ?? 0; + mergeCountRecord(summary.archiveRetrievalFailureReasonCounts, diagnostic.archiveRetrievalFailureReasonCounts); } return summary.diagnosticEvents > 0 ? summary : undefined; @@ -342,6 +405,10 @@ function validateContextBudgetSummary(value: unknown): HarborCellContextBudgetSu 'contextBudgetSummary.activeArchiveFailures', ) ?? 0, archivePlaceholders: requireNumber(value.archivePlaceholders, 'contextBudgetSummary.archivePlaceholders'), + archivePlaceholderReasonCounts: optionalCountRecord( + value.archivePlaceholderReasonCounts, + 'contextBudgetSummary.archivePlaceholderReasonCounts', + ) ?? {}, archiveWriteFailures: requireNumber(value.archiveWriteFailures, 'contextBudgetSummary.archiveWriteFailures'), retrievedArchiveToolResults: requireNumber( value.retrievedArchiveToolResults, @@ -355,13 +422,38 @@ function validateContextBudgetSummary(value: unknown): HarborCellContextBudgetSu value.archiveRetrievalSkipped, 'contextBudgetSummary.archiveRetrievalSkipped', ), + archiveRetrievalSkippedReasonCounts: optionalCountRecord( + value.archiveRetrievalSkippedReasonCounts, + 'contextBudgetSummary.archiveRetrievalSkippedReasonCounts', + ) ?? {}, archiveRetrievalFailures: requireNumber( value.archiveRetrievalFailures, 'contextBudgetSummary.archiveRetrievalFailures', ), + archiveRetrievalFailureReasonCounts: optionalCountRecord( + value.archiveRetrievalFailureReasonCounts, + 'contextBudgetSummary.archiveRetrievalFailureReasonCounts', + ) ?? {}, }; } +function mergeCountRecord(target: Record, source: Record | undefined): void { + if (!source) return; + for (const [key, value] of Object.entries(source)) { + target[key] = (target[key] ?? 0) + value; + } +} + +function optionalCountRecord(value: unknown, path: string): Record | undefined { + if (value === undefined) return undefined; + if (!isRecord(value)) throw new Error(`${path} must be a JSON object`); + const result: Record = {}; + for (const [key, raw] of Object.entries(value)) { + result[key] = requireNumber(raw, `${path}.${key}`); + } + return result; +} + function validateContextBudgetPolicySnapshot(value: unknown): HarborCellContextBudgetPolicySnapshot { if (!isRecord(value)) throw new Error('contextBudgetPolicy must be a JSON object'); const enabled = requireBoolean(value.enabled, 'contextBudgetPolicy.enabled'); diff --git a/packages/headless/src/fixed-prompt-controller.ts b/packages/headless/src/fixed-prompt-controller.ts index 93633a7db0..72939a5d45 100644 --- a/packages/headless/src/fixed-prompt-controller.ts +++ b/packages/headless/src/fixed-prompt-controller.ts @@ -5,6 +5,7 @@ import { validateHarborCellOutput, type HarborCellContextBudgetPolicySnapshot, type HarborCellContextBudgetSummary, + type HarborCellContinuationSummary, type HarborCellOutput, type HarborCellTokenSummary, } from './cell-output.js'; @@ -89,6 +90,7 @@ export interface FixedPromptTaskCompletedEvent { tokenSummary: HarborCellTokenSummary; contextBudgetPolicy?: HarborCellContextBudgetPolicySnapshot; contextBudgetSummary?: HarborCellContextBudgetSummary; + continuationSummary?: HarborCellContinuationSummary; steps: number; durationMs: number; runtimeEventsPath: string; @@ -156,6 +158,7 @@ export interface FixedPromptTaskPlumbingFailedEvent { tokenSummary: HarborCellTokenSummary; contextBudgetPolicy?: HarborCellContextBudgetPolicySnapshot; contextBudgetSummary?: HarborCellContextBudgetSummary; + continuationSummary?: HarborCellContinuationSummary; steps: number; durationMs: number; runtimeEventsPath: string; @@ -612,6 +615,7 @@ function taskCompletedEvent(input: { tokenSummary: output.cell.tokenSummary, ...(output.cell.contextBudgetPolicy ? { contextBudgetPolicy: output.cell.contextBudgetPolicy } : {}), ...(output.cell.contextBudgetSummary ? { contextBudgetSummary: output.cell.contextBudgetSummary } : {}), + ...(output.cell.continuationSummary ? { continuationSummary: output.cell.continuationSummary } : {}), steps: output.cell.steps, durationMs: output.cell.durationMs, runtimeEventsPath: output.cell.runtimeEventsPath, @@ -658,6 +662,9 @@ function taskPlumbingFailedEvent(input: { ...(input.output.cell.contextBudgetSummary ? { contextBudgetSummary: input.output.cell.contextBudgetSummary } : {}), + ...(input.output.cell.continuationSummary + ? { continuationSummary: input.output.cell.continuationSummary } + : {}), steps: input.output.cell.steps, durationMs: input.output.cell.durationMs, runtimeEventsPath: input.output.cell.runtimeEventsPath, diff --git a/packages/headless/src/harbor-cell.ts b/packages/headless/src/harbor-cell.ts index 931bbbfdb3..3d421054e6 100644 --- a/packages/headless/src/harbor-cell.ts +++ b/packages/headless/src/harbor-cell.ts @@ -9,6 +9,7 @@ import type { LlmConnection, PricingConfig, ProviderType, + RuntimeEvent, } from '@maka/core'; import { PROVIDER_DEFAULTS } from '@maka/core'; import { @@ -64,10 +65,37 @@ export interface RunHarborCellInput { ) => void | Promise; realBackendIsolation?: RealBackendIsolation; contextBudgetPolicy?: HarborCellContextBudgetPolicySnapshot; + continuationPolicy?: HarborCellContinuationPolicy; now?: () => number; newId?: () => string; } +export interface HarborCellContinuationPolicy { + enabled: boolean; + maxTurns: number; + maxTotalRuntimeSteps: number; + prompt: string; +} + +export interface HarborCellContinuationSummary { + enabled: boolean; + maxTurns: number; + maxTotalRuntimeSteps: number; + turnsUsed: number; + continuedTurns: number; + stepCapHits: number; + capExhausted: boolean; + totalRuntimeSteps: number; + turns: HarborCellContinuationTurnSummary[]; +} + +export interface HarborCellContinuationTurnSummary { + turnIndex: number; + status: InvocationResult['status']; + stepCapHit: boolean; + runtimeSteps: number; +} + export interface RunHarborCellResult { invocation: InvocationResult; output: HarborCellOutput; @@ -77,6 +105,9 @@ export interface RunHarborCellResult { export type RunHarborCellEnv = Record; +export const HARBOR_CELL_DEFAULT_CONTINUATION_PROMPT = 'Continue the same benchmark task from the current workspace state. Do not restart. If the task is complete, provide the final response.'; +const HARBOR_CELL_DEFAULT_MAX_STEPS_PER_TURN = 50; + export interface RunHarborCellFromEnvOptions { registerBackends?: RunHarborCellInput['registerBackends']; now?: () => number; @@ -232,35 +263,67 @@ export async function runHarborCell(input: RunHarborCellInput): Promise= continuationPolicy.maxTotalRuntimeSteps) break; + if (!continuationPolicy.enabled || turnIndex + 1 >= continuationPolicy.maxTurns) break; + nextText = continuationPolicy.prompt; } } catch (error) { sendMessageError = error; } - if (!invocation) { - if (sendMessageError) throw sendMessageError; + if (sendMessageError) { + invocations.push(failedInvocationFromError(sendMessageError, { + newId, + now, + sessionId: session.id, + turnId: attemptedTurnId ?? newId(), + })); + } else if (invocations.length === 0) { throw new Error('Harbor cell finished without a runtime invocation result'); } + const combinedInvocation = combineInvocations(invocations); + const continuationSummary = continuationPolicy.enabled + ? buildContinuationSummary(continuationPolicy, invocations, stepCapHits) + : undefined; await mkdir(input.outputDir, { recursive: true }); const runtimeEventsPath = join(input.outputDir, HARBOR_CELL_RUNTIME_EVENTS_FILENAME); const outputPath = join(input.outputDir, HARBOR_CELL_OUTPUT_FILENAME); - await writeFile(runtimeEventsPath, runtimeEventsJsonl(invocation), 'utf8'); + await writeFile(runtimeEventsPath, runtimeEventsJsonl(combinedInvocation), 'utf8'); const output = validateHarborCellOutput(buildHarborCellOutput({ - invocation, + invocation: combinedInvocation, runtimeEventsPath, ...(input.contextBudgetPolicy ? { contextBudgetPolicy: input.contextBudgetPolicy } : {}), + ...(continuationSummary ? { continuationSummary } : {}), })); await writeFile(outputPath, `${JSON.stringify(output, null, 2)}\n`, 'utf8'); - return { invocation, output, outputPath, runtimeEventsPath }; + return { invocation: combinedInvocation, output, outputPath, runtimeEventsPath }; } export async function runHarborCellFromEnv( @@ -274,6 +337,7 @@ export async function runHarborCellFromEnv( const resolvedEnv: RunHarborCellEnv = { ...env, MAKA_OUTPUT_DIR: outputDir, MAKA_STORAGE_ROOT: storageRoot }; const backend = backendFromEnv(resolvedEnv.MAKA_BACKEND); const contextBudgetPolicy = buildHarborCellContextBudgetPolicySnapshot(resolvedEnv); + const continuationPolicy = buildHarborCellContinuationPolicy(resolvedEnv); const baseConfig = { id: resolvedEnv.MAKA_CONFIG_ID ?? 'harbor-cell', backend, @@ -348,6 +412,7 @@ export async function runHarborCellFromEnv( outputDir, storageRoot, ...(contextBudgetPolicy ? { contextBudgetPolicy } : {}), + ...(continuationPolicy ? { continuationPolicy } : {}), ...(registerBackends ? { registerBackends } : {}), ...(backendNeedsIsolation(backend) ? { @@ -363,6 +428,114 @@ export async function runHarborCellFromEnv( }); } +export function buildHarborCellContinuationPolicy( + env: RunHarborCellEnv = process.env, +): HarborCellContinuationPolicy | undefined { + const enabled = booleanEnv(env.MAKA_HARBOR_CONTINUATION, 'MAKA_HARBOR_CONTINUATION') ?? false; + if (!enabled) return undefined; + const maxTurns = positiveIntEnv(env.MAKA_HARBOR_CONTINUATION_MAX_TURNS, 'MAKA_HARBOR_CONTINUATION_MAX_TURNS') ?? 3; + return { + enabled: true, + maxTurns, + maxTotalRuntimeSteps: positiveIntEnv( + env.MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS, + 'MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS', + ) ?? maxTurns * HARBOR_CELL_DEFAULT_MAX_STEPS_PER_TURN, + prompt: env.MAKA_HARBOR_CONTINUATION_PROMPT ?? HARBOR_CELL_DEFAULT_CONTINUATION_PROMPT, + }; +} + +function isToolCallStepCap(invocation: InvocationResult): boolean { + return invocation.failure?.class === 'tool_step_cap_reached' + || invocation.failure?.class === 'incomplete_tool_calls'; +} + +function combineInvocations(invocations: readonly InvocationResult[]): InvocationResult { + const first = invocations[0]; + const last = invocations[invocations.length - 1]; + if (!first || !last) throw new Error('cannot combine empty Harbor invocations'); + return { + invocationId: last.invocationId, + sessionId: last.sessionId, + runId: last.runId, + turnId: last.turnId, + status: last.status, + ...(last.failure ? { failure: last.failure } : {}), + events: invocations.flatMap((candidate) => candidate.events), + startedAt: first.startedAt, + finishedAt: last.finishedAt, + }; +} + +function buildContinuationSummary( + policy: HarborCellContinuationPolicy, + invocations: readonly InvocationResult[], + stepCapHits: number, +): HarborCellContinuationSummary { + const turns = invocations.map((invocation, index) => continuationTurnSummary(invocation, index)); + const runtimeSteps = turns.reduce((sum, turn) => sum + turn.runtimeSteps, 0); + return { + enabled: policy.enabled, + maxTurns: policy.maxTurns, + maxTotalRuntimeSteps: policy.maxTotalRuntimeSteps, + turnsUsed: invocations.length, + continuedTurns: Math.max(0, invocations.length - 1), + stepCapHits, + capExhausted: stepCapHits > 0 + && isToolCallStepCap(invocations[invocations.length - 1]!) + && (invocations.length >= policy.maxTurns || runtimeSteps >= policy.maxTotalRuntimeSteps), + totalRuntimeSteps: runtimeSteps, + turns, + }; +} + +function totalRuntimeSteps(invocations: readonly InvocationResult[]): number { + return invocations.reduce((sum, candidate) => sum + invocationRuntimeSteps(candidate), 0); +} + +function continuationTurnSummary( + invocation: InvocationResult, + turnIndex: number, +): HarborCellContinuationTurnSummary { + return { + turnIndex, + status: invocation.status, + stepCapHit: isToolCallStepCap(invocation), + runtimeSteps: invocationRuntimeSteps(invocation), + }; +} + +function invocationRuntimeSteps(invocation: InvocationResult): number { + return invocation.events.reduce((sum, event) => { + const runtimeSteps = event.actions?.tokenUsage?.runtimeSteps; + return sum + (runtimeSteps ?? 0); + }, 0); +} + +function failedInvocationFromError(error: unknown, input: { + newId: () => string; + now: () => number; + sessionId: string; + turnId: string; +}): InvocationResult { + const ts = input.now(); + const failureClass = error instanceof Error ? error.name : 'Error'; + return { + invocationId: input.newId(), + sessionId: input.sessionId, + runId: input.newId(), + turnId: input.turnId, + status: 'failed', + failure: { + class: failureClass, + message: error instanceof Error ? error.message : String(error), + }, + events: [], + startedAt: ts, + finishedAt: ts, + }; +} + export function buildAiSdkCellBackendRegistration(input: { provider: ProviderType; model: string; @@ -598,7 +771,7 @@ export function buildHarborCellContextBudgetPolicySnapshot( ? { activeToolResultPrune: { enabled: contextBudget.activeToolResultPrune.enabled, - maxCurrentResultEstimatedTokens: contextBudget.activeToolResultPrune.maxCurrentResultEstimatedTokens ?? 8192, + maxCurrentResultEstimatedTokens: contextBudget.activeToolResultPrune.maxCurrentResultEstimatedTokens ?? 2048, minStepNumber: contextBudget.activeToolResultPrune.minStepNumber ?? 1, }, } diff --git a/packages/headless/src/heavy-task-finalization.ts b/packages/headless/src/heavy-task-finalization.ts index fe71a75373..8502473b59 100644 --- a/packages/headless/src/heavy-task-finalization.ts +++ b/packages/headless/src/heavy-task-finalization.ts @@ -180,7 +180,12 @@ function classifyCapKind(input: HeavyTaskCompletionInput, reason: string | undef ...decisionReasons(input.decisions), ].filter((value): value is string => typeof value === 'string' && value.length > 0).join(' ').toLowerCase(); - if (haystack.includes('incomplete_tool_calls') || haystack.includes('tool_call_step') || haystack.includes('tool call step')) { + if ( + haystack.includes('incomplete_tool_calls') + || haystack.includes('tool_step_cap') + || haystack.includes('tool_call_step') + || haystack.includes('tool call step') + ) { return 'tool_call_step_cap'; } if (haystack.includes('max_tokens') || haystack.includes('max token') || haystack.includes('token cap') || haystack.includes('truncated')) { diff --git a/packages/headless/src/runtime-policy-ab-run.ts b/packages/headless/src/runtime-policy-ab-run.ts index 334ed06d74..32f959e56e 100644 --- a/packages/headless/src/runtime-policy-ab-run.ts +++ b/packages/headless/src/runtime-policy-ab-run.ts @@ -16,8 +16,15 @@ import { } from './harbor-cell.js'; export const RUNTIME_POLICY_CONTEXT_ENV_KEYS = HARBOR_CELL_CONTEXT_ENV_KEYS; +export const RUNTIME_POLICY_SHARED_AGENT_ENV_KEYS = [ + 'MAKA_HARBOR_CONTINUATION', + 'MAKA_HARBOR_CONTINUATION_MAX_TURNS', + 'MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS', + 'MAKA_HARBOR_CONTINUATION_PROMPT', +] as const; export type RuntimePolicyContextEnvKey = HarborCellContextEnvKey; +export type RuntimePolicySharedAgentEnvKey = typeof RUNTIME_POLICY_SHARED_AGENT_ENV_KEYS[number]; export interface RuntimePolicyAbArmInput { id: string; @@ -36,6 +43,7 @@ export interface RunRuntimePolicyAbComparisonInput { resumeFingerprint?: string; budgetMs?: number; nonInferiorityMargin?: number; + sharedAgentEnv?: Partial>; harborRunner: HarborTaskRunner; now?: () => number; newId?: () => string; @@ -49,13 +57,15 @@ export interface RuntimePolicyAbRunManifestInput extends Omit>; } export type RuntimePolicyAbRunManifest = AbRunManifest; export function buildRuntimePolicyAbRunManifest(input: RuntimePolicyAbRunManifestInput): RuntimePolicyAbRunManifest { - const { arms, promptHash, provider, baseUrl, model, ...abInput } = input; - const sharedMetadata = { promptHash, provider, baseUrl, model }; + const { arms, promptHash, provider, baseUrl, model, sharedAgentEnv: rawSharedAgentEnv, ...abInput } = input; + const sharedAgentEnv = sanitizeSharedAgentEnv(rawSharedAgentEnv ?? {}); + const sharedMetadata = { promptHash, provider, baseUrl, model, sharedAgentEnv }; return buildAbRunManifest({ ...abInput, experimentKind: 'runtime', @@ -70,6 +80,7 @@ export async function runRuntimePolicyAbComparison( input: RunRuntimePolicyAbComparisonInput, ): Promise { const sharedConfigFingerprint = runtimePolicySharedConfigFingerprint(input.config); + const sharedAgentEnv = sanitizeSharedAgentEnv(input.sharedAgentEnv ?? {}); return runAbComparison({ runId: input.runId, arms: [runtimeArmSpec(input.arms[0]), runtimeArmSpec(input.arms[1])], @@ -84,9 +95,11 @@ export async function runRuntimePolicyAbComparison( const contextEnv = sanitizeContextEnv(runtimeArm.contextEnv); const resumeFingerprint = runtimePolicyResumeFingerprint({ sharedConfigFingerprint, + sharedAgentEnvFingerprint: sharedAgentEnvFingerprint(sharedAgentEnv), armContextEnvFingerprint: contextEnvFingerprint(contextEnv), callerResumeFingerprint: input.resumeFingerprint, }); + const agentEnv = { ...sharedAgentEnv, ...contextEnv }; const result = await runFixedPromptController({ runId: input.runId, roundId, @@ -96,7 +109,7 @@ export async function runRuntimePolicyAbComparison( resultsTsvPath: `${input.resultsJsonlPath}.${roundId}.tsv`, tasks: [task], resumeFingerprint, - harborRunner: (runnerInput) => input.harborRunner({ ...runnerInput, agentEnv: contextEnv }), + harborRunner: (runnerInput) => input.harborRunner({ ...runnerInput, agentEnv }), ...(input.now ? { now: input.now } : {}), ...(input.newId ? { newId: input.newId } : {}), }); @@ -130,10 +143,30 @@ function sanitizeContextEnv( return normalizeHarborCellContextEnv(env); } +function sanitizeSharedAgentEnv( + env: Partial>, +): Partial> { + const allowed = new Set(RUNTIME_POLICY_SHARED_AGENT_ENV_KEYS); + const result: Partial> = {}; + for (const [key, value] of Object.entries(env)) { + if (!allowed.has(key)) { + throw new Error(`unsupported runtime policy shared agent env key: ${key}`); + } + if (value !== undefined) { + result[key as RuntimePolicySharedAgentEnvKey] = value; + } + } + return result; +} + function contextEnvFingerprint(env: Partial>): string { return `sha256:${createHash('sha256').update(canonicalJson(sanitizeContextEnv(env))).digest('hex')}`; } +function sharedAgentEnvFingerprint(env: Partial>): string { + return `sha256:${createHash('sha256').update(canonicalJson(sanitizeSharedAgentEnv(env))).digest('hex')}`; +} + function runtimePolicySharedConfigFingerprint(config: Config): string { const { systemPrompt: _systemPrompt, ...effectiveConfig } = config; return `sha256:${createHash('sha256').update(canonicalJson(effectiveConfig)).digest('hex')}`; @@ -141,12 +174,14 @@ function runtimePolicySharedConfigFingerprint(config: Config): string { function runtimePolicyResumeFingerprint(input: { sharedConfigFingerprint: string; + sharedAgentEnvFingerprint: string; armContextEnvFingerprint: string; callerResumeFingerprint?: string; }): string { return `sha256:${createHash('sha256').update(canonicalJson({ version: 'maka-runtime-policy-resume-v1', sharedConfigFingerprint: input.sharedConfigFingerprint, + sharedAgentEnvFingerprint: input.sharedAgentEnvFingerprint, armContextEnvFingerprint: input.armContextEnvFingerprint, callerResumeFingerprint: input.callerResumeFingerprint, })).digest('hex')}`; diff --git a/packages/headless/src/scorer.ts b/packages/headless/src/scorer.ts index 9882c5e90f..fa6c24104b 100644 --- a/packages/headless/src/scorer.ts +++ b/packages/headless/src/scorer.ts @@ -106,7 +106,12 @@ function taxonomyFromFailureClass(errorClass: string | undefined): AutonomousRes if (normalized.includes('budget') || normalized.includes('limit') || normalized.includes('max_tokens')) return 'budget_exhausted'; if (normalized.includes('blocked')) return 'blocked'; if (normalized.includes('policy') || normalized.includes('permission') || normalized.includes('denied')) return 'policy_denied'; - if (normalized.includes('incomplete') || normalized.includes('tool_calls') || normalized.includes('truncated')) return 'agent_incomplete'; + if ( + normalized.includes('incomplete') + || normalized.includes('tool_calls') + || normalized.includes('tool_step_cap') + || normalized.includes('truncated') + ) return 'agent_incomplete'; if (normalized.includes('infra')) return 'infra_failed'; return 'agent_failed'; } diff --git a/packages/headless/src/task-contracts.ts b/packages/headless/src/task-contracts.ts index f14f6d9a35..4d852c6a18 100644 --- a/packages/headless/src/task-contracts.ts +++ b/packages/headless/src/task-contracts.ts @@ -88,7 +88,7 @@ export function taxonomyFromResultRecord(record: ResultRecord): AutonomousResult } if (includesAny(failureText, ['blocked', 'waiting_permission'])) return 'blocked'; if (includesAny(failureText, ['policy', 'permission', 'denied'])) return 'policy_denied'; - if (includesAny(failureText, ['incomplete', 'tool_calls', 'no_submit', 'truncated'])) return 'agent_incomplete'; + if (includesAny(failureText, ['incomplete', 'tool_calls', 'tool_step_cap', 'no_submit', 'truncated'])) return 'agent_incomplete'; if (includesAny(failureText, ['verification_error'])) return 'verification_error'; if (includesAny(failureText, ['verification_failed'])) return 'verification_failed'; if (includesAny(failureText, ['unsupported_adapter'])) return 'unsupported_adapter'; diff --git a/packages/runtime/src/__tests__/active-tool-result-prune.test.ts b/packages/runtime/src/__tests__/active-tool-result-prune.test.ts index 8ca7697fdd..5f7221834e 100644 --- a/packages/runtime/src/__tests__/active-tool-result-prune.test.ts +++ b/packages/runtime/src/__tests__/active-tool-result-prune.test.ts @@ -17,6 +17,27 @@ const ZERO_USAGE: LanguageModelV3Usage = { }; describe('active current-turn tool-result pruning', () => { + test('defaults to pruning current-turn tool results above 2048 estimated tokens', async () => { + const belowDefault = await rewriteActiveToolResultsInMessages({ + messages: [largeTextToolMessage('Read', 'tool-small', 'a'.repeat(1000 * 4))], + policy: { enabled: true }, + stepNumber: 1, + turnId: 'turn-1', + archiveToolResult: () => ({ artifactId: 'unused' }), + }); + const aboveDefault = await rewriteActiveToolResultsInMessages({ + messages: [largeTextToolMessage('Read', 'tool-large', 'a'.repeat((2048 * 4) + 4))], + policy: { enabled: true }, + stepNumber: 1, + turnId: 'turn-1', + archiveToolResult: () => ({ artifactId: 'artifact-tool-large' }), + }); + + assert.equal(belowDefault.rewritten, 0); + assert.equal(aboveDefault.rewritten, 1); + assert.equal(aboveDefault.diagnosticPatch.activePrunedToolResults, 1); + }); + test('prepareStep composition returns both activeTools and rewritten messages', async () => { const originalMessages = [largeToolMessage('Read', 'tool-1', 'SECRET'.repeat(20))]; const activePrune = async () => { diff --git a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts index 5eee1430cb..00f16c4a26 100644 --- a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts +++ b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts @@ -194,6 +194,134 @@ describe('AiSdkBackend model history', () => { ]); }); + test('replays interleaved parallel RuntimeEvent tool calls as one provider tool-call block', async () => { + const model = completionModel(); + const backend = new AiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + permissionEngine: new PermissionEngine({ newId: () => 'permission-id', now: () => 1 }), + modelFactory: () => model, + tools: [], + newId: idGenerator(), + now: monotonicClock(), + }); + + await drain(backend.send({ + turnId: 'turn-current', + text: 'current user', + context: [], + runtimeContext: [ + runtimeTextEvent({ id: 'rt-u', turnId: 'turn-prev', role: 'user', author: 'user', text: 'inspect files' }), + runtimeEvent({ + id: 'rt-call-0', + turnId: 'turn-prev', + role: 'model', + author: 'agent', + content: { kind: 'function_call', id: 'tool-0', name: 'Read', args: { path: 'main.cpp' } }, + }), + runtimeEvent({ + id: 'rt-call-1', + turnId: 'turn-prev', + role: 'model', + author: 'agent', + content: { kind: 'function_call', id: 'tool-1', name: 'Read', args: { path: 'user.cpp' } }, + }), + runtimeEvent({ + id: 'rt-result-0', + turnId: 'turn-prev', + role: 'tool', + author: 'tool', + content: { kind: 'function_response', id: 'tool-0', name: 'Read', result: 'main', isError: false }, + }), + runtimeEvent({ + id: 'rt-call-2', + turnId: 'turn-prev', + role: 'model', + author: 'agent', + content: { kind: 'function_call', id: 'tool-2', name: 'Glob', args: { pattern: '*' } }, + }), + runtimeEvent({ + id: 'rt-result-1', + turnId: 'turn-prev', + role: 'tool', + author: 'tool', + content: { kind: 'function_response', id: 'tool-1', name: 'Read', result: 'user', isError: false }, + }), + runtimeEvent({ + id: 'rt-result-2', + turnId: 'turn-prev', + role: 'tool', + author: 'tool', + content: { kind: 'function_response', id: 'tool-2', name: 'Glob', result: ['main.cpp', 'user.cpp'], isError: false }, + }), + ], + })); + + assert.deepEqual(compactPrompt(model), [ + { role: 'user', content: [{ type: 'text', text: 'inspect files' }] }, + { + role: 'assistant', + content: [ + { + type: 'tool-call', + toolCallId: 'tool-0', + toolName: 'Read', + input: { path: 'main.cpp' }, + providerExecuted: undefined, + providerOptions: undefined, + }, + { + type: 'tool-call', + toolCallId: 'tool-1', + toolName: 'Read', + input: { path: 'user.cpp' }, + providerExecuted: undefined, + providerOptions: undefined, + }, + { + type: 'tool-call', + toolCallId: 'tool-2', + toolName: 'Glob', + input: { pattern: '*' }, + providerExecuted: undefined, + providerOptions: undefined, + }, + ], + }, + { + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId: 'tool-0', + toolName: 'Read', + output: { type: 'text', value: 'main' }, + providerOptions: undefined, + }, + { + type: 'tool-result', + toolCallId: 'tool-1', + toolName: 'Read', + output: { type: 'text', value: 'user' }, + providerOptions: undefined, + }, + { + type: 'tool-result', + toolCallId: 'tool-2', + toolName: 'Glob', + output: { type: 'json', value: ['main.cpp', 'user.cpp'] }, + providerOptions: undefined, + }, + ], + }, + { role: 'user', content: [{ type: 'text', text: 'current user' }] }, + ]); + }); + test('archives stale RuntimeEvent tool results before replay placeholder rewrite', async () => { const model = completionModel(); const archiveRequests: Array<{ runtimeEventId: string; serializedResult: string; bodySha256: string }> = []; diff --git a/packages/runtime/src/__tests__/runtime-runner.test.ts b/packages/runtime/src/__tests__/runtime-runner.test.ts index 4defa7df1f..605819c72f 100644 --- a/packages/runtime/src/__tests__/runtime-runner.test.ts +++ b/packages/runtime/src/__tests__/runtime-runner.test.ts @@ -341,7 +341,7 @@ describe('RuntimeRunner', () => { expect(result.events.at(-1)?.status).toBe('completed'); }); - test('raw tool-calls finish reason marks a completed terminal event incomplete', async () => { + test('raw tool-calls finish reason marks a completed terminal event as a tool step cap', async () => { const providers = makeProviders(); const flow = new ScriptFlow((ctx) => [ flowTokenUsageEvent(ctx, 'tool-calls'), @@ -352,7 +352,7 @@ describe('RuntimeRunner', () => { const result = await runner.run(makeRequest()); expect(result.status).toBe('failed'); - expect(result.failure?.class).toBe('incomplete_tool_calls'); + expect(result.failure?.class).toBe('tool_step_cap_reached'); expect(result.failure?.message).toMatch(/tool-call step cap/); }); @@ -442,7 +442,7 @@ describe('RuntimeRunner', () => { // the terminal RuntimeEvent has status='failed' but no error content. // Previously this returned class='failed', indistinguishable from other // failures; now it returns 'runtime_error' so benchmark scoring can - // distinguish runtime failures from max_tokens / incomplete_tool_calls. + // distinguish runtime failures from max_tokens / tool_step_cap_reached. const providers = makeProviders(); const flow = new ScriptFlow((ctx) => [flowTerminalEvent(ctx, 'failed')]); const runner = new RuntimeRunner({ flow, providers }); diff --git a/packages/runtime/src/active-tool-result-prune.ts b/packages/runtime/src/active-tool-result-prune.ts index e5ff47ef88..224358e0c1 100644 --- a/packages/runtime/src/active-tool-result-prune.ts +++ b/packages/runtime/src/active-tool-result-prune.ts @@ -11,7 +11,7 @@ import { type ActiveToolResultPrunePolicy, } from './context-budget.js'; -const DEFAULT_MAX_CURRENT_RESULT_ESTIMATED_TOKENS = 8192; +const DEFAULT_MAX_CURRENT_RESULT_ESTIMATED_TOKENS = 2048; const DEFAULT_CHARS_PER_TOKEN = 4; export interface ActiveToolResultPruneArchiveInput extends ActiveToolResultArchiveCandidate { diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index 5b4ec1f57a..493448371b 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -503,6 +503,7 @@ export class AiSdkBackend implements AgentBackend { let streamStatus: LlmCallRecord['status'] = 'success'; let streamErrorClass: string | undefined; let rawFinishReason: string | undefined; + let runtimeSteps = 0; let requestShapeForTelemetry: RequestShapeDiagnostic | undefined; let promptSegmentsForTelemetry: PromptSegmentEstimate[] = []; let contextBudgetForTelemetry: ContextBudgetDiagnostic | undefined; @@ -711,6 +712,9 @@ export class AiSdkBackend implements AgentBackend { for await (const chunk of result.fullStream) { if (this.aborted) break; watchdog.markActivity(); + if (chunk.type === 'step-finish') { + runtimeSteps += 1; + } if (chunk.type === 'finish' || chunk.type === 'step-finish') { rawFinishReason = rawFinishReasonString(chunk.finishReason) ?? rawFinishReason; } @@ -743,6 +747,9 @@ export class AiSdkBackend implements AgentBackend { // user can choose to send "继续" for a fresh turn. const finishReasonForGrace = await result.finishReason.catch(() => 'stop'); rawFinishReason = rawFinishReason ?? rawFinishReasonString(finishReasonForGrace); + if (finishReasonForGrace === 'tool-calls' && runtimeSteps < this.maxSteps) { + runtimeSteps = this.maxSteps; + } if ( !this.aborted && assistantText.length === 0 @@ -822,6 +829,7 @@ export class AiSdkBackend implements AgentBackend { reasoning: tokenUsage.reasoningTokens, total: tokenUsage.totalTokens, ...(tokenUsage.rawFinishReason !== undefined ? { rawFinishReason: tokenUsage.rawFinishReason } : {}), + ...(runtimeSteps > 0 ? { runtimeSteps } : {}), ...(tokenUsage.cachedInputTokens > 0 ? { cacheRead: tokenUsage.cachedInputTokens } : {}), ...(tokenUsage.cacheWriteInputTokens > 0 ? { cacheCreation: tokenUsage.cacheWriteInputTokens } : {}), ...(tokenUsageCostUsd !== undefined ? { costUsd: tokenUsageCostUsd } : {}), @@ -848,6 +856,7 @@ export class AiSdkBackend implements AgentBackend { reasoning: tokenUsage.reasoningTokens, total: tokenUsage.totalTokens, ...(tokenUsage.rawFinishReason !== undefined ? { rawFinishReason: tokenUsage.rawFinishReason } : {}), + ...(runtimeSteps > 0 ? { runtimeSteps } : {}), ...(tokenUsage.cachedInputTokens > 0 ? { cacheRead: tokenUsage.cachedInputTokens } : {}), ...(tokenUsage.cacheWriteInputTokens > 0 ? { cacheCreation: tokenUsage.cacheWriteInputTokens } : {}), ...(tokenUsageCostUsd !== undefined ? { costUsd: tokenUsageCostUsd } : {}), @@ -1611,9 +1620,55 @@ export class AiSdkBackend implements AgentBackend { private materializeRuntimeReplayPlan(plan: RuntimeEventModelReplayPlan): ModelMessage[] { const out: ModelMessage[] = []; + let toolBlock: { + calls: Extract[]; + results: Map>; + pending: Set; + } | undefined; + const flushToolBlock = () => { + if (!toolBlock) return; + out.push({ + role: 'assistant', + content: toolBlock.calls.map((item) => ({ + type: 'tool-call', + toolCallId: item.toolCallId, + toolName: item.toolName, + input: item.input, + })), + }); + for (const call of toolBlock.calls) { + const result = toolBlock.results.get(call.toolCallId); + if (!result) continue; + out.push({ + role: 'tool', + content: [{ + type: 'tool-result', + toolCallId: result.toolCallId, + toolName: result.toolName, + output: toolResultOutput(result.output, result.isError), + }], + }); + } + toolBlock = undefined; + }; + for (const item of plan.items) { + if (item.kind === 'tool_call') { + toolBlock ??= { calls: [], results: new Map(), pending: new Set() }; + toolBlock.calls.push(item); + toolBlock.pending.add(item.toolCallId); + continue; + } + if (item.kind === 'tool_result' && toolBlock?.pending.has(item.toolCallId)) { + toolBlock.results.set(item.toolCallId, item); + toolBlock.pending.delete(item.toolCallId); + if (toolBlock.pending.size === 0) flushToolBlock(); + continue; + } + flushToolBlock(); out.push(this.materializeRuntimeReplayItem(item)); } + flushToolBlock(); return out; } diff --git a/packages/runtime/src/ai-sdk-flow.ts b/packages/runtime/src/ai-sdk-flow.ts index 442932676d..0790c6a97e 100644 --- a/packages/runtime/src/ai-sdk-flow.ts +++ b/packages/runtime/src/ai-sdk-flow.ts @@ -312,6 +312,7 @@ export function mapSessionEventToRuntimeEvent( ...(event.reasoning !== undefined ? { reasoning: event.reasoning } : {}), ...(event.total !== undefined ? { total: event.total } : {}), ...(event.rawFinishReason !== undefined ? { rawFinishReason: event.rawFinishReason } : {}), + ...(event.runtimeSteps !== undefined ? { runtimeSteps: event.runtimeSteps } : {}), ...(event.cacheRead !== undefined ? { cacheRead: event.cacheRead } : {}), ...(event.cacheCreation !== undefined ? { cacheCreation: event.cacheCreation } : {}), ...(event.costUsd !== undefined ? { costUsd: event.costUsd } : {}), diff --git a/packages/runtime/src/context-budget.ts b/packages/runtime/src/context-budget.ts index 537cead957..245ba584b6 100644 --- a/packages/runtime/src/context-budget.ts +++ b/packages/runtime/src/context-budget.ts @@ -54,7 +54,7 @@ export interface StaleToolResultPrunePolicy { export interface ActiveToolResultPrunePolicy { enabled: boolean; - /** Tool result payloads above this estimate are archived and replaced. Defaults to 8192. */ + /** Tool result payloads above this estimate are archived and replaced. Defaults to 2048. */ maxCurrentResultEstimatedTokens?: number; /** Do not rewrite before this SDK step. Defaults to 1, so step 0 is untouched. */ minStepNumber?: number; diff --git a/packages/runtime/src/runtime-event-read-model.ts b/packages/runtime/src/runtime-event-read-model.ts index 19e9e4ff53..11456504ba 100644 --- a/packages/runtime/src/runtime-event-read-model.ts +++ b/packages/runtime/src/runtime-event-read-model.ts @@ -583,6 +583,7 @@ function projectTokenUsage( ...(usage.reasoning !== undefined ? { reasoning: usage.reasoning } : {}), ...(usage.total !== undefined ? { total: usage.total } : {}), ...(usage.rawFinishReason !== undefined ? { rawFinishReason: usage.rawFinishReason } : {}), + ...(usage.runtimeSteps !== undefined ? { runtimeSteps: usage.runtimeSteps } : {}), ...(usage.cacheRead !== undefined ? { cacheRead: usage.cacheRead } : {}), ...(usage.cacheCreation !== undefined ? { cacheCreation: usage.cacheCreation } : {}), ...(usage.costUsd !== undefined ? { costUsd: usage.costUsd } : {}), diff --git a/packages/runtime/src/runtime-runner.ts b/packages/runtime/src/runtime-runner.ts index 2bd30745f6..3e7de52eb2 100644 --- a/packages/runtime/src/runtime-runner.ts +++ b/packages/runtime/src/runtime-runner.ts @@ -374,7 +374,7 @@ function failureFromRawFinishReason(rawFinishReason: string | undefined): Invoca const normalized = rawFinishReason.toLowerCase().replace(/_/g, '-'); if (normalized === 'tool-calls') { return { - class: 'incomplete_tool_calls', + class: 'tool_step_cap_reached', message: 'model stopped at the tool-call step cap before completing the invocation', }; }