From ea1c392fdcc618739c053dd91d9aec0d4e98c38f Mon Sep 17 00:00:00 2001 From: me2seeks Date: Thu, 30 Jul 2026 14:15:41 +0800 Subject: [PATCH 1/3] fix(headless): reject sparse verifier attempts Validate every attempt index explicitly so array holes cannot bypass sequence and infrastructure-prefix checks. --- .../__tests__/fixed-prompt-controller.test.ts | 38 +++++++++++++++++++ .../headless/src/fixed-prompt-controller.ts | 37 ++++++++---------- 2 files changed, 53 insertions(+), 22 deletions(-) diff --git a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts index 350fcc2cd7..91951c779a 100644 --- a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts +++ b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts @@ -3184,6 +3184,44 @@ describe('fixed prompt controller', () => { }); }); + test('rejects a sparse verifier attempt array as provider infrastructure output', async () => { + await withDir(async (dir) => { + const systemPromptPath = join(dir, 'system_prompt.md'); + await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); + const attempts = new Array(2) as NonNullable< + TaskRunOutput['harbor']['verifier'] + >['attempts']; + attempts[1] = { + attempt: 2, + classification: 'failed', + durationMs: 20, + reward: 0, + }; + + const result = await runFixedPromptController({ + runId: 'run-1', + roundId: 'round-1', + config, + systemPromptPath, + resultsJsonlPath: join(dir, 'results.jsonl'), + tasks: [{ id: 'task-a', path: '/bench/task-a' }], + taskRunner: async () => + harborOutput({ + taskId: 'task-a', + reward: 0, + status: 'failed', + errorClass: 'network', + verifier: { outcome: 'failed', attempts }, + }), + }); + + assert.equal(result.events[0]?.type, 'task_infra_failed'); + assert.equal(result.events[0]?.scored, false); + assert.equal(result.events[0]?.eligible, false); + assert.equal(result.events[0]?.errorClass, 'network'); + }); + }); + test('projects a stored structured verifier failure without resampling Harbor', async () => { await withDir(async (dir) => { const systemPromptPath = join(dir, 'system_prompt.md'); diff --git a/packages/headless/src/fixed-prompt-controller.ts b/packages/headless/src/fixed-prompt-controller.ts index 9894b33e9d..efcc5028a3 100644 --- a/packages/headless/src/fixed-prompt-controller.ts +++ b/packages/headless/src/fixed-prompt-controller.ts @@ -1211,29 +1211,22 @@ function structuredVerifierGrade(harbor: unknown): 'passed' | 'failed' | undefin verifier.attempts.length > 3 ) return undefined; - if ( - verifier.attempts.some( - (attempt, index) => - !isRecord(attempt) || - attempt.attempt !== index + 1 || - typeof attempt.durationMs !== 'number' || - !Number.isFinite(attempt.durationMs) || - attempt.durationMs < 0 || - (attempt.reward !== undefined && - (typeof attempt.reward !== 'number' || !Number.isFinite(attempt.reward))), + for (let index = 0; index < verifier.attempts.length; index += 1) { + const attempt = verifier.attempts[index]; + if ( + !isRecord(attempt) || + attempt.attempt !== index + 1 || + typeof attempt.durationMs !== 'number' || + !Number.isFinite(attempt.durationMs) || + attempt.durationMs < 0 || + (attempt.reward !== undefined && + (typeof attempt.reward !== 'number' || !Number.isFinite(attempt.reward))) || + (index < verifier.attempts.length - 1 && + attempt.classification !== 'infra_setup_failed' && + attempt.classification !== 'infra_failed') ) - ) - return undefined; - if ( - verifier.attempts - .slice(0, -1) - .some( - (attempt) => - attempt.classification !== 'infra_setup_failed' && - attempt.classification !== 'infra_failed', - ) - ) - return undefined; + return undefined; + } const finalAttempt = verifier.attempts.at(-1)!; if (!isRecord(finalAttempt)) return undefined; From f14cbba404428228c8ab745f5f61e24b2c666c26 Mon Sep 17 00:00:00 2001 From: me2seeks Date: Thu, 30 Jul 2026 14:16:34 +0800 Subject: [PATCH 2/3] fix(headless): require validated verifier grades Remove the verifier-presence fallback so malformed runner output cannot mark max-token failures scored or eligible. --- .../__tests__/fixed-prompt-controller.test.ts | 61 +++++++++++++++++++ .../headless/src/fixed-prompt-controller.ts | 8 +-- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts index 91951c779a..cf0e92dd84 100644 --- a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts +++ b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts @@ -3184,6 +3184,67 @@ describe('fixed prompt controller', () => { }); }); + test('keeps rejected runner verifier output ungraded', async () => { + await withDir(async (dir) => { + const systemPromptPath = join(dir, 'system_prompt.md'); + await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); + const sparseAttempts = new Array(2) as NonNullable< + TaskRunOutput['harbor']['verifier'] + >['attempts']; + sparseAttempts[1] = { + attempt: 2, + classification: 'failed', + durationMs: 20, + reward: 0, + }; + + for (const { label, reward, verifier } of [ + { label: 'missing-attempts', reward: 0, verifier: { outcome: 'failed' } }, + { + label: 'non-array-attempts', + reward: 0, + verifier: { outcome: 'failed', attempts: 'not-an-array' }, + }, + { + label: 'sparse-attempts', + reward: 0, + verifier: { outcome: 'failed', attempts: sparseAttempts }, + }, + { + label: 'reward-disagreement', + reward: 1, + verifier: { + outcome: 'failed', + attempts: [{ attempt: 1, classification: 'failed', durationMs: 20, reward: 0 }], + }, + }, + ] as const) { + const result = await runFixedPromptController({ + runId: `run-${label}`, + roundId: 'round-1', + config, + systemPromptPath, + resultsJsonlPath: join(dir, `results-${label}.jsonl`), + tasks: [{ id: 'task-a', path: '/bench/task-a' }], + taskRunner: async () => + harborOutput({ + taskId: 'task-a', + reward, + status: 'failed', + errorClass: 'max_tokens', + verifier: verifier as unknown as TaskRunOutput['harbor']['verifier'], + }), + }); + + assert.equal(result.events[0]?.type, 'task_completed'); + assert.equal(result.events[0]?.passed, false); + assert.equal(result.events[0]?.scored, false); + assert.equal(result.events[0]?.eligible, false); + assert.equal(result.events[0]?.errorClass, 'max_tokens'); + } + }); + }); + test('rejects a sparse verifier attempt array as provider infrastructure output', async () => { await withDir(async (dir) => { const systemPromptPath = join(dir, 'system_prompt.md'); diff --git a/packages/headless/src/fixed-prompt-controller.ts b/packages/headless/src/fixed-prompt-controller.ts index efcc5028a3..03cfc243ef 100644 --- a/packages/headless/src/fixed-prompt-controller.ts +++ b/packages/headless/src/fixed-prompt-controller.ts @@ -703,13 +703,7 @@ function taskCompletedEvent(input: { const deadlineSettled = output.cell.deadlineSettlement?.source === 'benchmark.deadline'; const verifierGrade = structuredVerifierGrade(output.harbor); const verifierGraded = - output.cell.status === 'completed' || - deadlineSettled || - verifierGrade !== undefined || - ((output.cell.errorClass === 'max_tokens' || - output.cell.errorClass === 'tool_step_cap_reached' || - output.cell.errorClass === 'policy_denied') && - output.harbor.verifier !== undefined); + output.cell.status === 'completed' || deadlineSettled || verifierGrade !== undefined; const passed = verifierGraded && output.harbor.reward > 0; const rawErrorClass = output.cell.errorClass ?? 'verification_failed'; const errorClass = passed From 08354425189add739e84111fe60a435012c9868e Mon Sep 17 00:00:00 2001 From: me2seeks Date: Mon, 3 Aug 2026 11:12:20 +0800 Subject: [PATCH 3/3] fix(headless): reproject rejected stored verifier grades --- .../__tests__/fixed-prompt-controller.test.ts | 181 ++++++++++++++++-- .../headless/src/fixed-prompt-controller.ts | 105 ++++++---- 2 files changed, 240 insertions(+), 46 deletions(-) diff --git a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts index cf0e92dd84..f7687735ba 100644 --- a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts +++ b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts @@ -3249,9 +3249,7 @@ describe('fixed prompt controller', () => { await withDir(async (dir) => { const systemPromptPath = join(dir, 'system_prompt.md'); await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); - const attempts = new Array(2) as NonNullable< - TaskRunOutput['harbor']['verifier'] - >['attempts']; + const attempts = new Array(2) as NonNullable['attempts']; attempts[1] = { attempt: 2, classification: 'failed', @@ -3297,7 +3295,7 @@ describe('fixed prompt controller', () => { passed: false, scored: false, eligible: false, - errorClass: 'runtime_error', + errorClass: 'infra_failed', harbor: { reward: 0, verifier: { @@ -3330,14 +3328,32 @@ describe('fixed prompt controller', () => { }); }); - test('keeps malformed stored verifier attempts on the ungraded path', async () => { + test('reprojects stale fallback scores when stored verifier data is rejected', async () => { await withDir(async (dir) => { const systemPromptPath = join(dir, 'system_prompt.md'); await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); + const sparseAttempts = new Array(2); + sparseAttempts[1] = { + attempt: 2, + classification: 'failed', + durationMs: 20, + reward: 0, + }; - for (const [label, verifier] of [ - ['missing', { outcome: 'failed' }], - ['non-array', { outcome: 'failed', attempts: 'not-an-array' }], + for (const [label, reward, verifier, errorClass] of [ + ['missing-attempts', 0, { outcome: 'failed' }, 'max_tokens'], + ['non-array-attempts', 0, { outcome: 'failed', attempts: 'not-an-array' }, 'max_tokens'], + ['sparse-attempts', 0, { outcome: 'failed', attempts: sparseAttempts }, 'max_tokens'], + [ + 'reward-disagreement', + 1, + { + outcome: 'failed', + attempts: [{ attempt: 1, classification: 'failed', durationMs: 20, reward: 0 }], + }, + 'max_tokens', + ], + ['policy-denied', 0, { outcome: 'failed' }, 'policy_denied'], ] as const) { const resultsJsonlPath = join(dir, `results-${label}.jsonl`); const stored = taskCompletedEvent({ taskId: 'task-a' }); @@ -3349,10 +3365,10 @@ describe('fixed prompt controller', () => { ...stored, status: 'failed', passed: false, - scored: false, - eligible: false, - errorClass: 'infra_failed', - harbor: { reward: 0, verifier }, + scored: true, + eligible: true, + errorClass, + harbor: { reward, verifier }, })}\n`, 'utf8', ); @@ -3373,13 +3389,152 @@ describe('fixed prompt controller', () => { assert.equal(runnerCalls, 0); assert.equal(result.events[0]?.type, 'task_completed'); + assert.equal(result.events[0]?.passed, false); assert.equal(result.events[0]?.scored, false); assert.equal(result.events[0]?.eligible, false); - assert.equal(result.events[0]?.errorClass, 'infra_failed'); + assert.equal(result.events[0]?.errorClass, errorClass); } }); }); + test('removes a stale pass from rejected legacy verifier output', async () => { + await withDir(async (dir) => { + const resultsJsonlPath = join(dir, 'results.jsonl'); + const stored = taskCompletedEvent({ taskId: 'task-a' }); + assert.equal(stored.type, 'task_completed'); + if (stored.type !== 'task_completed') throw new Error('expected completed fixture'); + const { errorClass: _storedErrorClass, ...legacy } = stored; + await writeFile( + resultsJsonlPath, + `${JSON.stringify({ + ...legacy, + status: 'failed', + passed: true, + scored: true, + eligible: true, + harbor: { + reward: 1, + verifier: { + outcome: 'failed', + attempts: [{ attempt: 1, classification: 'failed', durationMs: 20, reward: 0 }], + }, + }, + })}\n`, + 'utf8', + ); + + const [projected] = await readFixedPromptWal(resultsJsonlPath); + + assert.equal(projected?.type, 'task_completed'); + assert.equal(projected?.passed, false); + assert.equal(projected?.scored, false); + assert.equal(projected?.eligible, false); + assert.equal(projected?.errorClass, 'verification_failed'); + }); + }); + + test('preserves non-verifier scoring authorities while projecting stored WAL', async () => { + await withDir(async (dir) => { + const resultsJsonlPath = join(dir, 'results.jsonl'); + const base = taskCompletedEvent({ taskId: 'task-a' }); + assert.equal(base.type, 'task_completed'); + if (base.type !== 'task_completed') throw new Error('expected completed fixture'); + const malformedVerifier = { outcome: 'failed' }; + await writeFile( + resultsJsonlPath, + [ + { + ...base, + id: 'tool-step-cap', + taskId: 'tool-step-cap', + status: 'failed', + passed: false, + scored: true, + eligible: true, + errorClass: 'tool_step_cap_reached', + harbor: { reward: 0, verifier: malformedVerifier }, + }, + { + ...base, + id: 'deadline', + taskId: 'deadline', + status: 'failed', + passed: false, + scored: true, + eligible: true, + errorClass: 'budget_exhausted', + deadlineSettlement: { source: 'benchmark.deadline', mode: 'immediate' }, + harbor: { reward: 0, verifier: malformedVerifier }, + }, + { + ...base, + id: 'completed', + taskId: 'completed', + passed: false, + scored: true, + eligible: true, + errorClass: 'verification_failed', + harbor: { reward: 0, verifier: malformedVerifier }, + }, + { + ...base, + id: 'no-verifier', + taskId: 'no-verifier', + status: 'failed', + passed: false, + scored: true, + eligible: true, + errorClass: 'max_tokens', + harbor: { reward: 0 }, + }, + ] + .map((event) => JSON.stringify(event)) + .join('\n') + '\n', + 'utf8', + ); + + const projected = await readFixedPromptWal(resultsJsonlPath); + assert.deepEqual( + projected.map((event) => { + assert.equal(event.type, 'task_completed'); + if (event.type !== 'task_completed') throw new Error('expected completed event'); + return { + taskId: event.taskId, + scored: event.scored, + eligible: event.eligible, + errorClass: event.errorClass, + }; + }), + [ + { + taskId: 'tool-step-cap', + scored: false, + eligible: true, + errorClass: 'tool_step_cap_reached', + }, + { + taskId: 'deadline', + scored: true, + eligible: true, + errorClass: 'budget_exhausted', + }, + { + taskId: 'completed', + scored: true, + eligible: true, + errorClass: 'verification_failed', + }, + { + taskId: 'no-verifier', + scored: true, + eligible: true, + errorClass: 'max_tokens', + }, + ], + ); + }); + }); + test('projects a stored structured verifier pass without resampling Harbor', async () => { await withDir(async (dir) => { const systemPromptPath = join(dir, 'system_prompt.md'); diff --git a/packages/headless/src/fixed-prompt-controller.ts b/packages/headless/src/fixed-prompt-controller.ts index 03cfc243ef..f9bafe794b 100644 --- a/packages/headless/src/fixed-prompt-controller.ts +++ b/packages/headless/src/fixed-prompt-controller.ts @@ -700,22 +700,14 @@ function taskCompletedEvent(input: { }): FixedPromptTaskCompletedEvent { const { output } = input; const promptHash = attestedPromptHash(output.cell); - const deadlineSettled = output.cell.deadlineSettlement?.source === 'benchmark.deadline'; const verifierGrade = structuredVerifierGrade(output.harbor); - const verifierGraded = - output.cell.status === 'completed' || deadlineSettled || verifierGrade !== undefined; - const passed = verifierGraded && output.harbor.reward > 0; - const rawErrorClass = output.cell.errorClass ?? 'verification_failed'; - const errorClass = passed - ? undefined - : deadlineSettled - ? 'budget_exhausted' - : verifierGrade === 'failed' && isUnscoredCellFailure(rawErrorClass) - ? 'runtime_error' - : rawErrorClass; - const scored = - verifierGraded && (verifierGrade !== undefined || !isUnscoredCellFailure(errorClass)); - const agentFailure = output.cell.status === 'failed' && errorClass === 'tool_step_cap_reached'; + const scoring = taskCompletedScoringProjection({ + status: output.cell.status, + reward: output.harbor.reward, + errorClass: output.cell.errorClass, + deadlineSettled: output.cell.deadlineSettlement?.source === 'benchmark.deadline', + verifierGrade, + }); return { schemaVersion: FIXED_PROMPT_WAL_SCHEMA_VERSION, type: 'task_completed', @@ -726,10 +718,7 @@ function taskCompletedEvent(input: { ...(input.resumeFingerprint ? { resumeFingerprint: input.resumeFingerprint } : {}), taskId: input.taskId, status: output.cell.status, - passed, - scored, - eligible: scored || agentFailure, - ...(errorClass ? { errorClass } : {}), + ...scoring, ...(promptHash ? { promptHash } : {}), ...(output.cell.executionIdentity ? { executionIdentity: output.cell.executionIdentity } : {}), runtimeRefs: output.cell.runtimeRefs, @@ -764,6 +753,35 @@ function taskCompletedEvent(input: { }; } +function taskCompletedScoringProjection(input: { + status: FixedPromptTaskCompletedEvent['status']; + reward: number; + errorClass: string | undefined; + deadlineSettled: boolean; + verifierGrade: 'passed' | 'failed' | undefined; +}): Pick { + const verifierGraded = + input.status === 'completed' || input.deadlineSettled || input.verifierGrade !== undefined; + const passed = verifierGraded && input.reward > 0; + const rawErrorClass = input.errorClass ?? 'verification_failed'; + const errorClass = passed + ? undefined + : input.deadlineSettled + ? 'budget_exhausted' + : input.verifierGrade === 'failed' && isUnscoredCellFailure(rawErrorClass) + ? 'runtime_error' + : rawErrorClass; + const scored = + verifierGraded && (input.verifierGrade !== undefined || !isUnscoredCellFailure(errorClass)); + const agentFailure = input.status === 'failed' && errorClass === 'tool_step_cap_reached'; + return { + passed, + scored, + eligible: scored || agentFailure, + ...(errorClass ? { errorClass } : {}), + }; +} + function isUnscoredCellFailure( errorClass: string | undefined, ): errorClass is UnscoredCellFailureClass { @@ -1169,21 +1187,42 @@ function projectLegacyTimeoutOutcome(event: FixedPromptWalEvent): FixedPromptWal function projectStructuredVerifierOutcome(event: FixedPromptWalEvent): FixedPromptWalEvent { if (event.type !== 'task_completed') return event; const verifierGrade = structuredVerifierGrade(event.harbor); - if (verifierGrade === undefined) return event; - if (verifierGrade === 'failed') { - return { - ...event, - passed: false, - scored: true, - eligible: true, - }; - } - const { errorClass: _legacyFailureClass, ...rest } = event; + const legacyVerifierPresenceFallback = + event.passed === true || + event.errorClass === 'max_tokens' || + event.errorClass === 'tool_step_cap_reached' || + event.errorClass === 'policy_denied'; + const rejectedLegacyVerifierFallback = + verifierGrade === undefined && + event.status === 'failed' && + event.deadlineSettlement?.source !== 'benchmark.deadline' && + isRecord(event.harbor) && + event.harbor.verifier !== undefined && + legacyVerifierPresenceFallback; + if (verifierGrade === undefined && !rejectedLegacyVerifierFallback) return event; + + const scoring = taskCompletedScoringProjection({ + status: event.status, + reward: + isRecord(event.harbor) && + typeof event.harbor.reward === 'number' && + Number.isFinite(event.harbor.reward) + ? event.harbor.reward + : 0, + errorClass: event.errorClass, + deadlineSettled: event.deadlineSettlement?.source === 'benchmark.deadline', + verifierGrade, + }); + const { + passed: _legacyPassed, + scored: _legacyScored, + eligible: _legacyEligible, + errorClass: _legacyErrorClass, + ...stable + } = event; return { - ...rest, - passed: true, - scored: true, - eligible: true, + ...stable, + ...scoring, }; }