From 7a61b9107047fca98e9f03e5bc9739a61948c402 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Tue, 14 Jul 2026 04:03:05 +0800 Subject: [PATCH] fix(cu): preserve unknown dispatch outcomes --- .../src/__tests__/computer-use-tools.test.ts | 61 +++++++++++++++++++ packages/runtime/src/computer-use-tools.ts | 17 ++++-- 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/packages/runtime/src/__tests__/computer-use-tools.test.ts b/packages/runtime/src/__tests__/computer-use-tools.test.ts index 057c4d7150..b829aa8312 100644 --- a/packages/runtime/src/__tests__/computer-use-tools.test.ts +++ b/packages/runtime/src/__tests__/computer-use-tools.test.ts @@ -1200,6 +1200,67 @@ describe('buildComputerUseTools — the `maka_computer` MakaTool', () => { }); } + test('generic outcome_unknown remains model-visible while requiring reobserve', async () => { + const backend = fakeBackend() as CuDispatchBackend & { + observeApp: NonNullable; + }; + backend.observeApp = async () => observation(); + backend.run = async () => ({ + outcome: { + ok: false, + error: 'outcome_unknown', + message: 'delivery may have occurred', + }, + }); + const tools = buildComputerUseTools({ backend }); + const [tool] = tools; + const observed = await tool.impl( + { action: 'observe', app: 'Fixture' } as never, + ctx(), + ) as { text: string }; + + const result = await tool.impl({ + action: 'left_click', + observation_id: JSON.parse(observed.text).observation_id, + coordinate: [25, 30], + } as never, ctx()) as { text: string }; + + assert.match(result.text, /outcome_unknown/); + assert.doesNotMatch(result.text, /failed: reobserve_required/); + assert.equal(tools.sessionEvents.snapshot('s1').status, 'reobserve_required'); + }); + + test('semantic outcome_unknown remains model-visible while requiring reobserve', async () => { + const backend = fakeBackend() as CuDispatchBackend & { + observeApp: NonNullable; + runSemantic: NonNullable; + }; + backend.observeApp = async () => observation(); + backend.runSemantic = async () => ({ + outcome: { + ok: false, + error: 'outcome_unknown', + message: 'semantic delivery may have occurred', + }, + }); + const tools = buildComputerUseTools({ backend }); + const [tool] = tools; + const observed = await tool.impl( + { action: 'observe', app: 'Fixture' } as never, + ctx(), + ) as { text: string }; + + const result = await tool.impl({ + action: 'click_element', + observation_id: JSON.parse(observed.text).observation_id, + element_id: '5', + } as never, ctx()) as { text: string }; + + assert.match(result.text, /outcome_unknown/); + assert.doesNotMatch(result.text, /failed: reobserve_required/); + assert.equal(tools.sessionEvents.snapshot('s1').status, 'reobserve_required'); + }); + test('a queued keyboard mutation cannot silently target a newer frame', async () => { let releaseClick!: () => void; const clickGate = new Promise((resolve) => { diff --git a/packages/runtime/src/computer-use-tools.ts b/packages/runtime/src/computer-use-tools.ts index 6bc5821eb9..ab24de9df4 100644 --- a/packages/runtime/src/computer-use-tools.ts +++ b/packages/runtime/src/computer-use-tools.ts @@ -1368,10 +1368,15 @@ export function buildComputerUseTools(deps: { if (!presentation.result) return bindingFailure('capture_failed'); result = presentation.result; applyTypedOutcomeState(state, result.outcome); - const postDispatchFailure = validateActionLease(state, actionLease); - if (postDispatchFailure) { - presentation.finish(); - return postDispatchFailure; + if (result.outcome.ok) { + const postDispatchFailure = validateActionLease( + state, + actionLease, + ); + if (postDispatchFailure) { + presentation.finish(); + return postDispatchFailure; + } } } finally { consumeFailure = consumeBoundAction(record, binding); @@ -1481,7 +1486,7 @@ export function buildComputerUseTools(deps: { if (presentation.blocked) return presentation.blocked; result = presentation.result; if (result) applyTypedOutcomeState(state, result.outcome); - if (observationLease?.ok) { + if (result?.outcome.ok && observationLease?.ok) { const validated = state.validateObservationLease( observationLease.lease, ); @@ -1490,7 +1495,7 @@ export function buildComputerUseTools(deps: { return sessionFailure(validated.reason); } } - if (actionLease) { + if (result?.outcome.ok && actionLease) { const leaseFailure = validateActionLease(state, actionLease); if (leaseFailure) { presentation.finish();