From f7a0c30bf6793011aff2b2b7f0b91ff3bc326a75 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 25 Aug 2026 16:33:25 -0700 Subject: [PATCH 01/12] Classify replay stream failures by ownership Signed-off-by: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> --- .changeset/typed-replay-stream-failures.md | 5 + packages/world-vercel/src/events-v4.test.ts | 155 +++++++++---------- packages/world-vercel/src/events-v4.ts | 160 +++++++++++--------- packages/world-vercel/src/events.test.ts | 79 +++++++++- packages/world-vercel/src/events.ts | 10 +- packages/world-vercel/src/frames.ts | 18 ++- 6 files changed, 254 insertions(+), 173 deletions(-) create mode 100644 .changeset/typed-replay-stream-failures.md diff --git a/.changeset/typed-replay-stream-failures.md b/.changeset/typed-replay-stream-failures.md new file mode 100644 index 0000000000..3bdcfe0b90 --- /dev/null +++ b/.changeset/typed-replay-stream-failures.md @@ -0,0 +1,5 @@ +--- +'@workflow/world-vercel': patch +--- + +Classify incomplete replay streams and malformed event responses as typed world failures, leaving recovery to existing retry layers. diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index bf21528397..02bf89dfb7 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -456,7 +456,10 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { {}, { token: 'test-token', dispatcher: agent } ) - ).rejects.toThrow(); + ).rejects.toMatchObject({ + name: 'WorkflowWorldError', + code: 'SCHEMA_VALIDATION', + }); agent.assertNoPendingInterceptors(); }); @@ -536,120 +539,58 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { {}, { token: 'test-token', dispatcher: agent } ) - ).rejects.toThrow(); + ).rejects.toMatchObject({ + name: 'WorkflowWorldError', + code: 'SCHEMA_VALIDATION', + }); }); - it('throws when the stream ends without the end sentinel (truncated response)', async () => { + it.each([ + 'after a complete frame', + 'inside a frame', + ])('leaves retrying a stream that ends %s to its caller', async (endPosition) => { const origin = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; const agent = new MockAgent(); agent.disableNetConnect(); - - // A complete event frame but NO `{_end: 1}` sentinel — what a response - // truncated on a frame boundary looks like. Returning this as a - // successful page would silently drop events with hasMore=false. - const frames = encodeFrame( + const completeFrame = encodeFrame( { eventId: 'evnt_1', runId: 'wrun_1', eventType: 'run_created', - createdAt: '2026-06-10T00:00:00.000Z', + createdAt: CREATED_AT, eventData: { deploymentId: 'dpl_1', workflowName: 'workflow', input: null, }, }, - new Uint8Array(0) + new Uint8Array() ); + const responseBody = + endPosition === 'inside a frame' + ? completeFrame.slice(0, -1) + : completeFrame; agent .get(origin) .intercept({ - path: '/api/v4/runs/wrun_1/events?limit=500', + path: '/api/v4/runs/wrun_1/events?returnAll=true', method: 'GET', }) - .reply(200, frames, { + .reply(200, responseBody, { headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, }); - await expect( getWorkflowRunEventsV4( 'wrun_1', - { limit: 500 }, + {}, { token: 'test-token', dispatcher: agent } ) - ).rejects.toThrow(/end-of-stream sentinel/); - }); - - it('resumes a truncated full stream after its last accepted event', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); - - agent - .get(origin) - .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true', - method: 'GET', - }) - .reply( - 200, - encodeFrame( - { - eventId: 'evnt_1', - runId: 'wrun_1', - eventType: 'run_created', - createdAt: CREATED_AT, - eventData: { - deploymentId: 'dpl_1', - workflowName: 'workflow', - input: null, - }, - }, - new Uint8Array() - ), - { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } - ); - agent - .get(origin) - .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true&cursor=eid%3Aevnt_1', - method: 'GET', - }) - .reply( - 200, - Buffer.concat([ - encodeFrame( - { - eventId: 'evnt_2', - runId: 'wrun_1', - eventType: 'run_started', - createdAt: CREATED_AT, - }, - new Uint8Array() - ), - encodeFrame( - { _end: 1, next: 'eid:evnt_2', hasMore: false }, - new Uint8Array() - ), - ]), - { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } - ); - - const result = await getWorkflowRunEventsV4( - 'wrun_1', - {}, - { token: 'test-token', dispatcher: agent } - ); - - expect(result.events.map((event) => event.eventId)).toEqual([ - 'evnt_1', - 'evnt_2', - ]); - expect(result.cursor).toBe('eid:evnt_2'); - expect(result.hasMore).toBe(false); + ).rejects.toMatchObject({ + name: 'WorkflowWorldError', + code: 'TRANSPORT', + }); agent.assertNoPendingInterceptors(); }); }); @@ -861,6 +802,50 @@ describe('v4 transport uses global fetch (observability)', () => { }); describe('createWorkflowRunEventV4 over HTTP', () => { + it.each([ + ['an empty body', () => new Response(), 'PARSE_ERROR'], + [ + 'malformed CBOR', + () => new Response(new Uint8Array([0xff, 0xfe, 0xfd])), + 'PARSE_ERROR', + ], + [ + 'a body read failure', + () => + new Response( + new ReadableStream({ + start(controller) { + controller.error(new Error('socket closed')); + }, + }) + ), + 'TRANSPORT', + ], + ])('classifies %s', async (_case, response, code) => { + const fetchSpy = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValueOnce(response()); + + try { + await expect( + createWorkflowRunEventV4( + { + runId: 'wrun_1', + eventType: 'step_completed', + specVersion: 2, + correlationId: 'step_1', + }, + { token: 'test-token' } + ) + ).rejects.toMatchObject({ + name: 'WorkflowWorldError', + code, + }); + } finally { + fetchSpy.mockRestore(); + } + }); + it('POSTs to the /events/:eventType alias and decodes the response', async () => { const origin = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 6ffdbc51da..f5562964a3 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -42,6 +42,7 @@ import { type DecodedFrame, decodeFrames, encodeFrame, + IncompleteFrameError, V4_FRAME_CONTENT_TYPE, } from './frames.js'; import { @@ -809,7 +810,9 @@ export async function createWorkflowRunEventV4( const contentType = response.headers.get('content-type'); if (contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { - throw new Error('v4 createEvent: unexpected event page'); + throw new WorkflowWorldError('v4 createEvent: unexpected event page', { + code: 'SCHEMA_VALIDATION', + }); } return decodeCreateEventResponse(response, input.eventType); @@ -822,9 +825,19 @@ async function decodeCreateEventResponse( response: FrameResponseLike, eventType: T ): Promise & { event: Event }> { - const bodyBytes = new Uint8Array(await response.arrayBuffer()); + let bodyBytes: Uint8Array; + try { + bodyBytes = new Uint8Array(await response.arrayBuffer()); + } catch (cause) { + throw new WorkflowWorldError( + 'v4 createEvent: failed to read response body', + { code: 'TRANSPORT', cause } + ); + } if (bodyBytes.byteLength === 0) { - throw new Error('v4 createEvent: empty response body'); + throw new WorkflowWorldError('v4 createEvent: empty response body', { + code: 'PARSE_ERROR', + }); } const schema: z.ZodType & { event: Event }> = CreateEventV4BodySchemas[eventType].refine( @@ -833,7 +846,16 @@ async function decodeCreateEventResponse( (eventType === 'hook_created' && event.eventType === 'hook_conflict'), { path: ['event', 'eventType'] } ); - const parsedBody = schema.safeParse(decode(bodyBytes)); + let decoded: unknown; + try { + decoded = decode(bodyBytes); + } catch (cause) { + throw new WorkflowWorldError('v4 createEvent: invalid CBOR response body', { + code: 'PARSE_ERROR', + cause, + }); + } + const parsedBody = schema.safeParse(decoded); if (!parsedBody.success) { throw new WorkflowWorldError('v4 createEvent: invalid response body', { code: 'SCHEMA_VALIDATION', @@ -852,9 +874,13 @@ export async function createWorkflowRunStartedEventV4( 'event-stream', config ); - const events: Event[] = []; - const page = await consumeEventFrameStream(response, 'createEvent', events); - assert(page.cursor, 'v4 createEvent: event stream missing cursor'); + const page = await consumeEventFrameStream(response, 'createEvent'); + if (!page.cursor) { + throw new WorkflowWorldError( + 'v4 createEvent: event stream missing cursor', + { code: 'SCHEMA_VALIDATION' } + ); + } const maxEvents = MaxEventsHeaderSchema.safeParse( response.headers.get(MAX_EVENTS_HEADER) ); @@ -865,7 +891,7 @@ export async function createWorkflowRunStartedEventV4( }); } - return { events, ...page, maxEvents: maxEvents.data }; + return { ...page, maxEvents: maxEvents.data }; } /** One event of a v4 batch POST, index-aligned with the response results. */ @@ -1308,14 +1334,12 @@ export async function createHookReceivedPreloadEventV4( }; } - const events: Event[] = []; - const page = await consumeEventFrameStream(response, 'createEvent', events); + const page = await consumeEventFrameStream(response, 'createEvent'); const maxEvents = MaxEventsHeaderSchema.safeParse( response.headers.get(MAX_EVENTS_HEADER) ); return { kind: 'stream', - events, ...page, canonicalEventId: response.headers.get(EVENT_ID_HEADER) ?? undefined, maxEvents: maxEvents.success ? maxEvents.data : undefined, @@ -1447,35 +1471,58 @@ function streamErrorFrameToError( async function consumeEventFrameStream( response: Response, - opName: string, - events: Event[] -): Promise> { + opName: string +): Promise { const contentType = response.headers.get('content-type'); if (!contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { - throw new Error( - `v4 ${opName}: expected ${V4_FRAME_CONTENT_TYPE}, got ${contentType ?? '(none)'}` + throw new WorkflowWorldError( + `v4 ${opName}: expected ${V4_FRAME_CONTENT_TYPE}, got ${contentType ?? '(none)'}`, + { code: 'SCHEMA_VALIDATION' } ); } + if (!response.body) { + throw new WorkflowWorldError(`v4 ${opName}: response body is missing`, { + code: 'TRANSPORT', + }); + } - const chunks = response.body as unknown as AsyncIterable; - - for await (const frame of decodeFrames(chunks)) { - if (frame.meta._end === 1) { - const end = EventStreamEndSchema.parse(frame.meta); - return { cursor: end.next ?? null, hasMore: end.hasMore }; - } - if (frame.meta._error === 1) { - throw streamErrorFrameToError(frame.meta, opName); + const events: Event[] = []; + try { + for await (const frame of decodeFrames(response.body)) { + if (frame.meta._end === 1) { + const end = EventStreamEndSchema.parse(frame.meta); + return { + events, + cursor: end.next ?? null, + hasMore: end.hasMore, + }; + } + if (frame.meta._error === 1) { + throw streamErrorFrameToError(frame.meta, opName); + } + if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { + throw new Error(`v4 ${opName}: unexpected control frame`); + } + events.push(decodeEventFrame(frame)); } - if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { - throw new Error(`v4 ${opName}: unexpected control frame`); + } catch (cause) { + if (CorruptedEventLogError.is(cause) || WorkflowWorldError.is(cause)) { + throw cause; } - events.push(decodeEventFrame(frame)); + const incomplete = cause instanceof IncompleteFrameError; + throw new WorkflowWorldError( + `v4 ${opName}: ${incomplete ? 'incomplete' : 'invalid'} event frame stream`, + { + code: incomplete ? 'TRANSPORT' : 'SCHEMA_VALIDATION', + cause, + } + ); } - throw new Error( + throw new WorkflowWorldError( `v4 ${opName}: frame stream ended without the end-of-stream sentinel ` + - `(${events.length} events read) — truncated response?` + `(${events.length} events read)`, + { code: 'TRANSPORT' } ); } @@ -1492,16 +1539,15 @@ async function consumeListFrameStream( url: string, headers: Headers, config: APIConfig | undefined, - opName: string, - events: Event[] -): Promise> { + opName: string +): Promise { const response = await fetchV4( url, { method: 'GET', headers }, config, opName ); - return consumeEventFrameStream(response, opName, events); + return consumeEventFrameStream(response, opName); } /** @@ -1533,8 +1579,8 @@ function paginationToQuery(params: ListEventsV4Params): string { * cursor from the sentinel frame. * * Eagerly drains the stream into memory to match the existing - * `getWorkflowRunEvents` contract. A truncated full response resumes - * after its last validated event instead of downloading accepted frames again. + * `getWorkflowRunEvents` contract. An incomplete response is a transport + * failure; the caller owns retrying the operation. */ export async function getWorkflowRunEventsV4( runId: string, @@ -1542,37 +1588,10 @@ export async function getWorkflowRunEventsV4( config?: APIConfig ): Promise { const { baseUrl, headers } = await getHttpConfig(config); - const events: Event[] = []; - let cursor = params.cursor; - - while (true) { - const url = - `${baseUrl}/v4/runs/${encodeURIComponent(runId)}/events` + - paginationToQuery({ ...params, cursor }); - try { - const page = await consumeListFrameStream( - url, - headers, - config, - 'listEvents', - events - ); - return { events, ...page }; - } catch (error) { - if (CorruptedEventLogError.is(error) || WorkflowWorldError.is(error)) { - throw error; - } - const lastEvent = events.at(-1); - if ( - params.limit !== undefined || - !lastEvent || - `eid:${lastEvent.eventId}` === cursor - ) { - throw error; - } - cursor = `eid:${lastEvent.eventId}`; - } - } + const url = + `${baseUrl}/v4/runs/${encodeURIComponent(runId)}/events` + + paginationToQuery(params); + return consumeListFrameStream(url, headers, config, 'listEvents'); } /** @@ -1601,13 +1620,10 @@ export async function getEventsByCorrelationIdV4( sp.set('runId', runId); appendListParams(sp, params); const url = `${baseUrl}/v4/events?${sp.toString()}`; - const events: Event[] = []; - const page = await consumeListFrameStream( + return consumeListFrameStream( url, headers, config, - 'listEventsByCorrelationId', - events + 'listEventsByCorrelationId' ); - return { events, ...page }; } diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index 5fea9675a4..bf81002c91 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -4,7 +4,7 @@ import type { AnyEventRequest, CreateEventParams } from '@workflow/world'; import { decode, encode } from 'cbor-x'; import { ulid } from 'ulid'; import { MockAgent } from 'undici'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { createWorkflowRunEvent, getWorkflowRunEvents, @@ -1046,6 +1046,52 @@ describe('createWorkflowRunEvent response coercion', () => { agent.assertNoPendingInterceptors(); }); + it('classifies a run_started stream missing lifecycle events as a world schema error', async () => { + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce( + new Response( + Buffer.concat([ + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: STARTED_AT, + specVersion: 5, + eventData: {}, + }, + new Uint8Array() + ), + encodeFrame( + { _end: 1, next: 'eid:evnt_1', hasMore: false }, + new Uint8Array() + ), + ]), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-max-events': '10000', + }, + } + ) + ); + + try { + await expect( + createWorkflowRunEvent( + 'wrun_1', + { eventType: 'run_started', specVersion: 5 }, + undefined, + { token: 'test-token' } + ) + ).rejects.toMatchObject({ + name: 'WorkflowWorldError', + code: 'SCHEMA_VALIDATION', + }); + } finally { + fetchSpy.mockRestore(); + } + }); + it('threads the wait entity through to the EventResult', async () => { const agent = mockAgent(); agent @@ -1842,7 +1888,7 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { agent.assertNoPendingInterceptors(); }); - it('rejects a truncated preload stream (no end sentinel)', async () => { + it('retries a truncated preload by repeating the idempotent POST', async () => { const agent = mockAgent(); agent .get(ORIGIN) @@ -1868,13 +1914,30 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { ), { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } ); - - await expect( - createWorkflowRunEvent('wrun_1', hookReceivedRequest(), preloadParams, { - token: 'test-token', - dispatcher: agent, + agent + .get(ORIGIN) + .intercept({ + path: '/api/v4/runs/wrun_1/events/hook_received', + method: 'POST', + headers: { accept: V4_FRAME_CONTENT_TYPE }, }) - ).rejects.toThrow(/end-of-stream sentinel/); + .reply(200, hookReplayStreamResponse(), { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-event-id': 'evnt_4', + 'x-wf-max-events': '10000', + }, + }); + + const result = await createWorkflowRunEvent( + 'wrun_1', + hookReceivedRequest(), + preloadParams, + { token: 'test-token', dispatcher: agent } + ); + + expect(result.event?.eventId).toBe('evnt_4'); + expect(result.events).toHaveLength(4); agent.assertNoPendingInterceptors(); }); diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 92d0be103a..3588849b10 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -759,13 +759,15 @@ async function createWorkflowRunEventInner( (event) => event.eventType === 'run_started' ); if (!runCreated) { - throw new Error( - 'v4 createEvent: run_started stream is missing run_created' + throw new WorkflowWorldError( + 'v4 createEvent: run_started stream is missing run_created', + { code: 'SCHEMA_VALIDATION' } ); } if (!runStarted) { - throw new Error( - 'v4 createEvent: run_started stream is missing run_started' + throw new WorkflowWorldError( + 'v4 createEvent: run_started stream is missing run_started', + { code: 'SCHEMA_VALIDATION' } ); } diff --git a/packages/world-vercel/src/frames.ts b/packages/world-vercel/src/frames.ts index 93809ab179..2e839dd6e7 100644 --- a/packages/world-vercel/src/frames.ts +++ b/packages/world-vercel/src/frames.ts @@ -18,6 +18,9 @@ export interface DecodedFrame { body: Uint8Array; } +/** The response body stopped before the next complete frame was available. */ +export class IncompleteFrameError extends Error {} + // The protocol consumer validates the event or control-frame shape after the // body is available. The byte codec only requires a CBOR object here. const CborObjectSchema = z.record(z.string(), z.unknown()); @@ -71,7 +74,14 @@ export async function* decodeFrames( const parts: Uint8Array[] = [buffer]; let byteLength = buffer.byteLength; while (byteLength < needed) { - const chunk = await chunks.next(); + let chunk: IteratorResult; + try { + chunk = await chunks.next(); + } catch (cause) { + throw new IncompleteFrameError('decodeFrames: source stream failed', { + cause, + }); + } if (chunk.done) return false; if (chunk.value.byteLength === 0) continue; parts.push(chunk.value); @@ -104,12 +114,12 @@ export async function* decodeFrames( take(4); if (!(await refill(metaLen))) { - throw new Error('decodeFrames: truncated meta block'); + throw new IncompleteFrameError('decodeFrames: truncated meta block'); } const meta = CborObjectSchema.parse(decode(take(metaLen))); if (!(await refill(4))) { - throw new Error('decodeFrames: truncated body length'); + throw new IncompleteFrameError('decodeFrames: truncated body length'); } const bodyLen = new DataView( buffer.buffer, @@ -119,7 +129,7 @@ export async function* decodeFrames( take(4); if (bodyLen > 0 && !(await refill(bodyLen))) { - throw new Error('decodeFrames: truncated body bytes'); + throw new IncompleteFrameError('decodeFrames: truncated body bytes'); } // Slice (not subarray) so the yielded body owns its bytes, so later // reads into the buffer won't overwrite it; bodyLen 0 yields empty. From 2653939b49d22dd7cdcb9cae7bea2615e4c67853 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 26 Aug 2026 13:19:51 -0700 Subject: [PATCH 02/12] fix(world-vercel): resume truncated replay streams --- packages/world-vercel/src/events-v4.test.ts | 157 ++++++++++++++++++-- packages/world-vercel/src/events-v4.ts | 154 +++++++++++++++---- packages/world-vercel/src/events.test.ts | 46 +++--- 3 files changed, 295 insertions(+), 62 deletions(-) diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 02bf89dfb7..b06ab064f8 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -545,10 +545,78 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { }); }); - it.each([ - 'after a complete frame', - 'inside a frame', - ])('leaves retrying a stream that ends %s to its caller', async (endPosition) => { + it('resumes a truncated full stream after its last complete event', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events?returnAll=true', + method: 'GET', + }) + .reply( + 200, + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: CREATED_AT, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + input: null, + }, + }, + new Uint8Array() + ), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events?returnAll=true&cursor=eid%3Aevnt_1', + method: 'GET', + }) + .reply( + 200, + Buffer.concat([ + encodeFrame( + { + eventId: 'evnt_2', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: CREATED_AT, + }, + new Uint8Array() + ), + encodeFrame( + { _end: 1, next: 'eid:evnt_2', hasMore: false }, + new Uint8Array() + ), + ]), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + + const result = await getWorkflowRunEventsV4( + 'wrun_1', + {}, + { token: 'test-token', dispatcher: agent } + ); + + expect(result.events.map((event) => event.eventId)).toEqual([ + 'evnt_1', + 'evnt_2', + ]); + expect(result.cursor).toBe('eid:evnt_2'); + expect(result.hasMore).toBe(false); + agent.assertNoPendingInterceptors(); + }); + + it('surfaces a truncated stream that provides no recovery cursor', async () => { const origin = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; const agent = new MockAgent(); @@ -567,10 +635,6 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { }, new Uint8Array() ); - const responseBody = - endPosition === 'inside a frame' - ? completeFrame.slice(0, -1) - : completeFrame; agent .get(origin) @@ -578,7 +642,7 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { path: '/api/v4/runs/wrun_1/events?returnAll=true', method: 'GET', }) - .reply(200, responseBody, { + .reply(200, completeFrame.slice(0, -1), { headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, }); await expect( @@ -1020,6 +1084,81 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); + it('continues a truncated run_started replay without re-posting it', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + headers: { accept: V4_FRAME_CONTENT_TYPE }, + }) + .reply( + 200, + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: CREATED_AT, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + input: null, + }, + }, + new Uint8Array() + ), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-max-events': '10000', + }, + } + ); + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events?returnAll=true&cursor=eid%3Aevnt_1&remoteRefBehavior=resolve', + method: 'GET', + }) + .reply( + 200, + Buffer.concat([ + encodeFrame( + { + eventId: 'evnt_2', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: CREATED_AT, + }, + new Uint8Array() + ), + encodeFrame( + { _end: 1, next: 'eid:evnt_2', hasMore: false }, + new Uint8Array() + ), + ]), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + + const result = await createWorkflowRunStartedEventV4( + { runId: 'wrun_1', specVersion: 5 }, + { token: 'test-token', dispatcher: agent } + ); + + expect(result.events.map((event) => event.eventId)).toEqual([ + 'evnt_1', + 'evnt_2', + ]); + expect(result.cursor).toBe('eid:evnt_2'); + agent.assertNoPendingInterceptors(); + }); + it('requires the event-stream response requested by run_started', async () => { const origin = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index f5562964a3..0deebd8add 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -874,7 +874,7 @@ export async function createWorkflowRunStartedEventV4( 'event-stream', config ); - const page = await consumeEventFrameStream(response, 'createEvent'); + const page = await consumeReplayLogResponse(response, input.runId, config); if (!page.cursor) { throw new WorkflowWorldError( 'v4 createEvent: event stream missing cursor', @@ -1312,9 +1312,10 @@ export type HookReceivedPreloadV4Result = * A server that supports the lazy-hook replay stream answers the consumer's * idempotent re-ensure with the run's complete replay log as v4 frames: * the same event-frame sequence LIST uses, ending with the `_end` sentinel. - * A truncated stream (EOF without the sentinel) throws; the write is - * deduplicated by the server's `(runId, resumeId)` constraint, so retrying - * the whole request is safe and converges on the same canonical event. + * A truncated stream resumes after its last validated event. If it ends before + * any event is available to form a cursor, the write is deduplicated by the + * server's `(runId, resumeId)` constraint, so retrying the whole request is + * still safe and converges on the same canonical event. */ export async function createHookReceivedPreloadEventV4( input: CreateEventV4InputBase, @@ -1334,7 +1335,7 @@ export async function createHookReceivedPreloadEventV4( }; } - const page = await consumeEventFrameStream(response, 'createEvent'); + const page = await consumeReplayLogResponse(response, input.runId, config); const maxEvents = MaxEventsHeaderSchema.safeParse( response.headers.get(MAX_EVENTS_HEADER) ); @@ -1469,10 +1470,30 @@ function streamErrorFrameToError( ); } +const MAX_PARTIAL_EVENT_STREAM_RETRIES = 3; + +type EventFrameStreamResult = + | ({ kind: 'complete' } & ListEventsV4Result) + | { + kind: 'partial'; + events: Event[]; + error: WorkflowWorldError; + }; + +function decodeStreamEventFrame(frame: DecodedFrame, opName: string): Event { + if (frame.meta._error === 1) { + throw streamErrorFrameToError(frame.meta, opName); + } + if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { + throw new Error(`v4 ${opName}: unexpected control frame`); + } + return decodeEventFrame(frame); +} + async function consumeEventFrameStream( response: Response, opName: string -): Promise { +): Promise { const contentType = response.headers.get('content-type'); if (!contentType?.startsWith(V4_FRAME_CONTENT_TYPE)) { throw new WorkflowWorldError( @@ -1492,38 +1513,85 @@ async function consumeEventFrameStream( if (frame.meta._end === 1) { const end = EventStreamEndSchema.parse(frame.meta); return { + kind: 'complete', events, cursor: end.next ?? null, hasMore: end.hasMore, }; } - if (frame.meta._error === 1) { - throw streamErrorFrameToError(frame.meta, opName); - } - if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { - throw new Error(`v4 ${opName}: unexpected control frame`); - } - events.push(decodeEventFrame(frame)); + events.push(decodeStreamEventFrame(frame, opName)); } } catch (cause) { if (CorruptedEventLogError.is(cause) || WorkflowWorldError.is(cause)) { throw cause; } const incomplete = cause instanceof IncompleteFrameError; - throw new WorkflowWorldError( + const error = new WorkflowWorldError( `v4 ${opName}: ${incomplete ? 'incomplete' : 'invalid'} event frame stream`, { code: incomplete ? 'TRANSPORT' : 'SCHEMA_VALIDATION', cause, } ); + if (!incomplete) throw error; + return { kind: 'partial', events, error }; + } + + return { + kind: 'partial', + events, + error: new WorkflowWorldError( + `v4 ${opName}: frame stream ended without the end-of-stream sentinel ` + + `(${events.length} events read)`, + { code: 'TRANSPORT' } + ), + }; +} + +/** + * Finish a replay-log POST without throwing away frames that were already + * validated. A graceful partial page and a transport-truncated body both + * continue with the ordinary GET endpoint from the response's last cursor. + */ +async function consumeReplayLogResponse( + response: Response, + runId: string, + config?: APIConfig +): Promise { + const consumed = await consumeEventFrameStream(response, 'createEvent'); + if (consumed.kind === 'complete' && !consumed.hasMore) { + return { + events: consumed.events, + cursor: consumed.cursor, + hasMore: consumed.hasMore, + }; + } + + const lastEvent = consumed.events.at(-1); + const cursor = + consumed.kind === 'complete' + ? consumed.cursor + : lastEvent + ? `eid:${lastEvent.eventId}` + : null; + if (!cursor) { + if (consumed.kind === 'partial') throw consumed.error; + throw new WorkflowWorldError( + 'v4 createEvent: partial event stream missing cursor', + { code: 'SCHEMA_VALIDATION' } + ); } - throw new WorkflowWorldError( - `v4 ${opName}: frame stream ended without the end-of-stream sentinel ` + - `(${events.length} events read)`, - { code: 'TRANSPORT' } + const suffix = await getWorkflowRunEventsV4( + runId, + { cursor, remoteRefBehavior: 'resolve' }, + config ); + return { + events: [...consumed.events, ...suffix.events], + cursor: suffix.cursor ?? cursor, + hasMore: suffix.hasMore, + }; } /** @@ -1540,7 +1608,7 @@ async function consumeListFrameStream( headers: Headers, config: APIConfig | undefined, opName: string -): Promise { +): Promise { const response = await fetchV4( url, { method: 'GET', headers }, @@ -1579,8 +1647,10 @@ function paginationToQuery(params: ListEventsV4Params): string { * cursor from the sentinel frame. * * Eagerly drains the stream into memory to match the existing - * `getWorkflowRunEvents` contract. An incomplete response is a transport - * failure; the caller owns retrying the operation. + * `getWorkflowRunEvents` contract. A truncated full response resumes after its + * last validated event, with a bounded retry count and a forward-progress + * guard. Explicitly paginated requests retain their one-page contract and + * surface truncation to the caller. */ export async function getWorkflowRunEventsV4( runId: string, @@ -1588,10 +1658,36 @@ export async function getWorkflowRunEventsV4( config?: APIConfig ): Promise { const { baseUrl, headers } = await getHttpConfig(config); - const url = - `${baseUrl}/v4/runs/${encodeURIComponent(runId)}/events` + - paginationToQuery(params); - return consumeListFrameStream(url, headers, config, 'listEvents'); + const events: Event[] = []; + let cursor = params.cursor ?? null; + + for (let partialRetries = 0; ; partialRetries++) { + const url = + `${baseUrl}/v4/runs/${encodeURIComponent(runId)}/events` + + paginationToQuery({ ...params, cursor: cursor ?? undefined }); + const consumed = await consumeListFrameStream( + url, + headers, + config, + 'listEvents' + ); + events.push(...consumed.events); + if (consumed.kind === 'complete') { + return { events, cursor: consumed.cursor, hasMore: consumed.hasMore }; + } + + const lastEvent = events.at(-1); + const nextCursor = lastEvent ? `eid:${lastEvent.eventId}` : null; + if ( + params.limit !== undefined || + partialRetries === MAX_PARTIAL_EVENT_STREAM_RETRIES || + !nextCursor || + nextCursor === cursor + ) { + throw consumed.error; + } + cursor = nextCursor; + } } /** @@ -1620,10 +1716,16 @@ export async function getEventsByCorrelationIdV4( sp.set('runId', runId); appendListParams(sp, params); const url = `${baseUrl}/v4/events?${sp.toString()}`; - return consumeListFrameStream( + const consumed = await consumeListFrameStream( url, headers, config, 'listEventsByCorrelationId' ); + if (consumed.kind === 'partial') throw consumed.error; + return { + events: consumed.events, + cursor: consumed.cursor, + hasMore: consumed.hasMore, + }; } diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index bf81002c91..2f953515a8 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -1603,8 +1603,8 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { return out; } - function hookReplayStreamResponse(): Uint8Array { - return concatFrames([ + function hookReplayFrames(): Uint8Array[] { + return [ encodeFrame( { eventId: 'evnt_1', @@ -1660,7 +1660,11 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { { _end: 1, next: 'eid:evnt_4', hasMore: false }, new Uint8Array() ), - ]); + ]; + } + + function hookReplayStreamResponse(): Uint8Array { + return concatFrames(hookReplayFrames()); } it('decodes a streamed replay log into event + reconstructed run + page', async () => { @@ -1888,7 +1892,7 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { agent.assertNoPendingInterceptors(); }); - it('retries a truncated preload by repeating the idempotent POST', async () => { + it('continues a truncated preload after its last validated event', async () => { const agent = mockAgent(); agent .get(ORIGIN) @@ -1897,35 +1901,22 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { method: 'POST', headers: { accept: V4_FRAME_CONTENT_TYPE }, }) - .reply( - 200, - encodeFrame( - { - eventId: 'evnt_4', - runId: 'wrun_1', - eventType: 'hook_received', - correlationId: 'hook_1', - createdAt: new Date('2026-06-10T00:00:03.000Z'), - specVersion: 2, - resumeId: RESUME_ID, - eventData: { token: 'tok-preload' }, - }, - PAYLOAD - ), - { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } - ); + .reply(200, concatFrames(hookReplayFrames().slice(0, 2)), { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-event-id': 'evnt_4', + 'x-wf-max-events': '10000', + }, + }); agent .get(ORIGIN) .intercept({ - path: '/api/v4/runs/wrun_1/events/hook_received', - method: 'POST', - headers: { accept: V4_FRAME_CONTENT_TYPE }, + path: /\/api\/v4\/runs\/wrun_1\/events\?.*cursor=eid%3Aevnt_2/, + method: 'GET', }) - .reply(200, hookReplayStreamResponse(), { + .reply(200, concatFrames(hookReplayFrames().slice(2)), { headers: { 'content-type': V4_FRAME_CONTENT_TYPE, - 'x-wf-event-id': 'evnt_4', - 'x-wf-max-events': '10000', }, }); @@ -1938,6 +1929,7 @@ describe('createWorkflowRunEvent hook_received replay preload', () => { expect(result.event?.eventId).toBe('evnt_4'); expect(result.events).toHaveLength(4); + expect(result.maxEvents).toBe(10000); agent.assertNoPendingInterceptors(); }); From cf646d92eafb39255451329dedc98fbcc0682a7b Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 26 Aug 2026 13:35:06 -0700 Subject: [PATCH 03/12] fix(world-vercel): require replay continuation cursor --- packages/world-vercel/src/events-v4.test.ts | 71 +++++++++++++++++++++ packages/world-vercel/src/events-v4.ts | 6 ++ 2 files changed, 77 insertions(+) diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index b06ab064f8..70014393f4 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -1159,6 +1159,77 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); + it('rejects a non-empty run_started continuation without its trailing cursor', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + headers: { accept: V4_FRAME_CONTENT_TYPE }, + }) + .reply( + 200, + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: CREATED_AT, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + input: null, + }, + }, + new Uint8Array() + ), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-max-events': '10000', + }, + } + ); + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events?returnAll=true&cursor=eid%3Aevnt_1&remoteRefBehavior=resolve', + method: 'GET', + }) + .reply( + 200, + Buffer.concat([ + encodeFrame( + { + eventId: 'evnt_2', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: CREATED_AT, + }, + new Uint8Array() + ), + encodeFrame({ _end: 1, hasMore: false }, new Uint8Array()), + ]), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + + await expect( + createWorkflowRunStartedEventV4( + { runId: 'wrun_1', specVersion: 5 }, + { token: 'test-token', dispatcher: agent } + ) + ).rejects.toMatchObject({ + code: 'SCHEMA_VALIDATION', + message: 'v4 createEvent: non-empty continuation missing cursor', + }); + agent.assertNoPendingInterceptors(); + }); + it('requires the event-stream response requested by run_started', async () => { const origin = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 0deebd8add..4bd1b025d0 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -1587,6 +1587,12 @@ async function consumeReplayLogResponse( { cursor, remoteRefBehavior: 'resolve' }, config ); + if (suffix.events.length > 0 && !suffix.cursor) { + throw new WorkflowWorldError( + 'v4 createEvent: non-empty continuation missing cursor', + { code: 'SCHEMA_VALIDATION' } + ); + } return { events: [...consumed.events, ...suffix.events], cursor: suffix.cursor ?? cursor, From bd05dca3777580037dfe7b97625b4089d8c86166 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 26 Aug 2026 13:43:44 -0700 Subject: [PATCH 04/12] refactor(world-vercel): simplify replay recovery --- packages/world-vercel/src/events-v4.test.ts | 103 +++++-------------- packages/world-vercel/src/events-v4.ts | 105 ++++++++------------ 2 files changed, 63 insertions(+), 145 deletions(-) diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 70014393f4..954b7eeb38 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -1084,7 +1084,10 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); - it('continues a truncated run_started replay without re-posting it', async () => { + it.each([ + ['continues a truncated run_started replay', 'eid:evnt_2'], + ['rejects a continuation without its trailing cursor', undefined], + ])('%s', async (_name, suffixCursor) => { const origin = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; const agent = new MockAgent(); @@ -1139,94 +1142,34 @@ describe('createWorkflowRunEventV4 over HTTP', () => { new Uint8Array() ), encodeFrame( - { _end: 1, next: 'eid:evnt_2', hasMore: false }, + { + _end: 1, + ...(suffixCursor ? { next: suffixCursor } : {}), + hasMore: false, + }, new Uint8Array() ), ]), { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } ); - const result = await createWorkflowRunStartedEventV4( + const request = createWorkflowRunStartedEventV4( { runId: 'wrun_1', specVersion: 5 }, { token: 'test-token', dispatcher: agent } ); - - expect(result.events.map((event) => event.eventId)).toEqual([ - 'evnt_1', - 'evnt_2', - ]); - expect(result.cursor).toBe('eid:evnt_2'); - agent.assertNoPendingInterceptors(); - }); - - it('rejects a non-empty run_started continuation without its trailing cursor', async () => { - const origin = - WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; - const agent = new MockAgent(); - agent.disableNetConnect(); - - agent - .get(origin) - .intercept({ - path: '/api/v4/runs/wrun_1/events/run_started', - method: 'POST', - headers: { accept: V4_FRAME_CONTENT_TYPE }, - }) - .reply( - 200, - encodeFrame( - { - eventId: 'evnt_1', - runId: 'wrun_1', - eventType: 'run_created', - createdAt: CREATED_AT, - eventData: { - deploymentId: 'dpl_1', - workflowName: 'workflow', - input: null, - }, - }, - new Uint8Array() - ), - { - headers: { - 'content-type': V4_FRAME_CONTENT_TYPE, - 'x-wf-max-events': '10000', - }, - } - ); - agent - .get(origin) - .intercept({ - path: '/api/v4/runs/wrun_1/events?returnAll=true&cursor=eid%3Aevnt_1&remoteRefBehavior=resolve', - method: 'GET', - }) - .reply( - 200, - Buffer.concat([ - encodeFrame( - { - eventId: 'evnt_2', - runId: 'wrun_1', - eventType: 'run_started', - createdAt: CREATED_AT, - }, - new Uint8Array() - ), - encodeFrame({ _end: 1, hasMore: false }, new Uint8Array()), - ]), - { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } - ); - - await expect( - createWorkflowRunStartedEventV4( - { runId: 'wrun_1', specVersion: 5 }, - { token: 'test-token', dispatcher: agent } - ) - ).rejects.toMatchObject({ - code: 'SCHEMA_VALIDATION', - message: 'v4 createEvent: non-empty continuation missing cursor', - }); + if (suffixCursor) { + const result = await request; + expect(result.events.map((event) => event.eventId)).toEqual([ + 'evnt_1', + 'evnt_2', + ]); + expect(result.cursor).toBe(suffixCursor); + } else { + await expect(request).rejects.toMatchObject({ + code: 'SCHEMA_VALIDATION', + message: 'v4 createEvent: non-empty continuation missing cursor', + }); + } agent.assertNoPendingInterceptors(); }); diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 4bd1b025d0..54c15b6819 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -1470,24 +1470,17 @@ function streamErrorFrameToError( ); } -const MAX_PARTIAL_EVENT_STREAM_RETRIES = 3; - -type EventFrameStreamResult = - | ({ kind: 'complete' } & ListEventsV4Result) - | { - kind: 'partial'; - events: Event[]; - error: WorkflowWorldError; - }; +type EventFrameStreamResult = ListEventsV4Result & { + partialError?: WorkflowWorldError; +}; -function decodeStreamEventFrame(frame: DecodedFrame, opName: string): Event { - if (frame.meta._error === 1) { - throw streamErrorFrameToError(frame.meta, opName); - } - if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { - throw new Error(`v4 ${opName}: unexpected control frame`); - } - return decodeEventFrame(frame); +function partialEventFrameStream( + events: Event[], + partialError: WorkflowWorldError +): EventFrameStreamResult { + const eventId = events.at(-1)?.eventId; + if (!eventId) throw partialError; + return { events, cursor: `eid:${eventId}`, hasMore: true, partialError }; } async function consumeEventFrameStream( @@ -1513,13 +1506,18 @@ async function consumeEventFrameStream( if (frame.meta._end === 1) { const end = EventStreamEndSchema.parse(frame.meta); return { - kind: 'complete', events, cursor: end.next ?? null, hasMore: end.hasMore, }; } - events.push(decodeStreamEventFrame(frame, opName)); + if (frame.meta._error === 1) { + throw streamErrorFrameToError(frame.meta, opName); + } + if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { + throw new Error(`v4 ${opName}: unexpected control frame`); + } + events.push(decodeEventFrame(frame)); } } catch (cause) { if (CorruptedEventLogError.is(cause) || WorkflowWorldError.is(cause)) { @@ -1534,18 +1532,17 @@ async function consumeEventFrameStream( } ); if (!incomplete) throw error; - return { kind: 'partial', events, error }; + return partialEventFrameStream(events, error); } - return { - kind: 'partial', + return partialEventFrameStream( events, - error: new WorkflowWorldError( + new WorkflowWorldError( `v4 ${opName}: frame stream ended without the end-of-stream sentinel ` + `(${events.length} events read)`, { code: 'TRANSPORT' } - ), - }; + ) + ); } /** @@ -1558,24 +1555,10 @@ async function consumeReplayLogResponse( runId: string, config?: APIConfig ): Promise { - const consumed = await consumeEventFrameStream(response, 'createEvent'); - if (consumed.kind === 'complete' && !consumed.hasMore) { - return { - events: consumed.events, - cursor: consumed.cursor, - hasMore: consumed.hasMore, - }; - } - - const lastEvent = consumed.events.at(-1); - const cursor = - consumed.kind === 'complete' - ? consumed.cursor - : lastEvent - ? `eid:${lastEvent.eventId}` - : null; - if (!cursor) { - if (consumed.kind === 'partial') throw consumed.error; + const page = await consumeEventFrameStream(response, 'createEvent'); + if (!page.hasMore) return page; + if (!page.cursor) { + if (page.partialError) throw page.partialError; throw new WorkflowWorldError( 'v4 createEvent: partial event stream missing cursor', { code: 'SCHEMA_VALIDATION' } @@ -1584,7 +1567,7 @@ async function consumeReplayLogResponse( const suffix = await getWorkflowRunEventsV4( runId, - { cursor, remoteRefBehavior: 'resolve' }, + { cursor: page.cursor, remoteRefBehavior: 'resolve' }, config ); if (suffix.events.length > 0 && !suffix.cursor) { @@ -1594,8 +1577,8 @@ async function consumeReplayLogResponse( ); } return { - events: [...consumed.events, ...suffix.events], - cursor: suffix.cursor ?? cursor, + events: [...page.events, ...suffix.events], + cursor: suffix.cursor ?? page.cursor, hasMore: suffix.hasMore, }; } @@ -1654,9 +1637,9 @@ function paginationToQuery(params: ListEventsV4Params): string { * * Eagerly drains the stream into memory to match the existing * `getWorkflowRunEvents` contract. A truncated full response resumes after its - * last validated event, with a bounded retry count and a forward-progress - * guard. Explicitly paginated requests retain their one-page contract and - * surface truncation to the caller. + * last validated event until the sentinel arrives. A forward-progress guard + * prevents retry loops. Explicitly paginated requests retain their one-page + * contract and surface truncation to the caller. */ export async function getWorkflowRunEventsV4( runId: string, @@ -1667,7 +1650,7 @@ export async function getWorkflowRunEventsV4( const events: Event[] = []; let cursor = params.cursor ?? null; - for (let partialRetries = 0; ; partialRetries++) { + for (;;) { const url = `${baseUrl}/v4/runs/${encodeURIComponent(runId)}/events` + paginationToQuery({ ...params, cursor: cursor ?? undefined }); @@ -1678,21 +1661,17 @@ export async function getWorkflowRunEventsV4( 'listEvents' ); events.push(...consumed.events); - if (consumed.kind === 'complete') { + if (!consumed.partialError) { return { events, cursor: consumed.cursor, hasMore: consumed.hasMore }; } - - const lastEvent = events.at(-1); - const nextCursor = lastEvent ? `eid:${lastEvent.eventId}` : null; if ( params.limit !== undefined || - partialRetries === MAX_PARTIAL_EVENT_STREAM_RETRIES || - !nextCursor || - nextCursor === cursor + !consumed.cursor || + consumed.cursor === cursor ) { - throw consumed.error; + throw consumed.partialError; } - cursor = nextCursor; + cursor = consumed.cursor; } } @@ -1728,10 +1707,6 @@ export async function getEventsByCorrelationIdV4( config, 'listEventsByCorrelationId' ); - if (consumed.kind === 'partial') throw consumed.error; - return { - events: consumed.events, - cursor: consumed.cursor, - hasMore: consumed.hasMore, - }; + if (consumed.partialError) throw consumed.partialError; + return consumed; } From b429134a36c1336f6447cbd6ca93393b5d7bc48a Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:20:45 -0700 Subject: [PATCH 05/12] fix(world-vercel): append replay pages safely --- packages/world-vercel/src/events-v4.ts | 35 +++++++++++++------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 54c15b6819..1600b36515 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -1649,30 +1649,29 @@ export async function getWorkflowRunEventsV4( const { baseUrl, headers } = await getHttpConfig(config); const events: Event[] = []; let cursor = params.cursor ?? null; + let consumed: EventFrameStreamResult; - for (;;) { + do { const url = `${baseUrl}/v4/runs/${encodeURIComponent(runId)}/events` + paginationToQuery({ ...params, cursor: cursor ?? undefined }); - const consumed = await consumeListFrameStream( - url, - headers, - config, - 'listEvents' - ); - events.push(...consumed.events); - if (!consumed.partialError) { - return { events, cursor: consumed.cursor, hasMore: consumed.hasMore }; + consumed = await consumeListFrameStream(url, headers, config, 'listEvents'); + for (const event of consumed.events) { + events.push(event); } - if ( - params.limit !== undefined || - !consumed.cursor || - consumed.cursor === cursor - ) { - throw consumed.partialError; + if (consumed.partialError) { + if ( + params.limit !== undefined || + !consumed.cursor || + consumed.cursor === cursor + ) { + throw consumed.partialError; + } + cursor = consumed.cursor; } - cursor = consumed.cursor; - } + } while (consumed.partialError); + + return { events, cursor: consumed.cursor, hasMore: consumed.hasMore }; } /** From 32cd767e1357bbe9a2d29d490078ae1c9ff66335 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:28:30 -0700 Subject: [PATCH 06/12] fix(world-vercel): bound partial stream continuations --- packages/world-vercel/src/events-v4.test.ts | 47 +++++++++++++++++++++ packages/world-vercel/src/events-v4.ts | 13 ++++-- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 954b7eeb38..66aca9f63f 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -616,6 +616,53 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); + it('limits truncated full-stream recovery to three continuations', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + for (const [cursor, eventId] of [ + [undefined, 'evnt_1'], + ['eid:evnt_1', 'evnt_2'], + ['eid:evnt_2', 'evnt_3'], + ['eid:evnt_3', 'evnt_4'], + ] as const) { + agent + .get(origin) + .intercept({ + path: + '/api/v4/runs/wrun_1/events?returnAll=true' + + (cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''), + method: 'GET', + }) + .reply( + 200, + encodeFrame( + { + eventId, + runId: 'wrun_1', + eventType: 'run_started', + createdAt: CREATED_AT, + }, + new Uint8Array() + ), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + } + + await expect( + getWorkflowRunEventsV4( + 'wrun_1', + {}, + { token: 'test-token', dispatcher: agent } + ) + ).rejects.toThrow( + 'frame stream ended without the end-of-stream sentinel (1 events read)' + ); + agent.assertNoPendingInterceptors(); + }); + it('surfaces a truncated stream that provides no recovery cursor', async () => { const origin = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 1600b36515..2be3f0c387 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -1474,6 +1474,8 @@ type EventFrameStreamResult = ListEventsV4Result & { partialError?: WorkflowWorldError; }; +const MAX_PARTIAL_STREAM_CONTINUATIONS = 3; + function partialEventFrameStream( events: Event[], partialError: WorkflowWorldError @@ -1637,9 +1639,9 @@ function paginationToQuery(params: ListEventsV4Params): string { * * Eagerly drains the stream into memory to match the existing * `getWorkflowRunEvents` contract. A truncated full response resumes after its - * last validated event until the sentinel arrives. A forward-progress guard - * prevents retry loops. Explicitly paginated requests retain their one-page - * contract and surface truncation to the caller. + * last validated event until the sentinel arrives, for up to three continuation + * requests. A forward-progress guard prevents retry loops. Explicitly paginated + * requests retain their one-page contract and surface truncation to the caller. */ export async function getWorkflowRunEventsV4( runId: string, @@ -1649,6 +1651,7 @@ export async function getWorkflowRunEventsV4( const { baseUrl, headers } = await getHttpConfig(config); const events: Event[] = []; let cursor = params.cursor ?? null; + let partialContinuations = 0; let consumed: EventFrameStreamResult; do { @@ -1663,10 +1666,12 @@ export async function getWorkflowRunEventsV4( if ( params.limit !== undefined || !consumed.cursor || - consumed.cursor === cursor + consumed.cursor === cursor || + partialContinuations === MAX_PARTIAL_STREAM_CONTINUATIONS ) { throw consumed.partialError; } + partialContinuations++; cursor = consumed.cursor; } } while (consumed.partialError); From edeaf78bc9fc9a7676632011efa2422f2b16a672 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:58:54 -0700 Subject: [PATCH 07/12] fix(world-vercel): bound replay continuations --- .changeset/typed-replay-stream-failures.md | 2 +- packages/world-vercel/src/events-v4.test.ts | 87 +++++++++++++++++++-- packages/world-vercel/src/events-v4.ts | 38 ++++++--- 3 files changed, 111 insertions(+), 16 deletions(-) diff --git a/.changeset/typed-replay-stream-failures.md b/.changeset/typed-replay-stream-failures.md index 3bdcfe0b90..3bcec935c4 100644 --- a/.changeset/typed-replay-stream-failures.md +++ b/.changeset/typed-replay-stream-failures.md @@ -2,4 +2,4 @@ '@workflow/world-vercel': patch --- -Classify incomplete replay streams and malformed event responses as typed world failures, leaving recovery to existing retry layers. +Classify malformed replay responses as typed world failures and resume incomplete streams from their last validated event. diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 66aca9f63f..d45ce47666 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -1132,9 +1132,10 @@ describe('createWorkflowRunEventV4 over HTTP', () => { }); it.each([ - ['continues a truncated run_started replay', 'eid:evnt_2'], - ['rejects a continuation without its trailing cursor', undefined], - ])('%s', async (_name, suffixCursor) => { + ['continues a truncated run_started replay', 'eid:evnt_2', true], + ['rejects a continuation without its trailing cursor', undefined, false], + ['rejects a non-advancing continuation cursor', 'eid:evnt_1', false], + ])('%s', async (_name, suffixCursor, succeeds) => { const origin = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; const agent = new MockAgent(); @@ -1204,7 +1205,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { { runId: 'wrun_1', specVersion: 5 }, { token: 'test-token', dispatcher: agent } ); - if (suffixCursor) { + if (succeeds) { const result = await request; expect(result.events.map((event) => event.eventId)).toEqual([ 'evnt_1', @@ -1214,12 +1215,88 @@ describe('createWorkflowRunEventV4 over HTTP', () => { } else { await expect(request).rejects.toMatchObject({ code: 'SCHEMA_VALIDATION', - message: 'v4 createEvent: non-empty continuation missing cursor', + message: 'v4 createEvent: continuation did not advance cursor', }); } agent.assertNoPendingInterceptors(); }); + it('shares the three-continuation limit with a partial run_started POST', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_started', + method: 'POST', + headers: { accept: V4_FRAME_CONTENT_TYPE }, + }) + .reply( + 200, + encodeFrame( + { + eventId: 'evnt_1', + runId: 'wrun_1', + eventType: 'run_created', + createdAt: CREATED_AT, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'workflow', + input: null, + }, + }, + new Uint8Array() + ), + { + headers: { + 'content-type': V4_FRAME_CONTENT_TYPE, + 'x-wf-max-events': '10000', + }, + } + ); + + for (const [cursor, eventId] of [ + ['eid:evnt_1', 'evnt_2'], + ['eid:evnt_2', 'evnt_3'], + ['eid:evnt_3', 'evnt_4'], + ] as const) { + agent + .get(origin) + .intercept({ + path: + '/api/v4/runs/wrun_1/events?returnAll=true' + + `&cursor=${encodeURIComponent(cursor)}&remoteRefBehavior=resolve`, + method: 'GET', + }) + .reply( + 200, + encodeFrame( + { + eventId, + runId: 'wrun_1', + eventType: 'run_started', + createdAt: CREATED_AT, + }, + new Uint8Array() + ), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + } + + await expect( + createWorkflowRunStartedEventV4( + { runId: 'wrun_1', specVersion: 5 }, + { token: 'test-token', dispatcher: agent } + ) + ).rejects.toThrow( + 'frame stream ended without the end-of-stream sentinel (1 events read)' + ); + agent.assertNoPendingInterceptors(); + }); + it('requires the event-stream response requested by run_started', async () => { const origin = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 2be3f0c387..0aad8b50dc 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -1476,6 +1476,13 @@ type EventFrameStreamResult = ListEventsV4Result & { const MAX_PARTIAL_STREAM_CONTINUATIONS = 3; +function isAdvancingCursor( + cursor: string | null, + previousCursor: string | null +): cursor is string { + return cursor !== null && cursor !== previousCursor; +} + function partialEventFrameStream( events: Event[], partialError: WorkflowWorldError @@ -1567,14 +1574,18 @@ async function consumeReplayLogResponse( ); } - const suffix = await getWorkflowRunEventsV4( + const suffix = await getWorkflowRunEventsV4WithRecovery( runId, { cursor: page.cursor, remoteRefBehavior: 'resolve' }, - config + config, + 1 ); - if (suffix.events.length > 0 && !suffix.cursor) { + if ( + (suffix.events.length > 0 || suffix.hasMore) && + !isAdvancingCursor(suffix.cursor, page.cursor) + ) { throw new WorkflowWorldError( - 'v4 createEvent: non-empty continuation missing cursor', + 'v4 createEvent: continuation did not advance cursor', { code: 'SCHEMA_VALIDATION' } ); } @@ -1643,15 +1654,15 @@ function paginationToQuery(params: ListEventsV4Params): string { * requests. A forward-progress guard prevents retry loops. Explicitly paginated * requests retain their one-page contract and surface truncation to the caller. */ -export async function getWorkflowRunEventsV4( +async function getWorkflowRunEventsV4WithRecovery( runId: string, - params: ListEventsV4Params = {}, - config?: APIConfig + params: ListEventsV4Params, + config: APIConfig | undefined, + partialContinuations: number ): Promise { const { baseUrl, headers } = await getHttpConfig(config); const events: Event[] = []; let cursor = params.cursor ?? null; - let partialContinuations = 0; let consumed: EventFrameStreamResult; do { @@ -1665,8 +1676,7 @@ export async function getWorkflowRunEventsV4( if (consumed.partialError) { if ( params.limit !== undefined || - !consumed.cursor || - consumed.cursor === cursor || + !isAdvancingCursor(consumed.cursor, cursor) || partialContinuations === MAX_PARTIAL_STREAM_CONTINUATIONS ) { throw consumed.partialError; @@ -1679,6 +1689,14 @@ export async function getWorkflowRunEventsV4( return { events, cursor: consumed.cursor, hasMore: consumed.hasMore }; } +export function getWorkflowRunEventsV4( + runId: string, + params: ListEventsV4Params = {}, + config?: APIConfig +): Promise { + return getWorkflowRunEventsV4WithRecovery(runId, params, config, 0); +} + /** * GET /api/v4/events?correlationId=...&runId=... * From 7b8d0ffa876742cb44642e7e19f2c9cd1e74b337 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:14:33 -0700 Subject: [PATCH 08/12] fix(world-vercel): retain recovered replay cursors --- packages/world-vercel/src/events-v4.test.ts | 36 ++++++++++++--------- packages/world-vercel/src/events-v4.ts | 8 +++-- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index d45ce47666..460b50b49f 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -583,23 +583,26 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { }) .reply( 200, - Buffer.concat([ - encodeFrame( - { - eventId: 'evnt_2', - runId: 'wrun_1', - eventType: 'run_started', - createdAt: CREATED_AT, - }, - new Uint8Array() - ), - encodeFrame( - { _end: 1, next: 'eid:evnt_2', hasMore: false }, - new Uint8Array() - ), - ]), + encodeFrame( + { + eventId: 'evnt_2', + runId: 'wrun_1', + eventType: 'run_started', + createdAt: CREATED_AT, + }, + new Uint8Array() + ), { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } ); + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events?returnAll=true&cursor=eid%3Aevnt_2', + method: 'GET', + }) + .reply(200, encodeFrame({ _end: 1, hasMore: false }, new Uint8Array()), { + headers: { 'content-type': V4_FRAME_CONTENT_TYPE }, + }); const result = await getWorkflowRunEventsV4( 'wrun_1', @@ -1134,6 +1137,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { it.each([ ['continues a truncated run_started replay', 'eid:evnt_2', true], ['rejects a continuation without its trailing cursor', undefined, false], + ['rejects an empty continuation cursor', '', false], ['rejects a non-advancing continuation cursor', 'eid:evnt_1', false], ])('%s', async (_name, suffixCursor, succeeds) => { const origin = @@ -1192,7 +1196,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { encodeFrame( { _end: 1, - ...(suffixCursor ? { next: suffixCursor } : {}), + ...(suffixCursor !== undefined ? { next: suffixCursor } : {}), hasMore: false, }, new Uint8Array() diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 0aad8b50dc..0587deac6b 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -1480,7 +1480,7 @@ function isAdvancingCursor( cursor: string | null, previousCursor: string | null ): cursor is string { - return cursor !== null && cursor !== previousCursor; + return cursor !== null && cursor.length > 0 && cursor !== previousCursor; } function partialEventFrameStream( @@ -1686,7 +1686,11 @@ async function getWorkflowRunEventsV4WithRecovery( } } while (consumed.partialError); - return { events, cursor: consumed.cursor, hasMore: consumed.hasMore }; + return { + events, + cursor: consumed.cursor ?? cursor, + hasMore: consumed.hasMore, + }; } export function getWorkflowRunEventsV4( From c3aae5c002f7aac817271301339298c7116b35da Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:22:43 -0700 Subject: [PATCH 09/12] fix(world-vercel): validate each recovery cursor --- packages/world-vercel/src/events-v4.test.ts | 2 +- packages/world-vercel/src/events-v4.ts | 32 ++++++++++++--------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 460b50b49f..d938955c1e 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -1219,7 +1219,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { } else { await expect(request).rejects.toMatchObject({ code: 'SCHEMA_VALIDATION', - message: 'v4 createEvent: continuation did not advance cursor', + message: 'v4 listEvents: response did not advance cursor', }); } agent.assertNoPendingInterceptors(); diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 0587deac6b..aa5dd0b728 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -1483,6 +1483,16 @@ function isAdvancingCursor( return cursor !== null && cursor.length > 0 && cursor !== previousCursor; } +function hasRequiredCursorProgress( + page: ListEventsV4Result, + previousCursor: string | null +): boolean { + return ( + (page.events.length === 0 && !page.hasMore) || + isAdvancingCursor(page.cursor, previousCursor) + ); +} + function partialEventFrameStream( events: Event[], partialError: WorkflowWorldError @@ -1580,15 +1590,6 @@ async function consumeReplayLogResponse( config, 1 ); - if ( - (suffix.events.length > 0 || suffix.hasMore) && - !isAdvancingCursor(suffix.cursor, page.cursor) - ) { - throw new WorkflowWorldError( - 'v4 createEvent: continuation did not advance cursor', - { code: 'SCHEMA_VALIDATION' } - ); - } return { events: [...page.events, ...suffix.events], cursor: suffix.cursor ?? page.cursor, @@ -1670,9 +1671,6 @@ async function getWorkflowRunEventsV4WithRecovery( `${baseUrl}/v4/runs/${encodeURIComponent(runId)}/events` + paginationToQuery({ ...params, cursor: cursor ?? undefined }); consumed = await consumeListFrameStream(url, headers, config, 'listEvents'); - for (const event of consumed.events) { - events.push(event); - } if (consumed.partialError) { if ( params.limit !== undefined || @@ -1683,12 +1681,20 @@ async function getWorkflowRunEventsV4WithRecovery( } partialContinuations++; cursor = consumed.cursor; + } else if (!hasRequiredCursorProgress(consumed, cursor)) { + throw new WorkflowWorldError( + 'v4 listEvents: response did not advance cursor', + { code: 'SCHEMA_VALIDATION' } + ); + } + for (const event of consumed.events) { + events.push(event); } } while (consumed.partialError); return { events, - cursor: consumed.cursor ?? cursor, + cursor: consumed.cursor || (events.length > 0 ? cursor : null), hasMore: consumed.hasMore, }; } From 83e1ac764518a07744090407991df4d4b6f26e07 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:24:18 -0700 Subject: [PATCH 10/12] test(world-vercel): include list response cursors --- packages/world-vercel/src/events.test.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index 2f953515a8..4a36c778a5 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -1228,7 +1228,10 @@ describe('getWorkflowRunEvents remoteRefBehavior mapping', () => { }, body ), - encodeFrame({ _end: 1, hasMore: false }, new Uint8Array(0)), + encodeFrame( + { _end: 1, next: 'eid:evnt_1', hasMore: false }, + new Uint8Array(0) + ), ]); } @@ -1375,7 +1378,10 @@ describe('getWorkflowRunEvents legacy structured-error compatibility', () => { }, body ), - encodeFrame({ _end: 1, hasMore: false }, new Uint8Array(0)), + encodeFrame( + { _end: 1, next: 'eid:evnt_1', hasMore: false }, + new Uint8Array(0) + ), ]); } From b3095a525bac89db2db79c0cc5fcb03139869369 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:35:18 -0700 Subject: [PATCH 11/12] refactor(world-vercel): simplify replay recovery --- packages/world-vercel/src/events-v4.test.ts | 1 - packages/world-vercel/src/events-v4.ts | 65 +++++++-------------- 2 files changed, 21 insertions(+), 45 deletions(-) diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index d938955c1e..5532638a14 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -629,7 +629,6 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { [undefined, 'evnt_1'], ['eid:evnt_1', 'evnt_2'], ['eid:evnt_2', 'evnt_3'], - ['eid:evnt_3', 'evnt_4'], ] as const) { agent .get(origin) diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index aa5dd0b728..8b38bb0851 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -1474,24 +1474,7 @@ type EventFrameStreamResult = ListEventsV4Result & { partialError?: WorkflowWorldError; }; -const MAX_PARTIAL_STREAM_CONTINUATIONS = 3; - -function isAdvancingCursor( - cursor: string | null, - previousCursor: string | null -): cursor is string { - return cursor !== null && cursor.length > 0 && cursor !== previousCursor; -} - -function hasRequiredCursorProgress( - page: ListEventsV4Result, - previousCursor: string | null -): boolean { - return ( - (page.events.length === 0 && !page.hasMore) || - isAdvancingCursor(page.cursor, previousCursor) - ); -} +const MAX_PARTIAL_STREAM_RETRIES = 2; function partialEventFrameStream( events: Event[], @@ -1584,11 +1567,10 @@ async function consumeReplayLogResponse( ); } - const suffix = await getWorkflowRunEventsV4WithRecovery( + const suffix = await getWorkflowRunEventsV4( runId, { cursor: page.cursor, remoteRefBehavior: 'resolve' }, - config, - 1 + config ); return { events: [...page.events, ...suffix.events], @@ -1651,19 +1633,19 @@ function paginationToQuery(params: ListEventsV4Params): string { * * Eagerly drains the stream into memory to match the existing * `getWorkflowRunEvents` contract. A truncated full response resumes after its - * last validated event until the sentinel arrives, for up to three continuation - * requests. A forward-progress guard prevents retry loops. Explicitly paginated - * requests retain their one-page contract and surface truncation to the caller. + * last validated event until the sentinel arrives, for up to two retries. A + * forward-progress guard prevents retry loops. Explicitly paginated requests + * retain their one-page contract and surface truncation to the caller. */ -async function getWorkflowRunEventsV4WithRecovery( +export async function getWorkflowRunEventsV4( runId: string, - params: ListEventsV4Params, - config: APIConfig | undefined, - partialContinuations: number + params: ListEventsV4Params = {}, + config?: APIConfig ): Promise { const { baseUrl, headers } = await getHttpConfig(config); const events: Event[] = []; let cursor = params.cursor ?? null; + let partialStreamRetries = 0; let consumed: EventFrameStreamResult; do { @@ -1671,42 +1653,37 @@ async function getWorkflowRunEventsV4WithRecovery( `${baseUrl}/v4/runs/${encodeURIComponent(runId)}/events` + paginationToQuery({ ...params, cursor: cursor ?? undefined }); consumed = await consumeListFrameStream(url, headers, config, 'listEvents'); + const cursorAdvanced = !!consumed.cursor && consumed.cursor !== cursor; if (consumed.partialError) { if ( params.limit !== undefined || - !isAdvancingCursor(consumed.cursor, cursor) || - partialContinuations === MAX_PARTIAL_STREAM_CONTINUATIONS + !cursorAdvanced || + partialStreamRetries === MAX_PARTIAL_STREAM_RETRIES ) { throw consumed.partialError; } - partialContinuations++; + assert(consumed.cursor); + partialStreamRetries++; cursor = consumed.cursor; - } else if (!hasRequiredCursorProgress(consumed, cursor)) { + } else if ( + !cursorAdvanced && + (consumed.events.length > 0 || consumed.hasMore) + ) { throw new WorkflowWorldError( 'v4 listEvents: response did not advance cursor', { code: 'SCHEMA_VALIDATION' } ); } - for (const event of consumed.events) { - events.push(event); - } + events.push(...consumed.events); } while (consumed.partialError); return { events, - cursor: consumed.cursor || (events.length > 0 ? cursor : null), + cursor: consumed.cursor || (partialStreamRetries > 0 ? cursor : null), hasMore: consumed.hasMore, }; } -export function getWorkflowRunEventsV4( - runId: string, - params: ListEventsV4Params = {}, - config?: APIConfig -): Promise { - return getWorkflowRunEventsV4WithRecovery(runId, params, config, 0); -} - /** * GET /api/v4/events?correlationId=...&runId=... * From e9d0c2270d77e872ec7baca1dec6fbc03e6c4cc4 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:37:36 -0700 Subject: [PATCH 12/12] Update packages/world-vercel/src/events-v4.ts Co-authored-by: vercel[bot] <35613825+vercel[bot]@users.noreply.github.com> Signed-off-by: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> --- packages/world-vercel/src/events-v4.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 8b38bb0851..12bc8adea5 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -1674,7 +1674,9 @@ export async function getWorkflowRunEventsV4( { code: 'SCHEMA_VALIDATION' } ); } - events.push(...consumed.events); + for (const event of consumed.events) { + events.push(event); + } } while (consumed.partialError); return {