{
- if (!hasPromptContent(data.prompt)) throw new Error('Prompt is required')
- if (!data.model) throw new Error('Model is required')
- return data
- })
+ .inputValidator(
+ (data: {
+ prompt: MediaPrompt
+ model: string
+ previousImage?: { url?: string; b64Json?: string }
+ }) => {
+ if (!hasPromptContent(data.prompt)) throw new Error('Prompt is required')
+ if (!data.model) throw new Error('Model is required')
+ return data
+ },
+ )
.handler(async ({ data }) => {
+ const previousImage =
+ data.previousImage?.url != null
+ ? { previousImage: { url: data.previousImage.url } }
+ : data.previousImage?.b64Json != null
+ ? { previousImage: { b64Json: data.previousImage.b64Json } }
+ : {}
// NOTE: Use string literals when instantiating adapters to preserve type safety
// The Fal adapater also accepts any string for very latest models which is why new models appear to accept any paramater
// Pass size information in modelOptions for the Fal adapter instead of size to be sure you are using the correct resolution
@@ -165,6 +177,7 @@ export const generateImageFn = createServerFn({ method: 'POST' })
prompt: asImagePrompt(data.prompt),
numberOfImages: 1,
size: '16:9',
+ ...previousImage,
})
}
case 'grok-imagine-image-2.0': {
@@ -176,6 +189,7 @@ export const generateImageFn = createServerFn({ method: 'POST' })
numberOfImages: 1,
size: '16:9',
modelOptions: { quality: 'medium' },
+ ...previousImage,
})
}
case 'grok-imagine-image-quality': {
@@ -184,6 +198,7 @@ export const generateImageFn = createServerFn({ method: 'POST' })
prompt: asImagePrompt(data.prompt),
numberOfImages: 1,
size: '16:9',
+ ...previousImage,
})
}
case 'fal-ai/flux-2/klein/9b': {
@@ -213,6 +228,7 @@ export const generateImageFn = createServerFn({ method: 'POST' })
prompt: asImagePrompt(data.prompt),
numberOfImages: 1,
size: '16:9_4K',
+ ...previousImage,
})
}
case 'gemini-3-pro-image': {
@@ -221,6 +237,7 @@ export const generateImageFn = createServerFn({ method: 'POST' })
prompt: asImagePrompt(data.prompt),
numberOfImages: 1,
size: '16:9_4K',
+ ...previousImage,
})
}
case 'imagen-4.0-ultra-generate-001': {
@@ -257,6 +274,7 @@ export const generateImageFn = createServerFn({ method: 'POST' })
prompt: asImagePrompt(data.prompt),
numberOfImages: 1,
size: '2K',
+ ...previousImage,
})
}
default:
@@ -269,10 +287,11 @@ interface VideoRequest {
prompt: MediaPrompt
model: string
/**
- * Gemini Omni Flash conversational editing: the jobId (interaction id)
- * of a prior Omni generation to refine. Ignored by other models.
+ * Prior generation's job id to edit instead of generating from scratch.
+ * Omni maps it onto `previous_interaction_id`; Grok/fal resolve the clip
+ * via `getVideoUrl`.
*/
- previousInteractionId?: string
+ previousJobId?: string
/**
* Gemini Omni Flash generation controls (ignored by other models):
* clip duration in seconds (3-10, fractional OK, default 10), output
@@ -329,8 +348,9 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable
{
pollingInterval: VIDEO_POLL_INTERVAL_MS,
adapter: falVideo('xai/grok-imagine-video/text-to-video'),
prompt: asTextPrompt(data.prompt),
- size: '16:9_720p',
- duration: 5,
+ ...(data.previousJobId
+ ? { previousJobId: data.previousJobId }
+ : { size: '16:9_720p' as const, duration: 5 }),
})
}
case 'grok-imagine-video': {
@@ -338,13 +358,15 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable {
// grok-imagine-video (v1.0) supports text-to-video; durations are
// 1-15 integer seconds. Completed jobs report usage.billed
// ({ quantity, unit: 'seconds' }) and usage.cost (exact USD).
+ // Follow-up edits inherit duration/aspect from the source clip.
return generateVideo({
stream: true,
pollingInterval: VIDEO_POLL_INTERVAL_MS,
adapter: grokVideo('grok-imagine-video'),
prompt: asTextPrompt(data.prompt),
- size: '16:9_720p',
- duration: 5,
+ ...(data.previousJobId
+ ? { previousJobId: data.previousJobId }
+ : { size: '16:9_720p' as const, duration: 5 }),
})
}
case 'grok-imagine-video-1.5': {
@@ -416,8 +438,9 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable {
pollingInterval: VIDEO_POLL_INTERVAL_MS,
adapter: falVideo('xai/grok-imagine-video/image-to-video'),
prompt: asImageToVideoPrompt(data.prompt),
- size: '16:9_720p',
- duration: 5,
+ ...(data.previousJobId
+ ? { previousJobId: data.previousJobId }
+ : { size: '16:9_720p' as const, duration: 5 }),
})
}
case 'grok-imagine-video-1.5/image-to-video': {
@@ -448,15 +471,14 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable {
// serves both UI entries; it accepts text, image, AND video prompt
// parts (sent as interaction content blocks: images, then videos,
// then text). Clips are 3–10s at 720p (default 10s when `duration`
- // is omitted); `size` is the output aspect ratio. Passing
- // `previous_interaction_id` chains a prompt onto a prior generation
- // for conversational editing.
+ // is omitted); `size` is the output aspect ratio. `previousJobId`
+ // chains a prompt onto a prior generation (its interaction id).
case 'gemini-omni-flash-preview':
case 'gemini-omni-flash-preview/image-to-video': {
const prompt = asOmniPrompt(data.prompt)
if (
data.model.endsWith('/image-to-video') &&
- !data.previousInteractionId &&
+ !data.previousJobId &&
(typeof prompt === 'string' ||
!prompt.some((part) => part.type === 'image'))
) {
@@ -470,15 +492,11 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable {
prompt,
size: aspectRatio ?? '16:9',
...(duration !== undefined ? { duration } : {}),
- ...(data.previousInteractionId || task
+ ...(data.previousJobId ? { previousJobId: data.previousJobId } : {}),
+ ...(task
? {
modelOptions: {
- ...(data.previousInteractionId
- ? { previous_interaction_id: data.previousInteractionId }
- : {}),
- ...(task
- ? { generation_config: { video_config: { task } } }
- : {}),
+ generation_config: { video_config: { task } },
},
}
: {}),
diff --git a/packages/ai-client/src/generation-types.ts b/packages/ai-client/src/generation-types.ts
index e7d9cb7d4e..f4b6ece47b 100644
--- a/packages/ai-client/src/generation-types.ts
+++ b/packages/ai-client/src/generation-types.ts
@@ -685,6 +685,11 @@ export interface ImageGenerateInput {
numberOfImages?: number
/** Image size in WIDTHxHEIGHT format (e.g., "1024x1024") */
size?: string
+ /**
+ * A previously generated image to edit — forwarded to generateImage's
+ * `previousImage`, which prepends it to the prompt as an image input.
+ */
+ previousImage?: { url?: string; b64Json?: string }
/** Model-specific options */
modelOptions?: Record
}
@@ -763,6 +768,12 @@ export interface VideoGenerateInput {
size?: string
/** Video duration in seconds */
duration?: number
+ /**
+ * Prior generation's job id to edit instead of generating from scratch.
+ * Job-kind adapters reference it server-side; media-kind adapters resolve
+ * the finished clip via `getVideoUrl`.
+ */
+ previousJobId?: string
/** Model-specific options */
modelOptions?: Record
}
diff --git a/packages/ai-fal/src/adapters/video.ts b/packages/ai-fal/src/adapters/video.ts
index dab045e6c5..01bf008ee5 100644
--- a/packages/ai-fal/src/adapters/video.ts
+++ b/packages/ai-fal/src/adapters/video.ts
@@ -1,5 +1,5 @@
import { fal } from '@fal-ai/client'
-import { resolveMediaPrompt } from '@tanstack/ai'
+import { generatedVideoUrlToVideoPart, resolveMediaPrompt } from '@tanstack/ai'
import { BaseVideoAdapter, snapToDurationOption } from '@tanstack/ai/adapters'
import {
configureFalClient,
@@ -11,10 +11,12 @@ import {
mapVideoSizeToFalFormat,
} from '../video/video-provider-options'
import { mapImageInputsToFalVideoFields } from '../image/image-inputs'
+import { FAL_VIDEO_EDIT_BY_SOURCE } from '../model-meta'
import type { DurationOptions } from '@tanstack/ai/adapters'
import type {
AudioPart,
MediaInputMetadata,
+ VideoEditKind,
VideoGenerationOptions,
VideoJobResult,
VideoPart,
@@ -26,18 +28,35 @@ import type {
FalModelInput,
FalModelVideoDuration,
FalModelVideoSize,
+ FalVideoEditKindFor,
FalVideoPromptModalitiesFor,
FalVideoProviderOptions,
} from '../model-meta'
import type { FalClientConfig } from '../utils/client'
+/**
+ * Endpoints whose source-video field deviates from the `video_url` default.
+ * Seedance 2.0's reference-to-video endpoints take reference clips as a
+ * `video_urls` list (referenced from the prompt as `@Video1`, `@Video2`, …)
+ * and have no singular `video_url` field.
+ */
+const FAL_VIDEO_SOURCE_FIELD_OVERRIDES: Record<
+ string,
+ 'video_url' | 'video_urls'
+> = {
+ 'bytedance/seedance-2.0/reference-to-video': 'video_urls',
+ 'bytedance/seedance-2.0/fast/reference-to-video': 'video_urls',
+}
+
/**
* Map video conditioning inputs onto fal field names.
* Video-to-video endpoints on fal almost universally use `video_url`; the
- * occasional model takes `video_urls` (rare). Mirror the image-input logic
+ * occasional model takes `video_urls` (list-only endpoints live in
+ * `FAL_VIDEO_SOURCE_FIELD_OVERRIDES`). Mirror the image-input logic
* positionally with a `reference` role escape hatch via `reference_video_urls`.
*/
function mapVideoInputsToFalFields(
+ model: string,
videoInputs?: ReadonlyArray>,
): Record {
if (!videoInputs || videoInputs.length === 0) return {}
@@ -56,7 +75,10 @@ function mapVideoInputsToFalFields(
}
const out: Record = {}
if (references.length > 0) out.reference_video_urls = references
- if (sources.length === 1) {
+ const sourceField = FAL_VIDEO_SOURCE_FIELD_OVERRIDES[model]
+ if (sourceField === 'video_urls') {
+ if (sources.length > 0) out.video_urls = sources
+ } else if (sources.length === 1) {
out.video_url = sources[0]
} else if (sources.length > 1) {
out.video_urls = sources
@@ -138,7 +160,8 @@ export class FalVideoAdapter extends BaseVideoAdapter<
Record>,
Record>,
Record>,
- Record>
+ Record>,
+ Record>
> {
override readonly kind = 'video' as const
readonly name = 'fal' as const
@@ -148,6 +171,19 @@ export class FalVideoAdapter extends BaseVideoAdapter<
configureFalClient(config)
}
+ /**
+ * fal endpoints consume a previously generated video by URL. Callers pass
+ * `previousJobId`; the adapter resolves the finished clip via
+ * `getVideoUrl`. Generate endpoints with a known edit sibling (see
+ * `FAL_VIDEO_EDIT_BY_SOURCE`) resolve on the generate model, then submit
+ * to the edit endpoint. The endpoint set is open-world, so this always
+ * reports `'media'` at runtime; per-endpoint support is narrowed at the
+ * type level via `FalVideoEditKindFor`.
+ */
+ override supportedEditKind(): VideoEditKind {
+ return 'media'
+ }
+
async createVideoJob(
options: VideoGenerationOptions<
FalVideoProviderOptions,
@@ -155,7 +191,7 @@ export class FalVideoAdapter extends BaseVideoAdapter<
FalModelVideoDuration
>,
): Promise {
- const { size, duration, modelOptions, logger } = options
+ const { size, duration, modelOptions, logger, previousJobId } = options
logger.request(`activity=generateVideo provider=fal model=${this.model}`, {
provider: 'fal',
@@ -164,12 +200,29 @@ export class FalVideoAdapter extends BaseVideoAdapter<
try {
const resolved = resolveMediaPrompt(options.prompt)
+ // Resolve the prior clip from previousJobId on this (generate) model, then
+ // submit to the edit sibling when one is mapped.
+ const editUrl = previousJobId
+ ? await this.resolvePreviousJobUrl(previousJobId)
+ : undefined
+ const submitModel =
+ previousJobId && this.model in FAL_VIDEO_EDIT_BY_SOURCE
+ ? FAL_VIDEO_EDIT_BY_SOURCE[
+ this.model as keyof typeof FAL_VIDEO_EDIT_BY_SOURCE
+ ]
+ : this.model
+ // The edited video rides the endpoint's regular video input: prepend
+ // it as a video part so the field mapping below routes it (video_url,
+ // or the endpoint's list field), ahead of any explicit video parts.
+ const videos = editUrl
+ ? [generatedVideoUrlToVideoPart(editUrl), ...resolved.videos]
+ : resolved.videos
const sizeParams = mapVideoSizeToFalFormat(size)
const inputImageFields = mapImageInputsToFalVideoFields(
- this.model,
+ submitModel,
resolved.images,
)
- const videoFields = mapVideoInputsToFalFields(resolved.videos)
+ const videoFields = mapVideoInputsToFalFields(submitModel, videos)
const audioFields = mapAudioInputsToFalFields(resolved.audios)
const input = {
@@ -188,14 +241,14 @@ export class FalVideoAdapter extends BaseVideoAdapter<
// Submit to queue and get request ID. Request-specific abortSignal only —
// never via fal.config() (global; would cancel concurrent jobs).
- const { request_id } = await fal.queue.submit(this.model, {
+ const { request_id } = await fal.queue.submit(submitModel, {
input,
...(options.abortSignal ? { abortSignal: options.abortSignal } : {}),
})
return {
jobId: request_id,
- model: this.model,
+ model: submitModel,
}
} catch (error) {
logger.errors('fal.createVideoJob fatal', {
diff --git a/packages/ai-fal/src/index.ts b/packages/ai-fal/src/index.ts
index 7351ef11db..8c27966c56 100644
--- a/packages/ai-fal/src/index.ts
+++ b/packages/ai-fal/src/index.ts
@@ -47,6 +47,9 @@ export {
type FalModelImageSize,
type FalModelVideoSize,
type FalModelVideoDuration,
+ type FalVideoEditKindFor,
+ type FalVideoEditSourceModel,
+ FAL_VIDEO_EDIT_BY_SOURCE,
} from './model-meta'
// ============================================================================
// Utils
diff --git a/packages/ai-fal/src/model-meta.ts b/packages/ai-fal/src/model-meta.ts
index 8c6c850632..8075b3a1ce 100644
--- a/packages/ai-fal/src/model-meta.ts
+++ b/packages/ai-fal/src/model-meta.ts
@@ -4,7 +4,7 @@
* These types give you full autocomplete and type safety for any model.
*/
import type { EndpointTypeMap } from '@fal-ai/client/endpoints'
-import type { MediaPromptModality } from '@tanstack/ai'
+import type { MediaPromptModality, VideoEditKind } from '@tanstack/ai'
import type { FalImageFieldName } from './image/generated/image-field-overrides'
export type { EndpointTypeMap } from '@fal-ai/client/endpoints'
@@ -210,6 +210,41 @@ export type FalVideoPromptModalitiesFor =
>
: ReadonlyArray
+/**
+ * Generate endpoints that edit via a dedicated video-to-video sibling.
+ * When `previousJobId` is set on the generate model, the fal adapter resolves
+ * the prior clip via `getVideoUrl(previousJobId)` on the generate endpoint,
+ * then submits to the mapped edit endpoint.
+ */
+export const FAL_VIDEO_EDIT_BY_SOURCE = {
+ 'xai/grok-imagine-video/text-to-video': 'xai/grok-imagine-video/edit-video',
+ 'xai/grok-imagine-video/image-to-video': 'xai/grok-imagine-video/edit-video',
+} as const
+
+export type FalVideoEditSourceModel = keyof typeof FAL_VIDEO_EDIT_BY_SOURCE
+
+/**
+ * Follow-up edit kind for a fal video endpoint. An endpoint supports
+ * `previousJobId` when:
+ * - it declares a video-conditioning input (`video_url` / `video_urls` /
+ * `reference_video_urls`), or
+ * - it is a generate endpoint with a known edit sibling in
+ * `FAL_VIDEO_EDIT_BY_SOURCE` (e.g. Grok text/image-to-video → edit-video).
+ * Endpoints unknown to the installed SDK are unconstrained and gated at the
+ * API instead.
+ */
+export type FalVideoEditKindFor =
+ TModel extends FalVideoEditSourceModel
+ ? 'media'
+ : TModel extends keyof EndpointTypeMap
+ ? Extract<
+ keyof FalModelInput,
+ 'video_url' | 'video_urls' | 'reference_video_urls'
+ > extends never
+ ? undefined
+ : 'media'
+ : VideoEditKind
+
/**
* Provider options for video generation, excluding fields TanStack AI handles.
* Use this for the `modelOptions` parameter in video generation.
diff --git a/packages/ai-fal/tests/video-adapter.test.ts b/packages/ai-fal/tests/video-adapter.test.ts
index f7bc66a410..ae96f64b69 100644
--- a/packages/ai-fal/tests/video-adapter.test.ts
+++ b/packages/ai-fal/tests/video-adapter.test.ts
@@ -292,6 +292,103 @@ describe('Fal Video Adapter', () => {
})
})
+ describe('previousJobId (video-to-video endpoints)', () => {
+ it('reports media-kind edit support', () => {
+ const adapter = createAdapter()
+ expect(adapter.supportedEditKind()).toBe('media')
+ })
+
+ it('resolves previousJobId and routes onto video_url', async () => {
+ mockQueueResult.mockResolvedValueOnce({
+ data: { video: { url: 'https://example.com/source.mp4' } },
+ })
+ mockQueueSubmit.mockResolvedValueOnce({ request_id: 'job-edit' })
+
+ const adapter = falVideo('xai/grok-imagine-video/edit-video', {
+ apiKey: 'test-key',
+ })
+
+ await adapter.createVideoJob({
+ model: 'xai/grok-imagine-video/edit-video',
+ prompt: 'Give the rider a red scarf',
+ previousJobId: 'prior-job',
+ logger: testLogger,
+ })
+
+ expect(mockQueueResult).toHaveBeenCalledWith(
+ 'xai/grok-imagine-video/edit-video',
+ { requestId: 'prior-job' },
+ )
+ const [, options] = mockQueueSubmit.mock.calls[0]!
+ expect(options.input).toEqual({
+ prompt: 'Give the rider a red scarf',
+ video_url: 'https://example.com/source.mp4',
+ })
+ })
+
+ it('routes previousJobId onto the video_urls list for Seedance reference-to-video', async () => {
+ mockQueueResult.mockResolvedValueOnce({
+ data: { video: { url: 'https://example.com/source.mp4' } },
+ })
+ mockQueueSubmit.mockResolvedValueOnce({ request_id: 'job-seedance' })
+
+ const adapter = falVideo('bytedance/seedance-2.0/reference-to-video', {
+ apiKey: 'test-key',
+ })
+
+ await adapter.createVideoJob({
+ model: 'bytedance/seedance-2.0/reference-to-video',
+ prompt: 'Replace the background with a beach (@Video1)',
+ previousJobId: 'prior-job',
+ logger: testLogger,
+ })
+
+ const [, options] = mockQueueSubmit.mock.calls[0]!
+ expect(options.input).toEqual({
+ prompt: 'Replace the background with a beach (@Video1)',
+ video_urls: ['https://example.com/source.mp4'],
+ })
+ })
+
+ it('routes generate-model edits onto the mapped edit-video endpoint', async () => {
+ mockQueueResult.mockResolvedValueOnce({
+ data: { video: { url: 'https://example.com/resolved.mp4' } },
+ })
+ mockQueueSubmit.mockResolvedValueOnce({ request_id: 'job-edit' })
+
+ const adapter = falVideo('xai/grok-imagine-video/text-to-video', {
+ apiKey: 'test-key',
+ })
+
+ const result = await adapter.createVideoJob({
+ model: 'xai/grok-imagine-video/text-to-video',
+ prompt: 'Make it rain',
+ previousJobId: 'prior-job',
+ logger: testLogger,
+ })
+
+ // Resolve against the generate endpoint that produced the job.
+ expect(mockQueueResult).toHaveBeenCalledWith(
+ 'xai/grok-imagine-video/text-to-video',
+ { requestId: 'prior-job' },
+ )
+ // Submit to the edit sibling.
+ expect(mockQueueSubmit).toHaveBeenCalledWith(
+ 'xai/grok-imagine-video/edit-video',
+ expect.objectContaining({
+ input: {
+ prompt: 'Make it rain',
+ video_url: 'https://example.com/resolved.mp4',
+ },
+ }),
+ )
+ expect(result).toEqual({
+ jobId: 'job-edit',
+ model: 'xai/grok-imagine-video/edit-video',
+ })
+ })
+ })
+
describe('availableDurations / snapDuration', () => {
it('returns discrete keyword durations for Veo3', () => {
const adapter = falVideo('fal-ai/veo3', { apiKey: 'test' })
diff --git a/packages/ai-gemini/src/adapters/video.ts b/packages/ai-gemini/src/adapters/video.ts
index 4fc2897fd5..994f5cefd3 100644
--- a/packages/ai-gemini/src/adapters/video.ts
+++ b/packages/ai-gemini/src/adapters/video.ts
@@ -15,6 +15,7 @@ import type {
ImagePart,
MediaInputMetadata,
TokenUsage,
+ VideoEditKind,
VideoGenerationOptions,
VideoJobResult,
VideoPart,
@@ -32,6 +33,7 @@ import type {
GeminiOmniVideoProviderOptions,
GeminiVideoModel,
GeminiVideoModelDurationByName,
+ GeminiVideoModelEditByName,
GeminiVideoModelInputModalitiesByName,
GeminiVideoModelProviderOptionsByName,
GeminiVideoModelSizeByName,
@@ -235,8 +237,7 @@ function interactionUsageToTokenUsage(
* Files API URI when the server delivers by reference). Image and video
* prompt parts are sent as interaction content blocks, grouped as images,
* then videos, then the text prompt (interleaving is not preserved); pass
- * `modelOptions.previous_interaction_id` to conversationally edit a prior
- * Omni generation.
+ * `previousJobId` to conversationally edit a prior Omni generation.
*
* @experimental Video generation is an experimental feature and may change.
*/
@@ -248,7 +249,8 @@ export class GeminiVideoAdapter<
GeminiVideoModelProviderOptionsByName,
GeminiVideoModelSizeByName,
GeminiVideoModelInputModalitiesByName,
- GeminiVideoModelDurationByName
+ GeminiVideoModelDurationByName,
+ GeminiVideoModelEditByName
> {
readonly name = 'gemini' as const
@@ -261,6 +263,15 @@ export class GeminiVideoAdapter<
this.allowUrlFetch = config.allowUrlFetch ?? false
}
+ /**
+ * Omni Flash edits a prior generation by chaining its interaction id
+ * (`previous_interaction_id`); Veo models cannot edit previous
+ * generations.
+ */
+ override supportedEditKind(): VideoEditKind | undefined {
+ return isInteractionsVideoModel(this.model) ? 'job' : undefined
+ }
+
async createVideoJob(
options: VideoGenerationOptions<
GeminiVideoModelProviderOptionsByName[TModel],
@@ -343,7 +354,7 @@ export class GeminiVideoAdapter<
GeminiVideoModelDurationByName[TModel]
>,
): Promise {
- const { prompt, size, duration, logger } = options
+ const { prompt, size, duration, logger, previousJobId } = options
const modelOptions = options.modelOptions as
| GeminiOmniVideoProviderOptions
| undefined
@@ -401,6 +412,9 @@ export class GeminiVideoAdapter<
const interaction = await this.client.interactions.create({
...modelOptions,
+ // previousJobId chains a follow-up edit onto the prior interaction;
+ // map it onto the wire's previous_interaction_id.
+ ...(previousJobId && { previous_interaction_id: previousJobId }),
model: this.model,
input: [{ type: 'user_input', content }],
response_modalities: ['video'],
diff --git a/packages/ai-gemini/src/index.ts b/packages/ai-gemini/src/index.ts
index 7e1e426b11..0dacac6ebb 100644
--- a/packages/ai-gemini/src/index.ts
+++ b/packages/ai-gemini/src/index.ts
@@ -114,6 +114,7 @@ export type {
GeminiOmniVideoProviderOptions,
GeminiVideoModel,
GeminiVideoModelDurationByName,
+ GeminiVideoModelEditByName,
GeminiVideoModelInputModalitiesByName,
GeminiVideoModelProviderOptionsByName,
GeminiVideoModelSizeByName,
diff --git a/packages/ai-gemini/src/video/video-provider-options.ts b/packages/ai-gemini/src/video/video-provider-options.ts
index 487e5d2df5..38571e09ed 100644
--- a/packages/ai-gemini/src/video/video-provider-options.ts
+++ b/packages/ai-gemini/src/video/video-provider-options.ts
@@ -89,12 +89,14 @@ export type GeminiVideoProviderOptions = Omit<
* - `tools` / `response_mime_type` — not applicable to video generation
*
* Notable passthroughs:
- * - `previous_interaction_id` — conversational video editing: chain a new
- * prompt onto a prior Omni interaction to refine its video
* - `generation_config.video_config.task` — pin the task mode
* (`'text_to_video' | 'image_to_video' | 'reference_to_video' | 'edit'`)
* instead of letting the model infer it
*
+ * Conversational video editing uses the top-level `previousJobId` option
+ * (mapped onto the wire's `previous_interaction_id` by the adapter), not
+ * a modelOptions field.
+ *
* @experimental Omni video generation is an experimental feature and may change.
*/
export type GeminiOmniVideoProviderOptions = Omit<
@@ -107,6 +109,7 @@ export type GeminiOmniVideoProviderOptions = Omit<
| 'response_format'
| 'response_mime_type'
| 'tools'
+ | 'previous_interaction_id'
>
/**
@@ -143,6 +146,20 @@ export type GeminiVideoModelInputModalitiesByName = {
: readonly ['image']
}
+/**
+ * Per-model follow-up edit support. Omni Flash conversationally edits a
+ * prior generation by chaining its interaction id (the job id) via the
+ * Interactions API's `previous_interaction_id`, so `previousJobId` is the
+ * prior generation's job id. Veo models cannot edit previous generations.
+ *
+ * @experimental Video generation is an experimental feature and may change.
+ */
+export type GeminiVideoModelEditByName = {
+ [TModel in GeminiVideoModel]: TModel extends GeminiInteractionsVideoModel
+ ? 'job'
+ : undefined
+}
+
/**
* Per-model duration unions (seconds, as numbers — Veo's
* `parameters.durationSeconds` field is numeric; Omni Flash accepts a
diff --git a/packages/ai-gemini/tests/video-adapter.test.ts b/packages/ai-gemini/tests/video-adapter.test.ts
index 627ad991b5..9d4c3e7e20 100644
--- a/packages/ai-gemini/tests/video-adapter.test.ts
+++ b/packages/ai-gemini/tests/video-adapter.test.ts
@@ -720,7 +720,7 @@ describe('Gemini Omni Flash Video Adapter (Interactions API)', () => {
await adapter.createVideoJob({
model: 'gemini-omni-flash-preview',
prompt: 'make the violin invisible',
- modelOptions: { previous_interaction_id: 'v1_prior-turn' },
+ previousJobId: 'v1_prior-turn',
logger: testLogger,
})
@@ -858,6 +858,41 @@ describe('Gemini Omni Flash Video Adapter (Interactions API)', () => {
})
})
+ describe('previousJobId (previous_interaction_id chaining)', () => {
+ it('reports job-kind edit support for Omni and none for Veo', () => {
+ expect(
+ createGeminiVideo(
+ 'gemini-omni-flash-preview',
+ 'test-key',
+ ).supportedEditKind(),
+ ).toBe('job')
+ expect(
+ createGeminiVideo(
+ 'veo-3.1-generate-preview',
+ 'test-key',
+ ).supportedEditKind(),
+ ).toBeUndefined()
+ })
+
+ it('maps previousJobId onto previous_interaction_id', async () => {
+ const stub = createInteractionsClientStub()
+ const adapter = new StubbedGeminiOmniVideoAdapter(stub)
+
+ await adapter.createVideoJob({
+ model: 'gemini-omni-flash-preview',
+ prompt: 'make the violin invisible',
+ previousJobId: 'v1_prior-turn',
+ logger: testLogger,
+ })
+
+ expect(stub.interactions.create).toHaveBeenCalledWith(
+ expect.objectContaining({
+ previous_interaction_id: 'v1_prior-turn',
+ }),
+ )
+ })
+ })
+
describe('getVideoStatus', () => {
const jobId = 'v1_omni-job-123'
diff --git a/packages/ai-grok/src/adapters/video.ts b/packages/ai-grok/src/adapters/video.ts
index 9131ef80eb..fa68d28fe0 100644
--- a/packages/ai-grok/src/adapters/video.ts
+++ b/packages/ai-grok/src/adapters/video.ts
@@ -3,6 +3,7 @@ import { BaseVideoAdapter, snapToDurationOption } from '@tanstack/ai/adapters'
import { toRunErrorPayload } from '@tanstack/ai/adapter-internals'
import { getGrokApiKeyFromEnv, withGrokDefaults } from '../utils/client'
import {
+ GROK_VIDEO_EDIT_KINDS,
GROK_VIDEO_MAX_REFERENCE_AUDIOS,
GROK_VIDEO_MAX_REFERENCE_IMAGES,
getGrokVideoDurationOptions,
@@ -16,6 +17,7 @@ import type {
ImagePart,
MediaInputMetadata,
TokenUsage,
+ VideoEditKind,
VideoGenerationOptions,
VideoJobResult,
VideoPart,
@@ -25,6 +27,7 @@ import type {
import type { GrokVideoModel } from '../model-meta'
import type {
GrokVideoModelDurationByName,
+ GrokVideoModelEditByName,
GrokVideoModelInputModalitiesByName,
GrokVideoModelProviderOptionsByName,
GrokVideoModelSizeByName,
@@ -136,7 +139,8 @@ export class GrokVideoAdapter<
GrokVideoModelProviderOptionsByName,
GrokVideoModelSizeByName,
GrokVideoModelInputModalitiesByName,
- GrokVideoModelDurationByName
+ GrokVideoModelDurationByName,
+ GrokVideoModelEditByName
> {
readonly name = 'grok' as const
@@ -189,6 +193,14 @@ export class GrokVideoAdapter<
return body
}
+ /**
+ * `grok-imagine-video` edits a prior clip by URL via `/videos/edits`;
+ * `grok-imagine-video-1.5` has no documented edit endpoint.
+ */
+ override supportedEditKind(): VideoEditKind | undefined {
+ return GROK_VIDEO_EDIT_KINDS[this.model]
+ }
+
async createVideoJob(
options: VideoGenerationOptions<
GrokVideoModelProviderOptionsByName[TModel],
@@ -198,6 +210,10 @@ export class GrokVideoAdapter<
): Promise {
const { model, size, modelOptions, logger } = options
+ if (options.previousJobId) {
+ return this.editVideoJob(options)
+ }
+
// `mode` is a routing hint for this adapter, not an API field — strip it
// before the remaining options are spread onto the request body. The
// per-model map narrows what callers can pass, but modelOptions often
@@ -546,6 +562,99 @@ export class GrokVideoAdapter<
}
}
+ /**
+ * Edit a previously generated clip via `POST /videos/edits`. The endpoint
+ * takes the source video by URL (public URL or base64 data URI) plus an
+ * edit prompt, and returns the same `{ request_id }` polled by
+ * `getVideoStatus` / `getVideoUrl`. The output inherits its duration and
+ * aspect ratio from the input (capped at 720p, truncated to 8 seconds),
+ * so conflicting options are rejected up front instead of being silently
+ * ignored by the API.
+ */
+ private async editVideoJob(
+ options: VideoGenerationOptions<
+ GrokVideoModelProviderOptionsByName[TModel],
+ GrokVideoModelSizeByName[TModel],
+ GrokVideoModelDurationByName[TModel]
+ >,
+ ): Promise {
+ const { model, size, duration, modelOptions, logger, previousJobId } =
+ options
+
+ if (this.supportedEditKind() === undefined) {
+ throw new Error(
+ `${this.name}: model "${model}" does not support editing previous generations (previousJobId).`,
+ )
+ }
+ if (!previousJobId) {
+ throw new Error(
+ `${this.name}: previousJobId is required to edit a previous generation with model "${model}".`,
+ )
+ }
+ if (
+ size !== undefined ||
+ duration !== undefined ||
+ modelOptions?.aspect_ratio !== undefined ||
+ modelOptions?.resolution !== undefined ||
+ modelOptions?.duration !== undefined
+ ) {
+ throw new Error(
+ `${this.name}: video edits inherit duration and aspect ratio from the source video — remove the size/duration options when using previousJobId.`,
+ )
+ }
+
+ const resolved = resolveMediaPrompt(options.prompt)
+ if (
+ resolved.images.length > 0 ||
+ resolved.videos.length > 0 ||
+ resolved.audios.length > 0
+ ) {
+ throw new Error(
+ `${this.name}: video edits accept only a text prompt; media prompt parts are not supported when previousJobId is set.`,
+ )
+ }
+ if (!resolved.text) {
+ throw new Error(
+ `${this.name}: video edits require a text prompt describing the edit.`,
+ )
+ }
+
+ const sourceUrl = await this.resolvePreviousJobUrl(previousJobId)
+
+ try {
+ logger.request(
+ `activity=video.edit provider=${this.name} model=${model}`,
+ { provider: this.name, model },
+ )
+
+ const response = await this.request('/videos/edits', {
+ method: 'POST',
+ body: JSON.stringify({
+ model,
+ prompt: resolved.text,
+ video_url: sourceUrl,
+ }),
+ })
+ if (!response.ok) {
+ throw new Error(
+ `grok: video edit request failed (${response.status} ${response.statusText}): ${await this.errorMessage(response)}`,
+ )
+ }
+
+ const result = (await response.json()) as GrokVideoCreateResponse
+ if (!result.request_id) {
+ throw new Error('grok: video edit response contained no request_id')
+ }
+ return { jobId: result.request_id, model }
+ } catch (error: unknown) {
+ logger.errors(`${this.name}.editVideoJob fatal`, {
+ error: toRunErrorPayload(error, `${this.name}.editVideoJob failed`),
+ source: `${this.name}.editVideoJob`,
+ })
+ throw error
+ }
+ }
+
private async retrieveJob(jobId: string): Promise {
const response = await this.request(`/videos/${jobId}`)
if (!response.ok) {
diff --git a/packages/ai-grok/src/index.ts b/packages/ai-grok/src/index.ts
index e1c935f734..6f6b57b7ac 100644
--- a/packages/ai-grok/src/index.ts
+++ b/packages/ai-grok/src/index.ts
@@ -42,6 +42,7 @@ export {
} from './adapters/video'
export {
GROK_VIDEO_DURATIONS,
+ GROK_VIDEO_EDIT_KINDS,
getGrokVideoDurationOptions,
} from './video/video-provider-options'
export type {
@@ -53,6 +54,7 @@ export type {
GrokVideoModelProviderOptionsByName,
GrokVideoModelSizeByName,
GrokVideoModelDurationByName,
+ GrokVideoModelEditByName,
GrokVideoAspectRatio,
GrokVideoResolution,
GrokVideoSize,
diff --git a/packages/ai-grok/src/video/video-provider-options.ts b/packages/ai-grok/src/video/video-provider-options.ts
index ca279e1575..e2d4312de6 100644
--- a/packages/ai-grok/src/video/video-provider-options.ts
+++ b/packages/ai-grok/src/video/video-provider-options.ts
@@ -403,3 +403,28 @@ export type GrokVideoModelInputModalitiesByName = {
'grok-imagine-video': readonly ['image', 'video']
'grok-imagine-video-1.5': readonly ['image']
}
+
+/**
+ * Per-model follow-up edit support. `grok-imagine-video` edits a previously
+ * generated clip via `POST /v1/videos/edits`. Callers pass `previousJobId`;
+ * the adapter resolves the finished clip via `getVideoUrl`. The edit
+ * endpoint is documented for `grok-imagine-video` only.
+ *
+ * @experimental Video generation is an experimental feature and may change.
+ */
+export type GrokVideoModelEditByName = {
+ 'grok-imagine-video': 'media'
+ 'grok-imagine-video-1.5': undefined
+}
+
+/**
+ * Runtime table backing `supportedEditKind()`.
+ *
+ * @experimental Video generation is an experimental feature and may change.
+ */
+export const GROK_VIDEO_EDIT_KINDS: {
+ readonly [TModel in GrokVideoModel]: GrokVideoModelEditByName[TModel]
+} = {
+ 'grok-imagine-video': 'media',
+ 'grok-imagine-video-1.5': undefined,
+}
diff --git a/packages/ai-grok/tests/video-adapter.test.ts b/packages/ai-grok/tests/video-adapter.test.ts
index 3f9fdd7a5a..b1976655ff 100644
--- a/packages/ai-grok/tests/video-adapter.test.ts
+++ b/packages/ai-grok/tests/video-adapter.test.ts
@@ -1315,4 +1315,147 @@ describe('Grok Video Adapter', () => {
expect(adapter.snapDuration(7)).toBe(7)
})
})
+
+ describe('previousJobId (/videos/edits)', () => {
+ it('reports media-kind edit support for grok-imagine-video only', () => {
+ expect(
+ createGrokVideo('grok-imagine-video', 'k').supportedEditKind(),
+ ).toBe('media')
+ expect(
+ createGrokVideo('grok-imagine-video-1.5', 'k').supportedEditKind(),
+ ).toBeUndefined()
+ })
+
+ it('posts the edit prompt and resolved source URL to /videos/edits', async () => {
+ const fetchMock = vi.fn(
+ async (input: string | URL | Request, _init?: RequestInit) => {
+ const url = String(input)
+ if (url.includes('/videos/edits')) {
+ return jsonResponse({ request_id: 'edit-req-1' })
+ }
+ return jsonResponse({
+ status: 'done',
+ video: { url: 'https://example.com/source.mp4' },
+ })
+ },
+ )
+ const adapter = createGrokVideo('grok-imagine-video', 'test-api-key', {
+ fetch: fetchMock,
+ })
+
+ const result = await adapter.createVideoJob({
+ model: 'grok-imagine-video',
+ prompt: 'Give the rider a red scarf',
+ previousJobId: 'prior-job',
+ logger: testLogger,
+ })
+
+ expect(result).toEqual({
+ jobId: 'edit-req-1',
+ model: 'grok-imagine-video',
+ })
+ expect(String(fetchMock.mock.calls[0]![0])).toContain('/videos/prior-job')
+ const [, editInit] = fetchMock.mock.calls[1]!
+ expect(String(fetchMock.mock.calls[1]![0])).toContain('/videos/edits')
+ expect(JSON.parse(String(editInit?.body))).toEqual({
+ model: 'grok-imagine-video',
+ prompt: 'Give the rider a red scarf',
+ video_url: 'https://example.com/source.mp4',
+ })
+ })
+
+ it('throws for the image-to-video-only model', async () => {
+ const fetchMock = mockFetch(() =>
+ jsonResponse({ request_id: 'edit-req-1' }),
+ )
+ const adapter = adapterWithFetch(fetchMock)
+
+ await expect(
+ adapter.createVideoJob({
+ model: 'grok-imagine-video-1.5',
+ prompt: 'x',
+ previousJobId: 'prior-job',
+ logger: testLogger,
+ }),
+ ).rejects.toThrow(/does not support editing previous generations/)
+ expect(fetchMock).not.toHaveBeenCalled()
+ })
+
+ it('rejects size/duration options and media parts on edit', async () => {
+ const fetchMock = mockFetch(() =>
+ jsonResponse({ request_id: 'edit-req-1' }),
+ )
+ const adapter = createGrokVideo('grok-imagine-video', 'test-api-key', {
+ fetch: fetchMock,
+ })
+ const previousJobId = 'prior-job'
+
+ await expect(
+ adapter.createVideoJob({
+ model: 'grok-imagine-video',
+ prompt: 'x',
+ size: '16:9_720p',
+ previousJobId,
+ logger: testLogger,
+ }),
+ ).rejects.toThrow(/inherit duration and aspect ratio/)
+
+ await expect(
+ adapter.createVideoJob({
+ model: 'grok-imagine-video',
+ prompt: 'x',
+ duration: 8,
+ previousJobId,
+ logger: testLogger,
+ }),
+ ).rejects.toThrow(/inherit duration and aspect ratio/)
+
+ await expect(
+ adapter.createVideoJob({
+ model: 'grok-imagine-video',
+ prompt: 'x',
+ modelOptions: { resolution: '720p' },
+ previousJobId,
+ logger: testLogger,
+ }),
+ ).rejects.toThrow(/inherit duration and aspect ratio/)
+
+ await expect(
+ adapter.createVideoJob({
+ model: 'grok-imagine-video',
+ prompt: i2vPrompt('x'),
+ previousJobId,
+ logger: testLogger,
+ }),
+ ).rejects.toThrow(/media prompt parts are not supported/)
+ expect(fetchMock).not.toHaveBeenCalled()
+ })
+
+ it('surfaces edit endpoint errors with the API message', async () => {
+ const fetchMock = vi.fn(
+ async (input: string | URL | Request, _init?: RequestInit) => {
+ const url = String(input)
+ if (url.includes('/videos/edits')) {
+ return jsonResponse({ code: 'bad', error: 'video too long' }, 400)
+ }
+ return jsonResponse({
+ status: 'done',
+ video: { url: 'https://example.com/source.mp4' },
+ })
+ },
+ )
+ const adapter = createGrokVideo('grok-imagine-video', 'test-api-key', {
+ fetch: fetchMock,
+ })
+
+ await expect(
+ adapter.createVideoJob({
+ model: 'grok-imagine-video',
+ prompt: 'x',
+ previousJobId: 'prior-job',
+ logger: testLogger,
+ }),
+ ).rejects.toThrow(/video edit request failed/)
+ })
+ })
})
diff --git a/packages/ai-openai/src/adapters/video.ts b/packages/ai-openai/src/adapters/video.ts
index 3e76478d62..1e74315321 100644
--- a/packages/ai-openai/src/adapters/video.ts
+++ b/packages/ai-openai/src/adapters/video.ts
@@ -11,6 +11,7 @@ import {
validateVideoSize,
} from '../video/video-provider-options'
import type {
+ VideoEditKind,
VideoGenerationOptions,
VideoJobResult,
VideoStatusResult,
@@ -19,6 +20,7 @@ import type {
import type OpenAI_SDK from 'openai'
import type { OpenAIVideoModel } from '../model-meta'
import type {
+ OpenAIVideoModelEditByName,
OpenAIVideoModelInputModalitiesByName,
OpenAIVideoModelProviderOptionsByName,
OpenAIVideoModelSizeByName,
@@ -81,7 +83,9 @@ export class OpenAIVideoAdapter<
OpenAIVideoProviderOptions,
OpenAIVideoModelProviderOptionsByName,
OpenAIVideoModelSizeByName,
- OpenAIVideoModelInputModalitiesByName
+ OpenAIVideoModelInputModalitiesByName,
+ Record,
+ OpenAIVideoModelEditByName
> {
readonly name = 'openai' as const
@@ -98,11 +102,20 @@ export class OpenAIVideoAdapter<
this.client = new OpenAI(clientOptions)
}
+ /** Sora models remix a completed video by its job id. */
+ override supportedEditKind(): VideoEditKind {
+ return 'job'
+ }
+
async createVideoJob(
options: VideoGenerationOptions,
): Promise {
const { model, size, duration, modelOptions } = options
+ if (options.previousJobId) {
+ return this.remixVideoJob(options)
+ }
+
const resolvedSize = size ?? modelOptions?.size
validateVideoSize(model, resolvedSize)
const seconds = duration ?? modelOptions?.seconds
@@ -177,6 +190,67 @@ export class OpenAIVideoAdapter<
}
}
+ /**
+ * Remix a previously generated Sora video (`POST /videos/{id}/remix`).
+ * The endpoint accepts only an updated prompt — the output inherits size
+ * and duration from the source video — so conflicting options are
+ * rejected up front instead of being silently dropped.
+ */
+ private async remixVideoJob(
+ options: VideoGenerationOptions,
+ ): Promise {
+ const { model, size, duration, modelOptions, previousJobId } = options
+ if (!previousJobId) {
+ throw new Error(
+ `${this.name}: previousJobId is required to remix a previous Sora generation.`,
+ )
+ }
+ if (size !== undefined || modelOptions?.size !== undefined) {
+ throw new Error(
+ `${this.name}: Sora remix accepts only a prompt — the output inherits the source video's size. Remove the size option.`,
+ )
+ }
+ if (duration !== undefined || modelOptions?.seconds !== undefined) {
+ throw new Error(
+ `${this.name}: Sora remix accepts only a prompt — the output inherits the source video's duration. Remove the duration/seconds option.`,
+ )
+ }
+
+ const resolved = resolveMediaPrompt(options.prompt)
+ if (
+ resolved.images.length > 0 ||
+ resolved.videos.length > 0 ||
+ resolved.audios.length > 0
+ ) {
+ throw new Error(
+ `${this.name}: Sora remix accepts only a text prompt; media prompt parts are not supported when previousJobId is set.`,
+ )
+ }
+ if (!resolved.text) {
+ throw new Error(
+ `${this.name}: Sora remix requires a text prompt describing the edit.`,
+ )
+ }
+
+ try {
+ options.logger.request(
+ `activity=video.remix provider=${this.name} model=${model} source=${previousJobId}`,
+ { provider: this.name, model },
+ )
+ const videosClient = this.getVideosClient()
+ const response = await videosClient.remix(previousJobId, {
+ prompt: resolved.text,
+ })
+ return { jobId: response.id, model }
+ } catch (error: any) {
+ options.logger.errors(`${this.name}.remixVideoJob fatal`, {
+ error: toRunErrorPayload(error, `${this.name}.remixVideoJob failed`),
+ source: `${this.name}.remixVideoJob`,
+ })
+ throw error
+ }
+ }
+
/**
* The video API on the OpenAI SDK is still experimental and shipped on some
* SDK versions but not others; access through `videosClient` lets us treat
@@ -184,6 +258,7 @@ export class OpenAIVideoAdapter<
*/
private getVideosClient(): {
create: (req: Record) => Promise<{ id: string }>
+ remix: (id: string, body: { prompt: string }) => Promise<{ id: string }>
retrieve: (id: string) => Promise<{
id: string
status: string
diff --git a/packages/ai-openai/src/index.ts b/packages/ai-openai/src/index.ts
index 4a4534b9b9..c29b199e2a 100644
--- a/packages/ai-openai/src/index.ts
+++ b/packages/ai-openai/src/index.ts
@@ -52,6 +52,7 @@ export {
export type {
OpenAIVideoProviderOptions,
OpenAIVideoModelProviderOptionsByName,
+ OpenAIVideoModelEditByName,
OpenAIVideoSize,
// OpenAIVideoDuration,
} from './video/video-provider-options'
diff --git a/packages/ai-openai/src/video/video-provider-options.ts b/packages/ai-openai/src/video/video-provider-options.ts
index 837c2c7b27..f8380ada2f 100644
--- a/packages/ai-openai/src/video/video-provider-options.ts
+++ b/packages/ai-openai/src/video/video-provider-options.ts
@@ -77,6 +77,19 @@ export type OpenAIVideoModelInputModalitiesByName = {
'sora-2-pro': readonly ['image']
}
+/**
+ * Per-model follow-up edit support. Sora models remix a completed video by
+ * its job id (`POST /videos/{id}/remix`), so `previousJobId` is the prior
+ * generation's job id. Remix accepts only a prompt — size, duration, and
+ * image inputs are rejected.
+ *
+ * @experimental Video generation is an experimental feature and may change.
+ */
+export type OpenAIVideoModelEditByName = {
+ 'sora-2': 'job'
+ 'sora-2-pro': 'job'
+}
+
/**
* Validate video size for a given model.
*
diff --git a/packages/ai-openai/tests/video-adapter.test.ts b/packages/ai-openai/tests/video-adapter.test.ts
index 158b85ce89..09123a334c 100644
--- a/packages/ai-openai/tests/video-adapter.test.ts
+++ b/packages/ai-openai/tests/video-adapter.test.ts
@@ -12,10 +12,11 @@ const testLogger = resolveDebugOption(false)
function mockedAdapter() {
const adapter = createOpenaiVideo('sora-2', 'test-api-key')
const mockCreate = vi.fn().mockResolvedValue({ id: 'video-job-1' })
+ const mockRemix = vi.fn().mockResolvedValue({ id: 'video-job-remix-1' })
;(adapter as unknown as { client: { videos: unknown } }).client = {
- videos: { create: mockCreate },
+ videos: { create: mockCreate, remix: mockRemix },
}
- return { adapter, mockCreate }
+ return { adapter, mockCreate, mockRemix }
}
describe('OpenAI Video Adapter', () => {
@@ -165,4 +166,92 @@ describe('OpenAI Video Adapter', () => {
expect(mockCreate).not.toHaveBeenCalled()
})
})
+
+ describe('previousJobId (Sora remix)', () => {
+ it('reports job-kind edit support', () => {
+ const adapter = createOpenaiVideo('sora-2', 'test-api-key')
+ expect(adapter.supportedEditKind()).toBe('job')
+ })
+
+ it('remixes the source video with the new prompt', async () => {
+ const { adapter, mockCreate, mockRemix } = mockedAdapter()
+
+ const result = await adapter.createVideoJob({
+ model: 'sora-2',
+ prompt: 'Make the sky stormy',
+ previousJobId: 'video-job-1',
+ logger: testLogger,
+ })
+
+ expect(mockRemix).toHaveBeenCalledWith('video-job-1', {
+ prompt: 'Make the sky stormy',
+ })
+ expect(mockCreate).not.toHaveBeenCalled()
+ expect(result).toEqual({ jobId: 'video-job-remix-1', model: 'sora-2' })
+ })
+
+ it('rejects size and duration options on remix', async () => {
+ const { adapter, mockRemix } = mockedAdapter()
+
+ await expect(
+ adapter.createVideoJob({
+ model: 'sora-2',
+ prompt: 'x',
+ size: '1280x720',
+ previousJobId: 'video-job-1',
+ logger: testLogger,
+ }),
+ ).rejects.toThrow(/inherits the source video's size/)
+
+ await expect(
+ adapter.createVideoJob({
+ model: 'sora-2',
+ prompt: 'x',
+ duration: 8,
+ previousJobId: 'video-job-1',
+ logger: testLogger,
+ }),
+ ).rejects.toThrow(/inherits the source video's duration/)
+
+ await expect(
+ adapter.createVideoJob({
+ model: 'sora-2',
+ prompt: 'x',
+ modelOptions: { seconds: '8' },
+ previousJobId: 'video-job-1',
+ logger: testLogger,
+ }),
+ ).rejects.toThrow(/inherits the source video's duration/)
+ expect(mockRemix).not.toHaveBeenCalled()
+ })
+
+ it('rejects media prompt parts and empty prompts on remix', async () => {
+ const { adapter, mockRemix } = mockedAdapter()
+
+ await expect(
+ adapter.createVideoJob({
+ model: 'sora-2',
+ prompt: [
+ { type: 'text', content: 'x' },
+ {
+ type: 'image',
+ source: { type: 'data', value: 'aGk=', mimeType: 'image/png' },
+ },
+ ],
+ previousJobId: 'video-job-1',
+ logger: testLogger,
+ }),
+ ).rejects.toThrow(/media prompt parts are not supported/)
+
+ await expect(
+ adapter.createVideoJob({
+ model: 'sora-2',
+ prompt: '',
+ previousJobId: 'video-job-1',
+ logger: testLogger,
+ }),
+ ).rejects.toThrow(/requires a text prompt/)
+ expect(mockRemix).not.toHaveBeenCalled()
+ })
+ })
})
diff --git a/packages/ai/skills/ai-core/media-generation/SKILL.md b/packages/ai/skills/ai-core/media-generation/SKILL.md
index 65ecaddb4e..bb5420748a 100644
--- a/packages/ai/skills/ai-core/media-generation/SKILL.md
+++ b/packages/ai/skills/ai-core/media-generation/SKILL.md
@@ -531,8 +531,8 @@ Image/video prompt parts are sent as interaction content blocks, grouped
as images, then videos, then text (no
`metadata.role` routing); `data` sources go inline, `url` sources pass
through as-is (never downloaded — use Gemini Files API URIs for remote
-media). For conversational editing, pass a prior generation's `jobId` as
-`modelOptions.previous_interaction_id` with a prompt describing the change:
+media). For conversational editing, pass a prior generation's job id via
+`previousJobId` with a prompt describing the change:
```typescript
import { geminiVideo } from '@tanstack/ai-gemini'
@@ -546,12 +546,27 @@ const first = await generateVideo({
const edited = await generateVideo({
adapter: omni,
prompt: 'Make the violin invisible',
- modelOptions: { previous_interaction_id: first.jobId },
+ previousJobId: first.jobId,
})
```
+`previousJobId` is the general follow-up-edit option for video. Callers pass
+the prior generation's job id; `'job'`-kind models reference it server-side
+(Sora 2 / Sora 2 Pro remix, Omni Flash interaction chaining) while
+`'media'`-kind models resolve the finished clip via `getVideoUrl`
+(`grokVideo('grok-imagine-video')` → xAI `/videos/edits`; fal video-to-video
+endpoints like `xai/grok-imagine-video/edit-video`). Non-editing models (Veo,
+`grok-imagine-video-1.5`) reject the option at compile time;
+`adapter.supportedEditKind()` reports `'job' | 'media' | undefined` at
+runtime. Sora remix and Grok edits take only a prompt — `size`/`duration`
+are rejected because the output inherits them from the source video.
+`generateImage` has the same option for image models that accept image
+inputs: `previousImage: priorResult.images[0]` prepends the previous image to
+the prompt so it flows through the model's regular edit path.
+
Other video adapters: `openaiVideo('sora-2')` (pixel sizes like `'1280x720'`,
-durations 4/8/12s, single `input_reference` image prompt part), `grokVideo(...)`
+durations 4/8/12s, single `input_reference` image prompt part, remix via
+`previousJobId`), `grokVideo(...)`
(`grok-imagine-video` and `grok-imagine-video-1.5` both do text-to-video + image-to-video;
1.5 adds reference-to-video — `'reference'`/`'character'`-roled image parts →
`reference_images` (max 7), preset voices via `modelOptions.reference_audios` (max 3) —
diff --git a/packages/ai/src/activities/generateImage/index.ts b/packages/ai/src/activities/generateImage/index.ts
index 713ca2ceca..46250d225c 100644
--- a/packages/ai/src/activities/generateImage/index.ts
+++ b/packages/ai/src/activities/generateImage/index.ts
@@ -23,15 +23,20 @@ import {
isActivityAbortError,
raceWithAbort,
} from '../../utilities/activity-abort'
-import { resolveMediaPrompt } from '../../utilities/media-prompt'
+import {
+ generatedImageToImagePart,
+ resolveMediaPrompt,
+} from '../../utilities/media-prompt'
import type { InternalLogger } from '../../logger/internal-logger'
import type { DebugOption } from '../../logger/types'
import type { GenerationMiddleware } from '../middleware/types'
import type { ImageAdapter } from './adapter'
import type {
+ GeneratedImage,
ImageGenerationResult,
MediaPrompt,
MediaPromptFor,
+ MediaPromptPart,
StreamChunk,
} from '../../types'
@@ -94,6 +99,35 @@ export type ImagePromptForModel =
: MediaPrompt
: MediaPrompt
+/**
+ * Previously generated image(s) accepted by `generateImage`'s `previousImage`:
+ * a single {@link GeneratedImage}, an array of them, or the whole prior
+ * {@link ImageGenerationResult} (its `images` are used).
+ */
+export type ImagePreviousSource =
+ | GeneratedImage
+ | ReadonlyArray
+ | Pick
+
+/**
+ * Extract the `previousImage` type for an ImageAdapter's model via ~types.
+ * Follow-up image edits work by re-passing the generated image as an image
+ * prompt part, so the option is offered exactly when the model accepts
+ * image inputs (`'image'` in its input-modality map); text-only models
+ * (DALL·E 3, Imagen) reject it at compile time. Adapters without a map fall
+ * back to accepting it, gated by the adapter's own runtime errors.
+ */
+export type ImagePreviousImageForModel =
+ TAdapter extends ImageAdapter
+ ? string extends keyof ModsByName
+ ? ImagePreviousSource
+ : TModel extends keyof ModsByName
+ ? 'image' extends ModsByName[TModel][number]
+ ? ImagePreviousSource
+ : never
+ : ImagePreviousSource
+ : ImagePreviousSource
+
// ===========================
// Activity Options Type
// ===========================
@@ -173,7 +207,22 @@ export type ImageActivityOptions<
TAdapter,
TAdapter['model']
>
- })
+ }) &
+ ([ImagePreviousImageForModel] extends [never]
+ ? {
+ /** This model does not accept image inputs, so it cannot edit previous generations. */
+ previousImage?: never
+ }
+ : {
+ /**
+ * Previously generated image(s) to edit instead of generating from
+ * scratch — pass a `GeneratedImage`, an array of them, or the whole
+ * prior `ImageGenerationResult`. They are prepended to the prompt as
+ * image parts and consumed by the model's existing edit path. Only
+ * offered for models that accept image inputs.
+ */
+ previousImage?: ImagePreviousImageForModel
+ })
// ===========================
// Activity Result Type
@@ -263,6 +312,30 @@ export function generateImage<
return runGenerateImage(options) as ImageActivityResult
}
+/**
+ * Normalize a `previousImage` value to the images it references and prepend
+ * them to the prompt as image parts, so the adapter's existing
+ * image-conditioned path (edit endpoint, `inlineData`, ...) consumes them.
+ */
+function prependPreviousImages(
+ prompt: MediaPrompt,
+ previousImage: ImagePreviousSource,
+): Array {
+ const images: ReadonlyArray =
+ 'images' in previousImage
+ ? previousImage.images
+ : Array.isArray(previousImage)
+ ? previousImage
+ : [previousImage]
+ if (images.length === 0) {
+ throw new Error('generateImage: previousImage contained no images.')
+ }
+ const imageParts = images.map(generatedImageToImagePart)
+ const promptParts: Array =
+ typeof prompt === 'string' ? [{ type: 'text', content: prompt }] : prompt
+ return [...imageParts, ...promptParts]
+}
+
/**
* Internal implementation of image generation (always non-streaming).
* Contains all devtools event emission logic.
@@ -277,12 +350,16 @@ async function runGenerateImage<
stream: _stream,
debug: _debug,
middleware,
+ previousImage,
threadId,
runId,
timeout,
abortSignal: callerAbortSignal,
...rest
} = options
+ const prompt: MediaPrompt = previousImage
+ ? prependPreviousImages(rest.prompt, previousImage)
+ : rest.prompt
const model = adapter.model
const requestId = createId('image')
const startTime = Date.now()
@@ -308,7 +385,7 @@ async function runGenerateImage<
// Devtools events carry the flattened prompt text plus media-part counts —
// the wire payload stays `prompt: string` regardless of the prompt shape.
- const resolved = resolveMediaPrompt(rest.prompt)
+ const resolved = resolveMediaPrompt(prompt)
aiEventClient.emit('image:request:started', {
requestId,
@@ -339,6 +416,7 @@ async function runGenerateImage<
const rawResult = await raceWithAbort(
adapter.generateImages({
...rest,
+ prompt,
model,
logger,
...(abortControls.signal ? { abortSignal: abortControls.signal } : {}),
diff --git a/packages/ai/src/activities/generateVideo/adapter.ts b/packages/ai/src/activities/generateVideo/adapter.ts
index 64dd0162e5..ad91b9b931 100644
--- a/packages/ai/src/activities/generateVideo/adapter.ts
+++ b/packages/ai/src/activities/generateVideo/adapter.ts
@@ -1,5 +1,7 @@
import type {
+ ModelEditKindByName,
ModelInputModalitiesByName,
+ VideoEditKind,
VideoGenerationOptions,
VideoJobResult,
VideoStatusResult,
@@ -56,6 +58,10 @@ export interface VideoAdapterConfig {
* - TModelDurationByName: Map from model name to its supported duration
* union. Defaults to `Record` so adapters that haven't
* declared a map keep today's `duration?: number` typing.
+ * - TModelEditByName: Map from model name to how it edits previously
+ * generated videos (`'job' | 'media' | undefined`). Defaults to the loose
+ * `ModelEditKindByName` so adapters that haven't declared a map keep an
+ * unconstrained `previousJobId` option (runtime-gated).
*/
export interface VideoAdapter<
TModel extends string = string,
@@ -69,6 +75,7 @@ export interface VideoAdapter<
ModelInputModalitiesByName,
TModelDurationByName extends Record =
Record,
+ TModelEditByName extends ModelEditKindByName = ModelEditKindByName,
> {
/** Discriminator for adapter kind - used to determine API shape */
readonly kind: 'video'
@@ -86,6 +93,7 @@ export interface VideoAdapter<
modelSizeByName: TModelSizeByName
modelInputModalitiesByName: TModelInputModalitiesByName
modelDurationByName: TModelDurationByName
+ modelEditByName: TModelEditByName
}
/**
@@ -123,13 +131,20 @@ export interface VideoAdapter<
* Returns `undefined` for models with no duration field.
*/
snapDuration: (seconds: number) => TModelDurationByName[TModel] | undefined
+
+ /**
+ * How this adapter's model edits previously generated videos, or
+ * `undefined` when it cannot. Consumed by the generateVideo() activity to
+ * gate `previousJobId` at runtime.
+ */
+ supportedEditKind: () => VideoEditKind | undefined
}
/**
* A VideoAdapter with any/unknown type parameters.
* Useful as a constraint in generic functions and interfaces.
*/
-export type AnyVideoAdapter = VideoAdapter
+export type AnyVideoAdapter = VideoAdapter
/**
* Abstract base class for video generation adapters.
@@ -151,13 +166,15 @@ export abstract class BaseVideoAdapter<
ModelInputModalitiesByName,
TModelDurationByName extends Record =
Record,
+ TModelEditByName extends ModelEditKindByName = ModelEditKindByName,
> implements VideoAdapter<
TModel,
TProviderOptions,
TModelProviderOptionsByName,
TModelSizeByName,
TModelInputModalitiesByName,
- TModelDurationByName
+ TModelDurationByName,
+ TModelEditByName
> {
readonly kind = 'video' as const
abstract readonly name: string
@@ -170,6 +187,7 @@ export abstract class BaseVideoAdapter<
modelSizeByName: TModelSizeByName
modelInputModalitiesByName: TModelInputModalitiesByName
modelDurationByName: TModelDurationByName
+ modelEditByName: TModelEditByName
}
protected config: VideoAdapterConfig
@@ -207,6 +225,32 @@ export abstract class BaseVideoAdapter<
return undefined
}
+ /**
+ * Default implementation returns `undefined` (no follow-up editing).
+ * Adapters whose models can edit previous generations should override.
+ */
+ supportedEditKind(): VideoEditKind | undefined {
+ return undefined
+ }
+
+ /**
+ * Resolve the source video URL for a media-kind edit by fetching the
+ * finished clip for `previousJobId`. Callers always pass the prior
+ * generation's job id; media-kind adapters call this instead of
+ * requiring a URL.
+ */
+ protected async resolvePreviousJobUrl(
+ previousJobId: string,
+ ): Promise {
+ const result = await this.getVideoUrl(previousJobId)
+ if (!result.url) {
+ throw new Error(
+ `${this.name}: could not resolve a video URL from previousJobId "${previousJobId}".`,
+ )
+ }
+ return result.url
+ }
+
protected generateId(): string {
return `${this.name}-${Date.now()}-${Math.random().toString(36).substring(7)}`
}
diff --git a/packages/ai/src/activities/generateVideo/index.ts b/packages/ai/src/activities/generateVideo/index.ts
index 9fdc977450..f83ac4c8a7 100644
--- a/packages/ai/src/activities/generateVideo/index.ts
+++ b/packages/ai/src/activities/generateVideo/index.ts
@@ -39,6 +39,7 @@ import type {
PersistedArtifactRef,
StreamChunk,
TokenUsage,
+ VideoEditKind,
VideoJobResult,
VideoStatusResult,
VideoUrlResult,
@@ -59,7 +60,7 @@ export const kind = 'video' as const
* Extract provider options from a VideoAdapter via ~types.
*/
export type VideoProviderOptions =
- TAdapter extends VideoAdapter
+ TAdapter extends VideoAdapter
? TAdapter['~types']['providerOptions']
: object
@@ -73,6 +74,7 @@ export type VideoSizeForAdapter =
any,
infer TSizeMap,
any,
+ any,
any
>
? TModel extends keyof TSizeMap
@@ -93,6 +95,7 @@ export type VideoPromptForAdapter =
any,
any,
infer ModsByName,
+ any,
any
>
? string extends keyof ModsByName
@@ -114,13 +117,46 @@ export type VideoDurationForAdapter =
any,
any,
any,
- infer TDurationMap
+ infer TDurationMap,
+ any
>
? TModel extends keyof TDurationMap
? TDurationMap[TModel]
: number
: number
+/**
+ * Extract the `previousJobId` type for a VideoAdapter's model via ~types.
+ *
+ * Models that support follow-up edits (any declared `VideoEditKind`) accept
+ * a prior generation's job id as a string. `'media'`-kind adapters resolve
+ * a URL from it via `getVideoUrl`. Models declared uneditable (`undefined`
+ * in the map) resolve to `never`, rejecting `previousJobId` at compile
+ * time. Adapters without a declared map — and open-world model unions
+ * (e.g. fal's arbitrary endpoint strings) — fall back to `string`, gated
+ * at runtime instead.
+ */
+export type VideoPreviousJobIdForAdapter =
+ TAdapter extends VideoAdapter<
+ infer TModel,
+ any,
+ any,
+ any,
+ any,
+ any,
+ infer TEditMap
+ >
+ ? string extends keyof TEditMap
+ ? string
+ : TModel extends keyof TEditMap
+ ? TEditMap[TModel] extends VideoEditKind
+ ? string
+ : VideoEditKind extends TEditMap[TModel]
+ ? string
+ : never
+ : never
+ : never
+
// ===========================
// Activity Options Types
@@ -134,7 +170,7 @@ function createId(prefix: string): string {
* The model is extracted from the adapter's model property.
*/
interface VideoActivityBaseOptions<
- TAdapter extends VideoAdapter,
+ TAdapter extends VideoAdapter,
> {
/** The video adapter to use (must be created with a model) */
adapter: TAdapter & { kind: typeof kind }
@@ -150,7 +186,7 @@ interface VideoActivityBaseOptions<
* @experimental Video generation is an experimental feature and may change.
*/
export type VideoCreateOptions<
- TAdapter extends VideoAdapter,
+ TAdapter extends VideoAdapter,
TStream extends boolean = false,
> = VideoActivityBaseOptions & {
/** Request type - create a new job (default if not specified) */
@@ -266,6 +302,21 @@ export type VideoCreateOptions<
}
: {
/** Provider-specific options for video generation */ modelOptions: VideoProviderOptions
+ }) &
+ ([VideoPreviousJobIdForAdapter] extends [never]
+ ? {
+ /** This model does not support editing previous generations. */
+ previousJobId?: never
+ }
+ : {
+ /**
+ * Edit a previously generated video instead of generating from
+ * scratch. Pass the prior generation's job id. `'job'`-kind models
+ * reference it server-side; `'media'`-kind models resolve the
+ * finished clip via `getVideoUrl`. Only offered for models that
+ * support follow-up edits.
+ */
+ previousJobId?: VideoPreviousJobIdForAdapter
})
/**
@@ -274,7 +325,7 @@ export type VideoCreateOptions<
* @experimental Video generation is an experimental feature and may change.
*/
export interface VideoStatusOptions<
- TAdapter extends VideoAdapter,
+ TAdapter extends VideoAdapter,
> extends VideoActivityBaseOptions {
/** Request type - get job status */
request: 'status'
@@ -288,7 +339,7 @@ export interface VideoStatusOptions<
* @experimental Video generation is an experimental feature and may change.
*/
export interface VideoUrlOptions<
- TAdapter extends VideoAdapter,
+ TAdapter extends VideoAdapter,
> extends VideoActivityBaseOptions {
/** Request type - get video URL */
request: 'url'
@@ -303,7 +354,7 @@ export interface VideoUrlOptions<
* @experimental Video generation is an experimental feature and may change.
*/
export type VideoActivityOptions<
- TAdapter extends VideoAdapter,
+ TAdapter extends VideoAdapter,
TRequest extends 'create' | 'status' | 'url' = 'create',
TStream extends boolean = false,
> = TRequest extends 'status'
@@ -387,7 +438,7 @@ export type VideoActivityResult<
* ```
*/
export function generateVideo<
- TAdapter extends VideoAdapter,
+ TAdapter extends VideoAdapter,
TStream extends boolean = false,
>(
options: VideoCreateOptions,
@@ -401,6 +452,35 @@ export function generateVideo<
return runCreateVideoJob(options) as VideoActivityResult<'create', TStream>
}
+/**
+ * Validate a `previousJobId` against the adapter's declared edit support
+ * before the adapter is called, so every provider surfaces the same clear
+ * errors. Media-kind adapters resolve a URL from the id themselves. Returns
+ * the validated id (or undefined when absent).
+ */
+function validatePreviousJobId(
+ adapter: {
+ name: string
+ model: string
+ supportedEditKind: () => VideoEditKind | undefined
+ },
+ previousJobId: string | undefined,
+): string | undefined {
+ if (previousJobId === undefined) return undefined
+ const kind = adapter.supportedEditKind()
+ if (kind === undefined) {
+ throw new Error(
+ `${adapter.name}: model "${adapter.model}" does not support editing previous generations (previousJobId).`,
+ )
+ }
+ if (!previousJobId) {
+ throw new Error(
+ `${adapter.name}: previousJobId is required to edit a previous generation with model "${adapter.model}".`,
+ )
+ }
+ return previousJobId
+}
+
/**
* The run id a non-streaming video job is filed under, derived from the
* provider job itself.
@@ -436,7 +516,7 @@ function videoRunIdForJob(provider: string, jobId: string): string {
* showing nothing.
*/
async function runCreateVideoJob<
- TAdapter extends VideoAdapter,
+ TAdapter extends VideoAdapter,
>(options: VideoCreateOptions): Promise {
const {
adapter,
@@ -489,6 +569,7 @@ async function runCreateVideoJob<
let jobResult: VideoJobResult
try {
+ const previousJobId = validatePreviousJobId(adapter, options.previousJobId)
jobResult = await raceWithAbort(
adapter.createVideoJob({
model,
@@ -497,6 +578,7 @@ async function runCreateVideoJob<
duration,
modelOptions,
logger,
+ ...(previousJobId ? { previousJobId } : {}),
...(abortControls.signal ? { abortSignal: abortControls.signal } : {}),
}),
abortControls.signal,
@@ -567,7 +649,7 @@ function sleep(ms: number, signal?: AbortSignal): Promise {
* Handles the full job lifecycle: create job → poll for status → stream updates → yield final result.
*/
async function* runStreamingVideoGeneration<
- TAdapter extends VideoAdapter,
+ TAdapter extends VideoAdapter,
>(options: VideoCreateOptions): AsyncIterable {
const {
adapter,
@@ -643,6 +725,7 @@ async function* runStreamingVideoGeneration<
let settled = false
try {
// Create the video generation job
+ const previousJobId = validatePreviousJobId(adapter, options.previousJobId)
const jobResult = await raceWithAbort(
adapter.createVideoJob({
model,
@@ -651,6 +734,7 @@ async function* runStreamingVideoGeneration<
duration,
modelOptions,
logger,
+ ...(previousJobId ? { previousJobId } : {}),
...(abortControls.signal ? { abortSignal: abortControls.signal } : {}),
}),
abortControls.signal,
@@ -908,7 +992,7 @@ export interface VideoJobStatusResult {
* ```
*/
export async function getVideoJobStatus<
- TAdapter extends VideoAdapter,
+ TAdapter extends VideoAdapter,
>(options: VideoJobStatusOptions): Promise {
const { adapter, jobId, middleware } = options
const requestId = createId('video-status')
@@ -1067,7 +1151,7 @@ export async function getVideoJobStatus<
* Create typed options for the generateVideo() function without executing.
*/
export function createVideoOptions<
- TAdapter extends VideoAdapter,
+ TAdapter extends VideoAdapter,
TStream extends boolean = false,
>(
options: VideoCreateOptions,
diff --git a/packages/ai/src/activities/index.ts b/packages/ai/src/activities/index.ts
index fc344859ab..07194a8ba0 100644
--- a/packages/ai/src/activities/index.ts
+++ b/packages/ai/src/activities/index.ts
@@ -96,6 +96,8 @@ export {
generateImage,
type ImageActivityOptions,
type ImageActivityResult,
+ type ImagePreviousImageForModel,
+ type ImagePreviousSource,
type ImageProviderOptionsForModel,
type ImageSizeForModel,
} from './generateImage/index'
@@ -141,6 +143,7 @@ export {
type VideoStatusOptions,
type VideoUrlOptions,
type VideoDurationForAdapter,
+ type VideoPreviousJobIdForAdapter,
} from './generateVideo/index'
export {
diff --git a/packages/ai/src/client.ts b/packages/ai/src/client.ts
index d5f048cf75..7e1b8cdff7 100644
--- a/packages/ai/src/client.ts
+++ b/packages/ai/src/client.ts
@@ -307,6 +307,8 @@ export type {
ContentPartUrlSource,
CustomEvent,
DocumentPart,
+ GeneratedImage,
+ GeneratedMediaSource,
ImagePart,
MediaInputMetadata,
MediaInputRole,
@@ -332,6 +334,7 @@ export type {
ToolResultPart,
UIMessage,
UIResourcePart,
+ VideoEditKind,
VideoPart,
InferSchemaType,
} from './types'
diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts
index 12f66efa37..bda88be054 100644
--- a/packages/ai/src/index.ts
+++ b/packages/ai/src/index.ts
@@ -362,7 +362,11 @@ export {
export { buildBaseUsage, type BaseUsageInput } from './utilities/usage'
// Media-generation prompt resolution (used by image / video adapters)
-export { resolveMediaPrompt } from './utilities/media-prompt'
+export {
+ resolveMediaPrompt,
+ generatedImageToImagePart,
+ generatedVideoUrlToVideoPart,
+} from './utilities/media-prompt'
export type { ResolvedMediaPrompt } from './utilities/media-prompt'
// Embedding input resolution (used by embedding adapters)
diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts
index d61a919220..6f5a5d9bd0 100644
--- a/packages/ai/src/types.ts
+++ b/packages/ai/src/types.ts
@@ -2383,6 +2383,27 @@ export interface AudioGenerationResult {
// Video Generation Types (Experimental)
// ============================================================================
+/**
+ * How a video model edits previously generated media in a follow-up run:
+ * - `'job'` — the provider references the prior generation by its job id
+ * (OpenAI Sora remix, Gemini Omni `previous_interaction_id`).
+ * - `'media'` — the provider takes the prior video itself, by URL or data URI
+ * (xAI Grok `/videos/edits`, fal video-to-video endpoints).
+ *
+ * @experimental Video generation is an experimental feature and may change.
+ */
+export type VideoEditKind = 'job' | 'media'
+
+/**
+ * Per-model map from model name to its edit kind (`undefined` = the model
+ * cannot edit previous generations). Adapters declare this map to narrow the
+ * `previousJobId` option per model at compile time, mirroring
+ * `ModelInputModalitiesByName`.
+ *
+ * @experimental Video generation is an experimental feature and may change.
+ */
+export type ModelEditKindByName = Record
+
/**
* Options for video generation.
* These are the common options supported across providers.
@@ -2415,6 +2436,14 @@ export interface VideoGenerationOptions<
duration?: TDuration
/** Model-specific options for video generation */
modelOptions?: TProviderOptions
+ /**
+ * Prior generation's job id to edit instead of generating from scratch.
+ * Only present when the model supports follow-up edits (see
+ * `VideoEditKind`); the activity validates support before the adapter is
+ * called. `'job'`-kind adapters reference it server-side; `'media'`-kind
+ * adapters resolve the finished clip via `getVideoUrl`.
+ */
+ previousJobId?: string
/**
* Internal logger threaded from the generateVideo() entry point. Adapters must
* call logger.request() before the SDK call and logger.errors() in catch blocks.
diff --git a/packages/ai/src/utilities/media-prompt.ts b/packages/ai/src/utilities/media-prompt.ts
index 9cefb64fff..fccf962e96 100644
--- a/packages/ai/src/utilities/media-prompt.ts
+++ b/packages/ai/src/utilities/media-prompt.ts
@@ -1,5 +1,8 @@
+import { detectImageMimeType } from '../utils'
import type {
AudioPart,
+ ContentPartSource,
+ GeneratedImage,
ImagePart,
MediaInputMetadata,
MediaPrompt,
@@ -84,3 +87,62 @@ export function resolveMediaPrompt(prompt: MediaPrompt): ResolvedMediaPrompt {
audios,
}
}
+
+/**
+ * Convert a media URL into a {@link ContentPartSource}: remote URLs pass
+ * through as `url` sources; `data:` URLs are decomposed into `data` sources
+ * so adapters that upload raw bytes (OpenAI edits, Gemini `inlineData`) get
+ * the payload without re-parsing.
+ */
+function mediaUrlToSource(url: string): ContentPartSource {
+ if (!url.startsWith('data:')) {
+ return { type: 'url', value: url }
+ }
+ const comma = url.indexOf(',')
+ const mimeType = comma === -1 ? '' : url.slice(5, comma).split(';')[0]
+ if (comma === -1 || !mimeType) {
+ throw new Error('data: URL is missing a mime type')
+ }
+ return { type: 'data', value: url.slice(comma + 1), mimeType }
+}
+
+/**
+ * Convert a previously generated image into an {@link ImagePart} so it can
+ * be fed back into a media prompt (follow-up edits, image-to-image). URL
+ * results pass through as `url` sources (`data:` URLs are decomposed into
+ * `data` sources); `b64Json` results become `data` sources with the mime
+ * type sniffed from the payload's magic bytes (defaulting to `image/png`,
+ * the format every provider's b64 output uses today).
+ */
+export function generatedImageToImagePart(
+ image: GeneratedImage,
+): ImagePart {
+ if (image.url !== undefined) {
+ return { type: 'image', source: mediaUrlToSource(image.url) }
+ }
+ return {
+ type: 'image',
+ source: {
+ type: 'data',
+ value: image.b64Json,
+ mimeType: detectImageMimeType(image.b64Json) ?? 'image/png',
+ },
+ }
+}
+
+/**
+ * Convert a previously generated video URL into a {@link VideoPart} so it
+ * can be fed back into a media prompt (follow-up edits). Remote URLs pass
+ * through; `data:` URLs (e.g. Gemini Omni's inline MP4 results) are
+ * decomposed into `data` sources.
+ */
+export function generatedVideoUrlToVideoPart(
+ url: string,
+ metadata?: MediaInputMetadata,
+): VideoPart {
+ return {
+ type: 'video',
+ source: mediaUrlToSource(url),
+ ...(metadata ? { metadata } : {}),
+ }
+}
diff --git a/packages/ai/tests/generate-image-previous-image.test.ts b/packages/ai/tests/generate-image-previous-image.test.ts
new file mode 100644
index 0000000000..7ef63d5e6c
--- /dev/null
+++ b/packages/ai/tests/generate-image-previous-image.test.ts
@@ -0,0 +1,210 @@
+/**
+ * Tests for generateImage's `previousImage` sugar: previously generated images
+ * are normalized and prepended to the prompt as image parts, and the option
+ * itself never leaks into the adapter's options.
+ */
+import { describe, expect, expectTypeOf, it, vi } from 'vitest'
+import { generateImage } from '../src/activities/generateImage'
+import { BaseImageAdapter } from '../src/activities/generateImage/adapter'
+import { generatedImageToImagePart } from '../src/utilities/media-prompt'
+import type { ImageActivityOptions } from '../src/activities/generateImage'
+import type {
+ ImageGenerationOptions,
+ ImageGenerationResult,
+ MediaPromptPart,
+} from '../src/types'
+
+function mockImageAdapter() {
+ const generateImages = vi.fn(async (options: ImageGenerationOptions) => ({
+ id: 'img-1',
+ model: options.model,
+ images: [{ url: 'https://example.com/out.png' }],
+ }))
+ const adapter = {
+ kind: 'image' as const,
+ name: 'mock',
+ model: 'mock-image-model',
+ generateImages,
+ }
+ return { adapter, generateImages }
+}
+
+function partsOf(call: ImageGenerationOptions): Array {
+ return call.prompt as Array
+}
+
+describe('generateImage previousImage', () => {
+ it('prepends a single GeneratedImage as an image part before the prompt', async () => {
+ const { adapter, generateImages } = mockImageAdapter()
+
+ await generateImage({
+ adapter: adapter as any,
+ prompt: 'make it night time',
+ previousImage: { url: 'https://example.com/v1.png' },
+ })
+
+ const options = generateImages.mock.calls[0]![0]
+ expect(partsOf(options)).toEqual([
+ {
+ type: 'image',
+ source: { type: 'url', value: 'https://example.com/v1.png' },
+ },
+ { type: 'text', content: 'make it night time' },
+ ])
+ expect('previousImage' in options).toBe(false)
+ })
+
+ it('accepts an array and the whole prior result, preserving order', async () => {
+ const { adapter, generateImages } = mockImageAdapter()
+
+ await generateImage({
+ adapter: adapter as any,
+ prompt: 'blend these',
+ previousImage: [
+ { url: 'https://example.com/a.png' },
+ { url: 'https://example.com/b.png' },
+ ],
+ })
+
+ await generateImage({
+ adapter: adapter as any,
+ prompt: 'refine',
+ previousImage: {
+ images: [{ url: 'https://example.com/c.png' }],
+ },
+ })
+
+ const first = partsOf(generateImages.mock.calls[0]![0])
+ expect(first.map((p) => p.type)).toEqual(['image', 'image', 'text'])
+ const second = partsOf(generateImages.mock.calls[1]![0])
+ expect(second[0]).toEqual({
+ type: 'image',
+ source: { type: 'url', value: 'https://example.com/c.png' },
+ })
+ })
+
+ it('prepends to an existing parts-array prompt', async () => {
+ const { adapter, generateImages } = mockImageAdapter()
+
+ await generateImage({
+ adapter: adapter as any,
+ prompt: [
+ { type: 'text', content: 'use the attached style' },
+ {
+ type: 'image',
+ source: { type: 'url', value: 'https://example.com/style.png' },
+ metadata: { role: 'reference' },
+ },
+ ],
+ previousImage: { url: 'https://example.com/v1.png' },
+ })
+
+ const parts = partsOf(generateImages.mock.calls[0]![0])
+ expect(parts).toHaveLength(3)
+ expect(parts[0]).toEqual({
+ type: 'image',
+ source: { type: 'url', value: 'https://example.com/v1.png' },
+ })
+ expect(parts[1]?.type).toBe('text')
+ })
+
+ it('throws when previousImage contains no images', async () => {
+ const { adapter, generateImages } = mockImageAdapter()
+
+ await expect(
+ generateImage({
+ adapter: adapter as any,
+ prompt: 'x',
+ previousImage: { images: [] },
+ }),
+ ).rejects.toThrow(/previousImage contained no images/)
+ expect(generateImages).not.toHaveBeenCalled()
+ })
+})
+
+describe('generatedImageToImagePart', () => {
+ it('passes remote URLs through as url sources', () => {
+ expect(
+ generatedImageToImagePart({ url: 'https://example.com/v1.png' }),
+ ).toEqual({
+ type: 'image',
+ source: { type: 'url', value: 'https://example.com/v1.png' },
+ })
+ })
+
+ it('decomposes data: URLs into data sources', () => {
+ expect(
+ generatedImageToImagePart({ url: 'data:image/jpeg;base64,/9j/AAA=' }),
+ ).toEqual({
+ type: 'image',
+ source: { type: 'data', value: '/9j/AAA=', mimeType: 'image/jpeg' },
+ })
+ })
+
+ it('throws on a data: URL without a mime type', () => {
+ expect(() => generatedImageToImagePart({ url: 'data:,hello' })).toThrow(
+ /missing a mime type/,
+ )
+ })
+
+ it('sniffs the mime type of b64Json payloads from magic bytes', () => {
+ expect(generatedImageToImagePart({ b64Json: 'iVBORw0KGgoAAA' })).toEqual({
+ type: 'image',
+ source: { type: 'data', value: 'iVBORw0KGgoAAA', mimeType: 'image/png' },
+ })
+ expect(
+ generatedImageToImagePart({ b64Json: '/9j/4AAQSkZJRg' }).source,
+ ).toMatchObject({ mimeType: 'image/jpeg' })
+ // Unknown payloads default to png.
+ expect(generatedImageToImagePart({ b64Json: 'AAAA' }).source).toMatchObject(
+ { mimeType: 'image/png' },
+ )
+ })
+})
+
+// ===========================
+// Compile-time typing
+// ===========================
+
+type MockEditModel = 'edit-capable' | 'text-only'
+
+type MockEditModelSizeByName = {
+ 'edit-capable': '1024x1024'
+ 'text-only': '1024x1024'
+}
+
+type MockEditModelInputModalitiesByName = {
+ 'edit-capable': readonly ['image']
+ 'text-only': readonly []
+}
+
+class MockEditImageAdapter<
+ TModel extends MockEditModel,
+> extends BaseImageAdapter<
+ TModel,
+ Record,
+ Record>,
+ MockEditModelSizeByName,
+ MockEditModelInputModalitiesByName
+> {
+ override readonly kind = 'image' as const
+ readonly name = 'mock' as const
+
+ generateImages = async (): Promise => {
+ return { id: 'mock-id', model: this.model, images: [] }
+ }
+}
+
+describe('previousImage per-model typing', () => {
+ it('offers previousImage only for models that accept image inputs', () => {
+ type EditCapable = ImageActivityOptions<
+ MockEditImageAdapter<'edit-capable'>
+ >['previousImage']
+ type TextOnly = ImageActivityOptions<
+ MockEditImageAdapter<'text-only'>
+ >['previousImage']
+
+ expectTypeOf<{ url: string }>().toExtend>()
+ expectTypeOf().toEqualTypeOf()
+ })
+})
diff --git a/packages/ai/tests/generate-video-previous-job-id.test.ts b/packages/ai/tests/generate-video-previous-job-id.test.ts
new file mode 100644
index 0000000000..b93f4cd9c3
--- /dev/null
+++ b/packages/ai/tests/generate-video-previous-job-id.test.ts
@@ -0,0 +1,207 @@
+/**
+ * Tests for generateVideo's `previousJobId` option: the core gate that
+ * validates edit support before the adapter is called, the passthrough into
+ * `createVideoJob`, and the per-model compile-time typing.
+ */
+import { describe, expect, expectTypeOf, it, vi } from 'vitest'
+import { generateVideo } from '../src/activities/generateVideo'
+import { BaseVideoAdapter } from '../src/activities/generateVideo/adapter'
+import type { VideoCreateOptions } from '../src/activities/generateVideo'
+import type {
+ VideoEditKind,
+ VideoGenerationOptions,
+ VideoJobResult,
+ VideoStatusResult,
+ VideoUrlResult,
+} from '../src/types'
+
+class MockVideoAdapter extends BaseVideoAdapter<'mock-model'> {
+ readonly name = 'mock'
+ editKind: VideoEditKind | undefined
+ lastOptions: VideoGenerationOptions | undefined
+
+ constructor(editKind?: VideoEditKind) {
+ super({}, 'mock-model')
+ this.editKind = editKind
+ }
+
+ override supportedEditKind(): VideoEditKind | undefined {
+ return this.editKind
+ }
+
+ createVideoJob = vi.fn(
+ async (options: VideoGenerationOptions): Promise => {
+ this.lastOptions = options
+ return { jobId: 'job-1', model: this.model }
+ },
+ )
+
+ getVideoStatus = vi.fn(
+ async (jobId: string): Promise => ({
+ jobId,
+ status: 'completed',
+ }),
+ )
+
+ getVideoUrl = vi.fn(
+ async (jobId: string): Promise => ({
+ jobId,
+ url: 'https://example.com/video.mp4',
+ }),
+ )
+}
+
+describe('generateVideo previousJobId gate', () => {
+ it('throws when the model does not support editing', async () => {
+ const adapter = new MockVideoAdapter(undefined)
+
+ await expect(
+ generateVideo({
+ adapter,
+ prompt: 'x',
+ previousJobId: 'prior-job',
+ }),
+ ).rejects.toThrow(/does not support editing previous generations/)
+ expect(adapter.createVideoJob).not.toHaveBeenCalled()
+ })
+
+ it('throws when previousJobId is empty', async () => {
+ const adapter = new MockVideoAdapter('job')
+
+ await expect(
+ generateVideo({
+ adapter,
+ prompt: 'x',
+ previousJobId: '',
+ }),
+ ).rejects.toThrow(/previousJobId is required/)
+ expect(adapter.createVideoJob).not.toHaveBeenCalled()
+ })
+
+ it('forwards previousJobId to media-kind adapters (URL resolve is adapter-side)', async () => {
+ const adapter = new MockVideoAdapter('media')
+
+ await generateVideo({
+ adapter,
+ prompt: 'x',
+ previousJobId: 'prior-job',
+ })
+
+ expect(adapter.lastOptions?.previousJobId).toBe('prior-job')
+ // Core does not resolve the URL — media adapters call getVideoUrl themselves.
+ expect(adapter.getVideoUrl).not.toHaveBeenCalled()
+ })
+
+ it('forwards a valid previousJobId to the adapter', async () => {
+ const adapter = new MockVideoAdapter('job')
+
+ const result = await generateVideo({
+ adapter,
+ prompt: 'make it stormy',
+ previousJobId: 'prior-job',
+ })
+
+ expect(result).toEqual({ jobId: 'job-1', model: 'mock-model' })
+ expect(adapter.lastOptions?.previousJobId).toBe('prior-job')
+ })
+
+ it('omits previousJobId from adapter options when not provided', async () => {
+ const adapter = new MockVideoAdapter('job')
+
+ await generateVideo({ adapter, prompt: 'x' })
+
+ expect(adapter.lastOptions).toBeDefined()
+ expect('previousJobId' in adapter.lastOptions!).toBe(false)
+ })
+
+ it('gates previousJobId in streaming mode too', async () => {
+ const adapter = new MockVideoAdapter(undefined)
+
+ const chunks = []
+ for await (const chunk of generateVideo({
+ adapter,
+ prompt: 'x',
+ previousJobId: 'prior-job',
+ stream: true,
+ pollingInterval: 1,
+ })) {
+ chunks.push(chunk)
+ }
+
+ const errorChunk = chunks.find((c) => c.type === 'RUN_ERROR')
+ expect(errorChunk).toBeDefined()
+ expect((errorChunk as { message: string }).message).toMatch(
+ /does not support editing previous generations/,
+ )
+ expect(adapter.createVideoJob).not.toHaveBeenCalled()
+ })
+
+ it('forwards previousJobId to the adapter in streaming mode', async () => {
+ const adapter = new MockVideoAdapter('media')
+
+ for await (const _chunk of generateVideo({
+ adapter,
+ prompt: 'x',
+ previousJobId: 'prior-job',
+ stream: true,
+ pollingInterval: 1,
+ })) {
+ // drain
+ }
+
+ expect(adapter.lastOptions?.previousJobId).toBe('prior-job')
+ })
+})
+
+// ===========================
+// Compile-time typing
+// ===========================
+
+type JobEditAdapter = BaseVideoAdapter<
+ 'job-model',
+ Record,
+ Record,
+ Record,
+ { 'job-model': readonly ['image'] },
+ Record,
+ { 'job-model': 'job' }
+>
+
+type MediaEditAdapter = BaseVideoAdapter<
+ 'media-model',
+ Record,
+ Record,
+ Record,
+ { 'media-model': readonly ['image'] },
+ Record,
+ { 'media-model': 'media' }
+>
+
+type NoEditAdapter = BaseVideoAdapter<
+ 'no-edit-model',
+ Record,
+ Record,
+ Record,
+ { 'no-edit-model': readonly [] },
+ Record,
+ { 'no-edit-model': undefined }
+>
+
+describe('previousJobId per-model typing', () => {
+ it('accepts a string job id for every editable model', () => {
+ type JobPreviousJobId = NonNullable<
+ VideoCreateOptions['previousJobId']
+ >
+ type MediaPreviousJobId = NonNullable<
+ VideoCreateOptions['previousJobId']
+ >
+
+ expectTypeOf().toEqualTypeOf()
+ expectTypeOf().toEqualTypeOf()
+ })
+
+ it('rejects previousJobId entirely for non-editing models', () => {
+ type NoPreviousJobId = VideoCreateOptions['previousJobId']
+ expectTypeOf().toEqualTypeOf()
+ })
+})
diff --git a/packages/ai/tests/stream-generation.test.ts b/packages/ai/tests/stream-generation.test.ts
index 2cb2741c3c..e70810853a 100644
--- a/packages/ai/tests/stream-generation.test.ts
+++ b/packages/ai/tests/stream-generation.test.ts
@@ -172,6 +172,7 @@ describe('generateVideo({ stream: true })', () => {
availableDurations: () => ({ kind: 'none' as const }),
snapDuration: () => undefined,
+ supportedEditKind: () => undefined,
createVideoJob: vi.fn(async () => ({
jobId: 'job-123',
diff --git a/testing/e2e/global-setup.ts b/testing/e2e/global-setup.ts
index f2f4d639c5..317981294d 100644
--- a/testing/e2e/global-setup.ts
+++ b/testing/e2e/global-setup.ts
@@ -96,6 +96,13 @@ export default async function globalSetup() {
// text tests.
mock.mount('/omni-video', geminiOmniVideoMount())
+ // Sora follow-up edits (`POST /v1/videos/{id}/remix`). aimock 1.29's native
+ // /v1/videos pipeline covers create/retrieve but not the remix endpoint,
+ // so this fall-through mount answers remix creates (and the poll of the
+ // remix job id) and returns false for everything else, which drops through
+ // to aimock's native handler.
+ mock.mount('/v1/videos', openaiVideoRemixMount())
+
// Anthropic server_tool_use bug reproduction (issue #604). aimock can't
// natively synthesize `server_tool_use` / `web_fetch_tool_result` content
// blocks, so this mount hand-crafts the raw SSE Claude would emit when a
@@ -801,9 +808,13 @@ function mistralEmbeddingsMount(): Mountable {
*/
function geminiOmniVideoMount(): Mountable {
const JOB_ID = 'v1_omni-video-e2e'
- // Minimal MP4-ish base64 payload — the spec only asserts the
- // element renders with the data: URL the adapter builds from it.
+ const EDIT_JOB_ID = 'v1_omni-video-edit-e2e'
+ // Minimal MP4-ish base64 payloads — the specs only assert the
+ // element renders with the data: URL the adapter builds from them. The
+ // edit payload differs from the create payload so the video-edit spec can
+ // prove the follow-up job produced a different clip.
const VIDEO_BASE64 = 'AAAAIGZ0eXBpc29tAAACAGlzb21pc28y'
+ const EDITED_VIDEO_BASE64 = 'AAAAIGZ0eXBpc29tAAACAGVkaXRlZA=='
return {
async handleRequest(
req: http.IncomingMessage,
@@ -814,12 +825,21 @@ function geminiOmniVideoMount(): Mountable {
pathname: string,
): Promise {
if (pathname === '/v1beta/interactions' && req.method === 'POST') {
- await drainBody(req)
+ const body = await readBody(req)
+ let isEdit = false
+ try {
+ const parsed = JSON.parse(body) as {
+ previous_interaction_id?: string
+ }
+ isEdit = typeof parsed.previous_interaction_id === 'string'
+ } catch {
+ // non-JSON body — treat as a plain create
+ }
res.statusCode = 200
res.setHeader('Content-Type', 'application/json')
res.end(
JSON.stringify({
- id: JOB_ID,
+ id: isEdit ? EDIT_JOB_ID : JOB_ID,
object: 'interaction',
status: 'in_progress',
model: 'gemini-omni-flash-preview',
@@ -830,6 +850,7 @@ function geminiOmniVideoMount(): Mountable {
const pollMatch = pathname.match(/^\/v1beta\/interactions\/([^/]+)$/)
if (pollMatch && req.method === 'GET') {
+ const isEdit = pollMatch[1] === EDIT_JOB_ID
res.statusCode = 200
res.setHeader('Content-Type', 'application/json')
res.end(
@@ -855,7 +876,7 @@ function geminiOmniVideoMount(): Mountable {
{
type: 'video',
mime_type: 'video/mp4',
- data: VIDEO_BASE64,
+ data: isEdit ? EDITED_VIDEO_BASE64 : VIDEO_BASE64,
},
],
},
@@ -870,6 +891,57 @@ function geminiOmniVideoMount(): Mountable {
}
}
+/**
+ * Sora follow-up edits (`POST /v1/videos/{id}/remix`). Returns false for
+ * create/retrieve so aimock's native videos handler still serves those.
+ */
+function openaiVideoRemixMount(): Mountable {
+ const REMIX_JOB_ID = 'video-job-remix-e2e'
+ const REMIX_URL = 'https://example.com/guitar-store-remixed.mp4'
+ return {
+ async handleRequest(
+ req: http.IncomingMessage,
+ res: http.ServerResponse,
+ pathname: string,
+ ): Promise {
+ const remixMatch = pathname.match(/^\/([^/]+)\/remix$/)
+ if (remixMatch && req.method === 'POST') {
+ await drainBody(req)
+ res.statusCode = 200
+ res.setHeader('Content-Type', 'application/json')
+ res.end(
+ JSON.stringify({
+ id: REMIX_JOB_ID,
+ object: 'video',
+ status: 'completed',
+ model: 'sora-2',
+ remixed_from_video_id: remixMatch[1],
+ url: REMIX_URL,
+ }),
+ )
+ return true
+ }
+
+ if (pathname === `/${REMIX_JOB_ID}` && req.method === 'GET') {
+ res.statusCode = 200
+ res.setHeader('Content-Type', 'application/json')
+ res.end(
+ JSON.stringify({
+ id: REMIX_JOB_ID,
+ object: 'video',
+ status: 'completed',
+ model: 'sora-2',
+ url: REMIX_URL,
+ }),
+ )
+ return true
+ }
+
+ return false
+ },
+ }
+}
+
/**
* Fake audio payloads keyed by Seed Speech's `audio_config.format`.
*
diff --git a/testing/e2e/src/components/ImageGenUI.tsx b/testing/e2e/src/components/ImageGenUI.tsx
index 93cec8b7ce..1a1b1352aa 100644
--- a/testing/e2e/src/components/ImageGenUI.tsx
+++ b/testing/e2e/src/components/ImageGenUI.tsx
@@ -15,6 +15,8 @@ interface ImageGenUIProps {
aimockPort?: number
/** Show a file input and send the prompt as multimodal parts (image-to-image). */
withImageInput?: boolean
+ /** Show an edit box on the generated image (follow-up edit via previousImage). */
+ withEditInput?: boolean
}
function fileToBase64(file: File): Promise {
@@ -39,8 +41,10 @@ export function ImageGenUI({
testId,
aimockPort,
withImageInput,
+ withEditInput,
}: ImageGenUIProps) {
const [prompt, setPrompt] = useState('')
+ const [editPrompt, setEditPrompt] = useState('')
const [imageFile, setImageFile] = useState(null)
const connectionOptions = () => {
@@ -53,7 +57,10 @@ export function ImageGenUI({
return { connection: fetchHttpStream('/api/image/stream'), body }
}
return {
- fetcher: async (input: { prompt: MediaPrompt }) => {
+ fetcher: async (input: {
+ prompt: MediaPrompt
+ previousImage?: { url?: string; b64Json?: string }
+ }) => {
return generateImageFn({
data: {
prompt: input.prompt,
@@ -61,6 +68,9 @@ export function ImageGenUI({
numberOfImages: 1,
aimockPort,
testId,
+ ...(input.previousImage
+ ? { previousImage: input.previousImage }
+ : {}),
},
}) as Promise
},
@@ -70,6 +80,19 @@ export function ImageGenUI({
const { generate, result, isLoading, error, status } =
useGenerateImage(connectionOptions())
+ // Follow-up edit: the previously generated image rides previousImage and the
+ // server prepends it to the prompt as an image input (adapter edit path).
+ const handleEdit = async () => {
+ const image = result?.images[0]
+ if (!image) return
+ await generate({
+ prompt: editPrompt,
+ previousImage: image.url
+ ? { url: image.url }
+ : { b64Json: image.b64Json },
+ })
+ }
+
const handleGenerate = async () => {
if (!imageFile) {
await generate({ prompt })
@@ -145,6 +168,26 @@ export function ImageGenUI({
))}
)}
+ {withEditInput && result && result.images.length > 0 && (
+