Skip to content

Commit 24bb871

Browse files
committed
fix: pass image as video input for lipsync models (sync-v2-pro requires video_url)
1 parent edb707e commit 24bb871

2 files changed

Lines changed: 9 additions & 3 deletions

File tree

src/react/renderers/talking-head.ts

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,16 @@ export async function renderTalkingHead(
5454
const characterImageData = await characterFile.arrayBuffer();
5555
const speechAudioData = await speechFile.arrayBuffer();
5656

57-
// Create a synthetic video element for the lipsync generation
57+
// Create a synthetic video element for the lipsync generation.
58+
// Lipsync models (sync-v2-pro, etc.) require `video_url`, not `image_url`,
59+
// so we pass the character image as the `video` input. The fal provider will
60+
// upload it and set `video_url` in the API request. Fal.ai accepts image
61+
// files as the video input for lipsync — it treats them as single-frame video.
5862
const videoElement: VargElement<"video"> = {
5963
type: "video",
6064
props: {
6165
prompt: {
62-
images: [characterImageData],
66+
video: characterImageData,
6367
audio: speechAudioData,
6468
},
6569
model: lipsyncModel,

src/react/resolve.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -808,10 +808,12 @@ export async function resolveTalkingHeadElement(
808808
const lipsyncModel = props.lipsyncModel ?? model;
809809
const generateVideo = getCachedGenerateVideo();
810810

811+
// Lipsync models require `video_url`, not `image_url`, so pass the
812+
// character image as the `video` input (fal accepts images as video input).
811813
const { video } = await generateVideo({
812814
model: lipsyncModel as Parameters<typeof generateVideoRaw>[0]["model"],
813815
prompt: {
814-
images: [characterBytes],
816+
video: characterBytes,
815817
audio: speechBytes,
816818
},
817819
duration: 0, // duration determined by audio length

0 commit comments

Comments
 (0)