diff --git a/packages/sdk/index.html b/packages/sdk/index.html index 58e86d2..fb604bc 100644 --- a/packages/sdk/index.html +++ b/packages/sdk/index.html @@ -242,6 +242,7 @@

Configuration

+ @@ -321,6 +322,12 @@

Image Reference (Style Transfer)

+
+ + +
@@ -380,6 +387,39 @@

Console Logs

let decartClient = null; let decartRealtime = null; let localStream = null; + + // Captures a low-res JPEG snapshot from the local camera stream. + // Mirrors the pattern used in tryonv1 so the bouncer's prompt enhancer can see what the person is currently wearing. + async function captureReferenceFrame() { + if (!localStream) return null; + const video = document.createElement('video'); + video.srcObject = localStream; + video.muted = true; + video.playsInline = true; + await video.play(); + await new Promise((resolve) => { + if ('requestVideoFrameCallback' in video) { + video.requestVideoFrameCallback(() => resolve()); + } else { + requestAnimationFrame(() => requestAnimationFrame(() => resolve())); + } + }); + if (!video.videoWidth) { + video.pause(); + video.srcObject = null; + return null; + } + const scale = 320 / video.videoWidth; + const canvas = document.createElement('canvas'); + canvas.width = 320; + canvas.height = Math.round(video.videoHeight * scale); + canvas.getContext('2d').drawImage(video, 0, 0, canvas.width, canvas.height); + video.pause(); + video.srcObject = null; + return new Promise((resolve) => { + canvas.toBlob((blob) => resolve(blob), 'image/jpeg', 0.5); + }); + } let isConnected = false; let processedVideoUrl = null; let currentSessionId = null; @@ -412,6 +452,7 @@

Console Logs

logsContainer: document.getElementById('logs-container'), // Image reference elements referenceImage: document.getElementById('reference-image'), + includeReferenceFrame: document.getElementById('include-reference-frame'), setImage: document.getElementById('send-image'), imageStatus: document.getElementById('image-status'), imageStatusText: document.getElementById('image-status-text'), @@ -757,7 +798,17 @@

Console Logs

addLog('Sending reference image...', 'info'); - await decartRealtime.setImage(file); + let sampleFrameData = null; + if (elements.includeReferenceFrame.checked) { + sampleFrameData = await captureReferenceFrame(); + if (sampleFrameData) { + addLog(`Captured reference frame from camera (${(sampleFrameData.size / 1024).toFixed(1)} KB)`, 'info'); + } else { + addLog('Camera stream not available — skipping reference frame', 'warning'); + } + } + + await decartRealtime.setImage(file, sampleFrameData ? { sampleFrameData } : undefined); elements.imageStatusText.textContent = '✅ Image sent successfully!'; elements.imageStatusText.style.color = '#4CAF50'; diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index 8327a37..3d46fa6 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -117,7 +117,12 @@ export type RealTimeClient = { subscribeToken: string | null; setImage: ( image: Blob | File | string | null, - options?: { prompt?: string; enhance?: boolean; timeout?: number }, + options?: { + prompt?: string; + enhance?: boolean; + timeout?: number; + sampleFrameData?: Blob | File | string | null; + }, ) => Promise; }; @@ -332,13 +337,31 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { }, setImage: async ( image: Blob | File | string | null, - options?: { prompt?: string; enhance?: boolean; timeout?: number }, + options?: { + prompt?: string; + enhance?: boolean; + timeout?: number; + sampleFrameData?: Blob | File | string | null; + }, ) => { + const { sampleFrameData, ...rest } = options ?? {}; + const managerOptions: { + prompt?: string; + enhance?: boolean; + timeout?: number; + sampleFrameDataBase64?: string | null; + } = rest; + + if (sampleFrameData !== undefined) { + managerOptions.sampleFrameDataBase64 = + sampleFrameData === null ? null : await imageToBase64(sampleFrameData); + } + if (image === null) { - return manager.setImage(null, options); + return manager.setImage(null, managerOptions); } const base64 = await imageToBase64(image); - return manager.setImage(base64, options); + return manager.setImage(base64, managerOptions); }, }; diff --git a/packages/sdk/src/realtime/methods.ts b/packages/sdk/src/realtime/methods.ts index 6755d41..bac6df2 100644 --- a/packages/sdk/src/realtime/methods.ts +++ b/packages/sdk/src/realtime/methods.ts @@ -10,6 +10,7 @@ const setInputSchema = z prompt: z.string().min(1).optional(), enhance: z.boolean().optional().default(true), image: z.union([z.instanceof(Blob), z.instanceof(File), z.string(), z.null()]).optional(), + sampleFrameData: z.union([z.instanceof(Blob), z.instanceof(File), z.string(), z.null()]).optional(), }) .refine((data) => data.prompt !== undefined || data.image !== undefined, { message: "At least one of 'prompt' or 'image' must be provided", @@ -41,14 +42,25 @@ export const realtimeMethods = ( throw parsed.error; } - const { prompt, enhance, image } = parsed.data; + const { prompt, enhance, image, sampleFrameData } = parsed.data; let imageBase64: string | null = null; if (image !== undefined && image !== null) { imageBase64 = await imageToBase64(image); } - await webrtcManager.setImage(imageBase64, { prompt, enhance, timeout: UPDATE_TIMEOUT_MS }); + const options: { + prompt?: string; + enhance: boolean; + timeout: number; + sampleFrameDataBase64?: string | null; + } = { prompt, enhance, timeout: UPDATE_TIMEOUT_MS }; + + if (sampleFrameData !== undefined) { + options.sampleFrameDataBase64 = sampleFrameData === null ? null : await imageToBase64(sampleFrameData); + } + + await webrtcManager.setImage(imageBase64, options); }; const setPrompt = async (prompt: string, { enhance }: { enhance?: boolean } = {}): Promise => { diff --git a/packages/sdk/src/realtime/types.ts b/packages/sdk/src/realtime/types.ts index d6fe21d..f92488d 100644 --- a/packages/sdk/src/realtime/types.ts +++ b/packages/sdk/src/realtime/types.ts @@ -40,6 +40,7 @@ export type SetAvatarImageMessage = { image_data: string | null; // Base64-encoded image data, or null to clear/use placeholder prompt?: string | null; // Optional prompt to send with the image, null for passthrough enhance_prompt?: boolean; // Optional flag to enhance the prompt + sample_frame_data?: string | null; // Optional base64-encoded sample frame (e.g. current camera frame) to give prompt enhancement extra context }; export type SetImageAckMessage = { diff --git a/packages/sdk/src/realtime/webrtc-connection.ts b/packages/sdk/src/realtime/webrtc-connection.ts index 6e59a86..a974e7c 100644 --- a/packages/sdk/src/realtime/webrtc-connection.ts +++ b/packages/sdk/src/realtime/webrtc-connection.ts @@ -311,7 +311,12 @@ export class WebRTCConnection { async setImageBase64( imageBase64: string | null, - options?: { prompt?: string | null; enhance?: boolean; timeout?: number }, + options?: { + prompt?: string | null; + enhance?: boolean; + timeout?: number; + sampleFrameDataBase64?: string | null; + }, ): Promise { return new Promise((resolve, reject) => { const timeoutId = setTimeout(() => { @@ -336,6 +341,7 @@ export class WebRTCConnection { image_data: string | null; prompt?: string | null; enhance_prompt?: boolean; + sample_frame_data?: string | null; } = { type: "set_image", image_data: imageBase64, @@ -347,6 +353,9 @@ export class WebRTCConnection { if (options?.enhance !== undefined) { message.enhance_prompt = options.enhance; } + if (options?.sampleFrameDataBase64 !== undefined) { + message.sample_frame_data = options.sampleFrameDataBase64; + } if (!this.send(message)) { clearTimeout(timeoutId); diff --git a/packages/sdk/src/realtime/webrtc-manager.ts b/packages/sdk/src/realtime/webrtc-manager.ts index 174c897..7e20d62 100644 --- a/packages/sdk/src/realtime/webrtc-manager.ts +++ b/packages/sdk/src/realtime/webrtc-manager.ts @@ -244,7 +244,12 @@ export class WebRTCManager { setImage( imageBase64: string | null, - options?: { prompt?: string; enhance?: boolean; timeout?: number }, + options?: { + prompt?: string; + enhance?: boolean; + timeout?: number; + sampleFrameDataBase64?: string | null; + }, ): Promise { return this.connection.setImageBase64(imageBase64, options); } diff --git a/packages/sdk/src/shared/model.ts b/packages/sdk/src/shared/model.ts index 678896c..e6179fb 100644 --- a/packages/sdk/src/shared/model.ts +++ b/packages/sdk/src/shared/model.ts @@ -3,6 +3,7 @@ import { createModelNotFoundError } from "../utils/errors"; const CANONICAL_MODEL_NAMES = [ "lucy-2.1", + "lucy-2.5", "lucy-2.1-vton", "lucy-vton-2", "lucy-restyle-2", @@ -10,7 +11,13 @@ const CANONICAL_MODEL_NAMES = [ "lucy-image-2", ] as const; -const CANONICAL_REALTIME_MODEL_NAMES = ["lucy-2.1", "lucy-2.1-vton", "lucy-vton-2", "lucy-restyle-2"] as const; +const CANONICAL_REALTIME_MODEL_NAMES = [ + "lucy-2.1", + "lucy-2.5", + "lucy-2.1-vton", + "lucy-vton-2", + "lucy-restyle-2", +] as const; const CANONICAL_VIDEO_MODEL_NAMES = [ "lucy-clip", "lucy-2.1", @@ -59,6 +66,7 @@ function warnDeprecated(model: string): void { export const realtimeModels = z.union([ // Canonical names z.literal("lucy-2.1"), + z.literal("lucy-2.5"), z.literal("lucy-2.1-vton"), z.literal("lucy-vton-2"), z.literal("lucy-restyle-2"), @@ -255,6 +263,7 @@ export const modelInputSchemas = { "lucy-image-2": imageEditSchema, "lucy-restyle-2": restyleSchema, "lucy-2.1": videoEdit2Schema, + "lucy-2.5": videoEdit2Schema, "lucy-2.1-vton": videoEdit2Schema, "lucy-vton-2": videoEdit2Schema, // Latest aliases (server-side resolution) @@ -327,6 +336,14 @@ const _models = { height: 624, inputSchema: z.object({}), }, + "lucy-2.5": { + urlPath: "/v1/stream", + name: "lucy-2.5" as const, + fps: 20, + width: 1088, + height: 624, + inputSchema: z.object({}), + }, "lucy-2.1-vton": { urlPath: "/v1/stream", name: "lucy-2.1-vton" as const, @@ -594,6 +611,7 @@ export const models = { * * Available options: * - `"lucy-2.1"` - Lucy 2.1 realtime video editing + * - `"lucy-2.5"` - Lucy 2.5 realtime video editing * - `"lucy-2.1-vton"` - Lucy 2.1 virtual try-on * - `"lucy-vton-2"` - Lucy virtual try-on 2 * - `"lucy-restyle-2"` - Realtime video restyling diff --git a/packages/sdk/tests/unit.test.ts b/packages/sdk/tests/unit.test.ts index c937fd3..710dd09 100644 --- a/packages/sdk/tests/unit.test.ts +++ b/packages/sdk/tests/unit.test.ts @@ -1257,6 +1257,76 @@ describe("WebRTCConnection", () => { vi.useRealTimers(); } }); + + it("includes sample_frame_data in the outgoing set_image message when provided", async () => { + vi.useFakeTimers(); + try { + const { WebRTCConnection } = await import("../src/realtime/webrtc-connection.js"); + const connection = new WebRTCConnection(); + const sendSpy = vi.spyOn(connection, "send").mockReturnValue(true); + + const promise = connection.setImageBase64("imgbase64", { sampleFrameDataBase64: "refbase64" }).catch(() => {}); + + expect(sendSpy).toHaveBeenCalledWith({ + type: "set_image", + image_data: "imgbase64", + sample_frame_data: "refbase64", + }); + + await vi.advanceTimersByTimeAsync(30001); + await promise; + sendSpy.mockRestore(); + } finally { + vi.useRealTimers(); + } + }); + + it("omits sample_frame_data when not provided", async () => { + vi.useFakeTimers(); + try { + const { WebRTCConnection } = await import("../src/realtime/webrtc-connection.js"); + const connection = new WebRTCConnection(); + const sendSpy = vi.spyOn(connection, "send").mockReturnValue(true); + + const promise = connection.setImageBase64("imgbase64").catch(() => {}); + + const sentMessage = sendSpy.mock.calls[0]?.[0] as Record; + expect(sentMessage).toEqual({ + type: "set_image", + image_data: "imgbase64", + }); + expect("sample_frame_data" in sentMessage).toBe(false); + + await vi.advanceTimersByTimeAsync(30001); + await promise; + sendSpy.mockRestore(); + } finally { + vi.useRealTimers(); + } + }); + + it("forwards explicit null sample_frame_data", async () => { + vi.useFakeTimers(); + try { + const { WebRTCConnection } = await import("../src/realtime/webrtc-connection.js"); + const connection = new WebRTCConnection(); + const sendSpy = vi.spyOn(connection, "send").mockReturnValue(true); + + const promise = connection.setImageBase64("imgbase64", { sampleFrameDataBase64: null }).catch(() => {}); + + expect(sendSpy).toHaveBeenCalledWith({ + type: "set_image", + image_data: "imgbase64", + sample_frame_data: null, + }); + + await vi.advanceTimersByTimeAsync(30001); + await promise; + sendSpy.mockRestore(); + } finally { + vi.useRealTimers(); + } + }); }); describe("setupNewPeerConnection", () => { @@ -1470,6 +1540,56 @@ describe("set()", () => { timeout: 30000, }); }); + + it("converts sampleFrameData Blob to base64 and forwards it", async () => { + const refBlob = new Blob(["ref-frame"], { type: "image/jpeg" }); + mockImageToBase64.mockImplementation(async (input) => (input === refBlob ? "refbase64" : "imgbase64")); + + await methods.set({ prompt: "a cat", image: "rawimg", sampleFrameData: refBlob }); + + expect(mockImageToBase64).toHaveBeenCalledWith(refBlob); + expect(mockManager.setImage).toHaveBeenCalledWith("imgbase64", { + prompt: "a cat", + enhance: true, + timeout: 30000, + sampleFrameDataBase64: "refbase64", + }); + }); + + it("forwards a base64 string sampleFrameData through imageToBase64", async () => { + mockImageToBase64.mockImplementation(async (input) => (input === "raw-ref" ? "refbase64" : "imgbase64")); + + await methods.set({ image: "raw-img", sampleFrameData: "raw-ref" }); + + expect(mockImageToBase64).toHaveBeenCalledWith("raw-ref"); + expect(mockManager.setImage).toHaveBeenCalledWith("imgbase64", { + prompt: undefined, + enhance: true, + timeout: 30000, + sampleFrameDataBase64: "refbase64", + }); + }); + + it("forwards null sampleFrameData without calling imageToBase64", async () => { + mockImageToBase64.mockResolvedValue("imgbase64"); + + await methods.set({ image: "raw-img", sampleFrameData: null }); + + expect(mockImageToBase64).toHaveBeenCalledTimes(1); + expect(mockImageToBase64).toHaveBeenCalledWith("raw-img"); + expect(mockManager.setImage).toHaveBeenCalledWith("imgbase64", { + prompt: undefined, + enhance: true, + timeout: 30000, + sampleFrameDataBase64: null, + }); + }); + + it("omits sampleFrameDataBase64 when sampleFrameData is not provided", async () => { + await methods.set({ prompt: "a cat" }); + const call = mockManager.setImage.mock.calls[0]?.[1] as Record; + expect("sampleFrameDataBase64" in call).toBe(false); + }); }); describe("Subscribe Token", () => { @@ -3314,7 +3434,13 @@ describe("Canonical Model Names", () => { expect(canonicalModelSchema.safeParse(alias).success).toBe(false); } - expect(canonicalRealtimeModels.options).toEqual(["lucy-2.1", "lucy-2.1-vton", "lucy-vton-2", "lucy-restyle-2"]); + expect(canonicalRealtimeModels.options).toEqual([ + "lucy-2.1", + "lucy-2.5", + "lucy-2.1-vton", + "lucy-vton-2", + "lucy-restyle-2", + ]); expect(canonicalVideoModels.options).toEqual([ "lucy-clip", "lucy-2.1", @@ -3380,7 +3506,7 @@ describe("Canonical Model Names", () => { it("lists all models when called without options", () => { const listedModels = listModels(); - expect(listedModels).toHaveLength(26); + expect(listedModels).toHaveLength(27); expect(listedModels.some((model) => model.kind === "realtime" && model.name === "lucy-2.1")).toBe(true); expect(listedModels.some((model) => model.kind === "video" && model.name === "lucy-clip")).toBe(true); expect(listedModels.some((model) => model.kind === "image" && model.name === "lucy-image-2")).toBe(true); @@ -3446,6 +3572,15 @@ describe("Canonical Model Names", () => { expect(model.height).toBe(624); }); + it("lucy-2.5 canonical name works", () => { + const model = models.realtime("lucy-2.5"); + expect(model.name).toBe("lucy-2.5"); + expect(model.urlPath).toBe("/v1/stream"); + expect(model.fps).toBe(20); + expect(model.width).toBe(1088); + expect(model.height).toBe(624); + }); + it("lucy-2.1-vton canonical name works", () => { const model = models.realtime("lucy-2.1-vton"); expect(model.name).toBe("lucy-2.1-vton");