feat(realtime): add optional referenceFrame to set/setImage

packages/sdk/index.html

@@ -242,6 +242,7 @@ <h3>Configuration</h3>
                 </optgroup>
                 <optgroup label="Canonical">
                     <option value="lucy-2.1" selected>Lucy 2.1</option>
+                    <option value="lucy-2.5">Lucy 2.5</option>
                     <option value="lucy-2.1-vton">Lucy 2.1 VTON</option>
                     <option value="lucy-vton-2">Lucy VTON 2</option>
                     <option value="lucy-restyle-2">Lucy Restyle 2</option>
@@ -321,6 +322,12 @@ <h3>Image Reference (Style Transfer)</h3>
         <div class="inline-controls">
             <button id="send-image" disabled>Send Reference Image</button>
         </div>
+        <div class="control-group" style="display: flex; align-items: center; gap: 8px; margin-top: 8px;">
+            <input type="checkbox" id="include-reference-frame">
+            <label for="include-reference-frame" style="margin: 0; font-weight: normal;">
+                Also capture &amp; attach camera reference frame
+            </label>
+        </div>
         <div id="image-status" style="margin-top: 10px; padding: 10px; background: #f8f9fa; border-radius: 6px; display: none;">
             <strong>Status:</strong> <span id="image-status-text">Ready</span>
         </div>
@@ -380,6 +387,39 @@ <h3>Console Logs</h3>
         let decartClient = null;
         let decartRealtime = null;
         let localStream = null;
+
+        // Captures a low-res JPEG snapshot from the local camera stream.
+        // Mirrors the pattern used in tryonv1 so the bouncer's prompt enhancer can see what the person is currently wearing.
+        async function captureReferenceFrame() {
+            if (!localStream) return null;
+            const video = document.createElement('video');
+            video.srcObject = localStream;
+            video.muted = true;
+            video.playsInline = true;
+            await video.play();
+            await new Promise((resolve) => {
+                if ('requestVideoFrameCallback' in video) {
+                    video.requestVideoFrameCallback(() => resolve());
+                } else {
+                    requestAnimationFrame(() => requestAnimationFrame(() => resolve()));
+                }
+            });
+            if (!video.videoWidth) {
+                video.pause();
+                video.srcObject = null;
+                return null;
+            }
+            const scale = 320 / video.videoWidth;
+            const canvas = document.createElement('canvas');
+            canvas.width = 320;
+            canvas.height = Math.round(video.videoHeight * scale);
+            canvas.getContext('2d').drawImage(video, 0, 0, canvas.width, canvas.height);
+            video.pause();
+            video.srcObject = null;
+            return new Promise((resolve) => {
+                canvas.toBlob((blob) => resolve(blob), 'image/jpeg', 0.5);
+            });
+        }
         let isConnected = false;
         let processedVideoUrl = null;
         let currentSessionId = null;
@@ -412,6 +452,7 @@ <h3>Console Logs</h3>
             logsContainer: document.getElementById('logs-container'),
             // Image reference elements
             referenceImage: document.getElementById('reference-image'),
+            includeReferenceFrame: document.getElementById('include-reference-frame'),
             setImage: document.getElementById('send-image'),
             imageStatus: document.getElementById('image-status'),
             imageStatusText: document.getElementById('image-status-text'),
@@ -757,7 +798,17 @@ <h3>Console Logs</h3>
 
                 addLog('Sending reference image...', 'info');
 
-                await decartRealtime.setImage(file);
+                let sampleFrameData = null;
+                if (elements.includeReferenceFrame.checked) {
+                    sampleFrameData = await captureReferenceFrame();
+                    if (sampleFrameData) {
+                        addLog(`Captured reference frame from camera (${(sampleFrameData.size / 1024).toFixed(1)} KB)`, 'info');
+                    } else {
+                        addLog('Camera stream not available — skipping reference frame', 'warning');
+                    }
+                }
+
+                await decartRealtime.setImage(file, sampleFrameData ? { sampleFrameData } : undefined);
 
                 elements.imageStatusText.textContent = '✅ Image sent successfully!';
                 elements.imageStatusText.style.color = '#4CAF50';

packages/sdk/src/realtime/client.ts

@@ -117,7 +117,12 @@ export type RealTimeClient = {
   subscribeToken: string | null;
   setImage: (
     image: Blob | File | string | null,
-    options?: { prompt?: string; enhance?: boolean; timeout?: number },
+    options?: {
+      prompt?: string;
+      enhance?: boolean;
+      timeout?: number;
+      sampleFrameData?: Blob | File | string | null;
+    },
   ) => Promise<void>;
 };
 
@@ -332,13 +337,31 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => {
         },
         setImage: async (
           image: Blob | File | string | null,
-          options?: { prompt?: string; enhance?: boolean; timeout?: number },
+          options?: {
+            prompt?: string;
+            enhance?: boolean;
+            timeout?: number;
+            sampleFrameData?: Blob | File | string | null;
+          },
         ) => {
+          const { sampleFrameData, ...rest } = options ?? {};
+          const managerOptions: {
+            prompt?: string;
+            enhance?: boolean;
+            timeout?: number;
+            sampleFrameDataBase64?: string | null;
+          } = rest;
+
+          if (sampleFrameData !== undefined) {
+            managerOptions.sampleFrameDataBase64 =
+              sampleFrameData === null ? null : await imageToBase64(sampleFrameData);
+          }
+
           if (image === null) {
-            return manager.setImage(null, options);
+            return manager.setImage(null, managerOptions);
           }
           const base64 = await imageToBase64(image);
-          return manager.setImage(base64, options);
+          return manager.setImage(base64, managerOptions);
         },
       };
 

packages/sdk/src/realtime/methods.ts

@@ -10,6 +10,7 @@ const setInputSchema = z
     prompt: z.string().min(1).optional(),
     enhance: z.boolean().optional().default(true),
     image: z.union([z.instanceof(Blob), z.instanceof(File), z.string(), z.null()]).optional(),
+    sampleFrameData: z.union([z.instanceof(Blob), z.instanceof(File), z.string(), z.null()]).optional(),
   })
   .refine((data) => data.prompt !== undefined || data.image !== undefined, {
     message: "At least one of 'prompt' or 'image' must be provided",
@@ -41,14 +42,25 @@ export const realtimeMethods = (
       throw parsed.error;
     }
 
-    const { prompt, enhance, image } = parsed.data;
+    const { prompt, enhance, image, sampleFrameData } = parsed.data;
 
     let imageBase64: string | null = null;
     if (image !== undefined && image !== null) {
       imageBase64 = await imageToBase64(image);
     }
 
-    await webrtcManager.setImage(imageBase64, { prompt, enhance, timeout: UPDATE_TIMEOUT_MS });
+    const options: {
+      prompt?: string;
+      enhance: boolean;
+      timeout: number;
+      sampleFrameDataBase64?: string | null;
+    } = { prompt, enhance, timeout: UPDATE_TIMEOUT_MS };
+
+    if (sampleFrameData !== undefined) {
+      options.sampleFrameDataBase64 = sampleFrameData === null ? null : await imageToBase64(sampleFrameData);
+    }
+
+    await webrtcManager.setImage(imageBase64, options);
   };
 
   const setPrompt = async (prompt: string, { enhance }: { enhance?: boolean } = {}): Promise<void> => {

packages/sdk/src/realtime/types.ts

@@ -40,6 +40,7 @@ export type SetAvatarImageMessage = {
   image_data: string | null; // Base64-encoded image data, or null to clear/use placeholder
   prompt?: string | null; // Optional prompt to send with the image, null for passthrough
   enhance_prompt?: boolean; // Optional flag to enhance the prompt
+  sample_frame_data?: string | null; // Optional base64-encoded sample frame (e.g. current camera frame) to give prompt enhancement extra context
 };
 
 export type SetImageAckMessage = {

packages/sdk/src/realtime/webrtc-connection.ts

@@ -311,7 +311,12 @@ export class WebRTCConnection {
 
   async setImageBase64(
     imageBase64: string | null,
-    options?: { prompt?: string | null; enhance?: boolean; timeout?: number },
+    options?: {
+      prompt?: string | null;
+      enhance?: boolean;
+      timeout?: number;
+      sampleFrameDataBase64?: string | null;
+    },
   ): Promise<void> {
     return new Promise((resolve, reject) => {
       const timeoutId = setTimeout(() => {
@@ -336,6 +341,7 @@ export class WebRTCConnection {
         image_data: string | null;
         prompt?: string | null;
         enhance_prompt?: boolean;
+        sample_frame_data?: string | null;
       } = {
         type: "set_image",
         image_data: imageBase64,
@@ -347,6 +353,9 @@ export class WebRTCConnection {
       if (options?.enhance !== undefined) {
         message.enhance_prompt = options.enhance;
       }
+      if (options?.sampleFrameDataBase64 !== undefined) {
+        message.sample_frame_data = options.sampleFrameDataBase64;
+      }
 
       if (!this.send(message)) {
         clearTimeout(timeoutId);

packages/sdk/src/realtime/webrtc-manager.ts

@@ -244,7 +244,12 @@ export class WebRTCManager {
 
   setImage(
     imageBase64: string | null,
-    options?: { prompt?: string; enhance?: boolean; timeout?: number },
+    options?: {
+      prompt?: string;
+      enhance?: boolean;
+      timeout?: number;
+      sampleFrameDataBase64?: string | null;
+    },
   ): Promise<void> {
     return this.connection.setImageBase64(imageBase64, options);
   }

packages/sdk/src/shared/model.ts

@@ -3,14 +3,21 @@ import { createModelNotFoundError } from "../utils/errors";
 
 const CANONICAL_MODEL_NAMES = [
   "lucy-2.1",
+  "lucy-2.5",
   "lucy-2.1-vton",
   "lucy-vton-2",
   "lucy-restyle-2",
   "lucy-clip",
   "lucy-image-2",
 ] as const;
 
-const CANONICAL_REALTIME_MODEL_NAMES = ["lucy-2.1", "lucy-2.1-vton", "lucy-vton-2", "lucy-restyle-2"] as const;
+const CANONICAL_REALTIME_MODEL_NAMES = [
+  "lucy-2.1",
+  "lucy-2.5",
+  "lucy-2.1-vton",
+  "lucy-vton-2",
+  "lucy-restyle-2",
+] as const;
 const CANONICAL_VIDEO_MODEL_NAMES = [
   "lucy-clip",
   "lucy-2.1",
@@ -59,6 +66,7 @@ function warnDeprecated(model: string): void {
 export const realtimeModels = z.union([
   // Canonical names
   z.literal("lucy-2.1"),
+  z.literal("lucy-2.5"),
   z.literal("lucy-2.1-vton"),
   z.literal("lucy-vton-2"),
   z.literal("lucy-restyle-2"),
@@ -255,6 +263,7 @@ export const modelInputSchemas = {
   "lucy-image-2": imageEditSchema,
   "lucy-restyle-2": restyleSchema,
   "lucy-2.1": videoEdit2Schema,
+  "lucy-2.5": videoEdit2Schema,
   "lucy-2.1-vton": videoEdit2Schema,
   "lucy-vton-2": videoEdit2Schema,
   // Latest aliases (server-side resolution)
@@ -327,6 +336,14 @@ const _models = {
       height: 624,
       inputSchema: z.object({}),
     },
+    "lucy-2.5": {
+      urlPath: "/v1/stream",
+      name: "lucy-2.5" as const,
+      fps: 20,
+      width: 1088,
+      height: 624,
+      inputSchema: z.object({}),
+    },
     "lucy-2.1-vton": {
       urlPath: "/v1/stream",
       name: "lucy-2.1-vton" as const,
@@ -594,6 +611,7 @@ export const models = {
    *
    * Available options:
    *   - `"lucy-2.1"` - Lucy 2.1 realtime video editing
+   *   - `"lucy-2.5"` - Lucy 2.5 realtime video editing
    *   - `"lucy-2.1-vton"` - Lucy 2.1 virtual try-on
    *   - `"lucy-vton-2"` - Lucy virtual try-on 2
    *   - `"lucy-restyle-2"` - Realtime video restyling