feat(js/plugins/google-genai): Added videoMetadata for vertexai (#3660)

Author: ifielker

js/plugins/google-genai/src/common/converters.ts

@@ -30,6 +30,7 @@ import {
   Part as GeminiPart,
   Schema,
   SchemaType,
+  VideoMetadata,
 } from './types.js';
 
 export function toGeminiTool(tool: ToolDefinition): FunctionDeclaration {
@@ -100,28 +101,37 @@ function toGeminiSchemaProperty(property?: ToolDefinition['inputSchema']) {
 }
 
 function toGeminiMedia(part: Part): GeminiPart {
+  let media: GeminiPart;
   if (part.media?.url.startsWith('data:')) {
     // Inline data
     const dataUrl = part.media.url;
     const b64Data = dataUrl.substring(dataUrl.indexOf(',')! + 1);
     const contentType =
       part.media.contentType ||
       dataUrl.substring(dataUrl.indexOf(':')! + 1, dataUrl.indexOf(';'));
-    return { inlineData: { mimeType: contentType, data: b64Data } };
+    media = { inlineData: { mimeType: contentType, data: b64Data } };
+  } else {
+    // File data
+    if (!part.media?.contentType) {
+      throw Error(
+        'Must supply a `contentType` when sending File URIs to Gemini.'
+      );
+    }
+    media = {
+      fileData: {
+        mimeType: part.media.contentType,
+        fileUri: part.media.url,
+      },
+    };
   }
 
-  // File data
-  if (!part.media?.contentType) {
-    throw Error(
-      'Must supply a `contentType` when sending File URIs to Gemini.'
-    );
+  // Video metadata
+  if (part.metadata?.videoMetadata) {
+    let videoMetadata = part.metadata.videoMetadata as VideoMetadata;
+    media.videoMetadata = { ...videoMetadata };
   }
-  return {
-    fileData: {
-      mimeType: part.media.contentType,
-      fileUri: part.media.url,
-    },
-  };
+
+  return media;
 }
 
 function toGeminiToolRequest(part: Part): GeminiPart {
@@ -195,6 +205,7 @@ function toGeminiPart(part: Part): GeminiPart {
   if (part.custom) {
     return toGeminiCustom(part);
   }
+
   throw new Error('Unsupported Part type ' + JSON.stringify(part));
 }
 

js/plugins/google-genai/src/common/types.ts

@@ -637,6 +637,27 @@ export declare interface CodeExecutionResult {
   output: string;
 }
 
+/**
+ * Can be added in the same part as video media to specify
+ * which part of the video to consider and how many frames
+ * per second to analyze. VertexAI only.
+ */
+export declare interface VideoMetadata {
+  /**
+   * The video offset to start at. e.g. '3.5s'
+   */
+  startOffset?: string;
+  /**
+   * The video offset to end at e.g. '10.5s'
+   */
+  endOffset?: string;
+  /**
+   * The number of frames to consider per second
+   * 0.0 to 24.0.
+   */
+  fps?: number;
+}
+
 /**
  * This is a Gemini Part. (Users never see this
  * structure, it is just built by the converters.)
@@ -651,6 +672,7 @@ export declare interface Part {
   thoughtSignature?: string;
   executableCode?: ExecutableCode;
   codeExecutionResult?: CodeExecutionResult;
+  videoMetadata?: VideoMetadata;
 }
 
 /**

js/plugins/google-genai/tests/common/converters_test.ts

@@ -162,6 +162,84 @@ describe('toGeminiMessage', () => {
         ],
       },
     },
+    {
+      should:
+        'should transform genkit message (fileData video content with metadata) correctly',
+      inputMessage: {
+        role: 'user',
+        content: [
+          { text: 'describe the following video:' },
+          {
+            media: {
+              contentType: 'video/mp4',
+              url: 'gs://bucket/video.mp4',
+            },
+            metadata: {
+              videoMetadata: {
+                startOffset: '10.0s',
+                endOffset: '20.5s',
+                fps: 0.5,
+              },
+            },
+          },
+        ],
+      },
+      expectedOutput: {
+        role: 'user',
+        parts: [
+          { text: 'describe the following video:' },
+          {
+            fileData: {
+              mimeType: 'video/mp4',
+              fileUri: 'gs://bucket/video.mp4',
+            },
+            videoMetadata: {
+              startOffset: '10.0s',
+              endOffset: '20.5s',
+              fps: 0.5,
+            },
+          },
+        ],
+      },
+    },
+    {
+      should:
+        'should transform genkit message (fileData video content with partial metadata) correctly',
+      inputMessage: {
+        role: 'user',
+        content: [
+          { text: 'describe the following video:' },
+          {
+            media: {
+              contentType: 'video/mp4',
+              url: 'gs://bucket/video.mp4',
+            },
+            metadata: {
+              videoMetadata: {
+                startOffset: '5.3s',
+                endOffset: '15.7s',
+              },
+            },
+          },
+        ],
+      },
+      expectedOutput: {
+        role: 'user',
+        parts: [
+          { text: 'describe the following video:' },
+          {
+            fileData: {
+              mimeType: 'video/mp4',
+              fileUri: 'gs://bucket/video.mp4',
+            },
+            videoMetadata: {
+              startOffset: '5.3s',
+              endOffset: '15.7s',
+            },
+          },
+        ],
+      },
+    },
     {
       should: 'should re-populate thoughtSignature from reasoning metadata',
       inputMessage: {

js/testapps/basic-gemini/src/index-vertexai.ts

@@ -69,13 +69,54 @@ ai.defineFlow('youtube-videos', async (_, { sendChunk }) => {
           url: 'https://www.youtube.com/watch?v=3p1P5grjXIQ',
           contentType: 'video/mp4',
         },
+        // Metadata is optional. You can leave it out if you
+        // want the whole video at default fps.
+        metadata: {
+          videoMetadata: {
+            fps: 0.5,
+            startOffset: '3.5s',
+            endOffset: '10.2s',
+          },
+        },
       },
     ],
   });
 
   return text;
 });
 
+export const videoUnderstanding = ai.defineFlow(
+  {
+    name: 'video-understanding-metadata',
+    inputSchema: z.void(),
+    outputSchema: z.any(),
+  },
+  async () => {
+    const llmResponse = await ai.generate({
+      model: vertexAI.model('gemini-2.5-flash'),
+      prompt: [
+        {
+          media: {
+            url: 'gs://cloud-samples-data/video/animals.mp4',
+            contentType: 'video/mp4',
+          },
+          metadata: {
+            videoMetadata: {
+              fps: 0.5,
+              startOffset: '3.5s',
+              endOffset: '10.2s',
+            },
+          },
+        },
+        {
+          text: 'describe this video',
+        },
+      ],
+    });
+    return llmResponse.text;
+  }
+);
+
 // streaming
 ai.defineFlow('streaming', async (_, { sendChunk }) => {
   const { stream } = ai.generateStream({