feat(js/plugins/google-genai): Added support for Google Maps (#3665)

Author: ifielker

js/plugins/google-genai/src/common/types.ts

@@ -878,6 +878,16 @@ export function isRetrievalTool(tool: Tool): tool is RetrievalTool {
   return (tool as RetrievalTool).retrieval !== undefined;
 }
 
+export declare interface GoogleMaps {
+  enableWidget: boolean;
+}
+export declare interface GoogleMapsTool {
+  googleMaps?: GoogleMaps;
+}
+export function isGoogleMapsTool(tool: Tool): tool is GoogleMapsTool {
+  return (tool as GoogleMapsTool).googleMaps !== undefined;
+}
+
 /**
  * Tool to retrieve public web data for grounding, powered by Google.
  */
@@ -893,6 +903,7 @@ export declare interface GoogleSearchRetrieval {
 export declare type Tool =
   | FunctionDeclarationsTool
   | RetrievalTool // Vertex AI Only
+  | GoogleMapsTool // Vertex AI Only
   | CodeExecutionTool // Google AI Only
   | GoogleSearchRetrievalTool;
 
@@ -973,10 +984,22 @@ export declare interface FunctionCallingConfig {
   allowedFunctionNames?: string[];
 }
 
+export declare interface LatLng {
+  latitude?: number;
+  longitude?: number;
+}
+
+export declare interface RetrievalConfig {
+  latLng?: LatLng;
+  languageCode?: string;
+}
+
 /** This config is shared for all tools provided in the request. */
 export declare interface ToolConfig {
   /** Function calling config. */
   functionCallingConfig?: FunctionCallingConfig;
+  /** Retrieval config */
+  retrievalConfig?: RetrievalConfig;
 }
 
 export declare interface GenerateContentRequest {

js/plugins/google-genai/src/vertexai/gemini.ts

@@ -255,6 +255,28 @@ export const GeminiConfigSchema = GenerationCommonConfigSchema.extend({
         'With NONE, the model is prohibited from making function calls.'
     )
     .optional(),
+  /**
+   * Retrieval config for search grounding and maps grounding
+   */
+  retrievalConfig: z
+    .object({
+      /**
+       * User location for search grounding or
+       * place location for maps grounding.
+       */
+      latLng: z
+        .object({
+          latitude: z.number().optional(),
+          longitude: z.number().optional(),
+        })
+        .describe('User location for Google search or Google maps grounding.')
+        .optional(),
+      /**
+       * Language code for the request. e.g. 'en-us'
+       */
+      languageCode: z.string().optional(),
+    })
+    .optional(),
   thinkingConfig: z
     .object({
       includeThoughts: z
@@ -466,6 +488,7 @@ export function defineModel(
       const {
         apiKey: apiKeyFromConfig,
         functionCallingConfig,
+        retrievalConfig,
         version: versionFromConfig,
         googleSearchRetrieval,
         tools: toolsFromConfig,
@@ -537,6 +560,13 @@ export function defineModel(
         };
       }
 
+      if (retrievalConfig) {
+        if (!toolConfig) {
+          toolConfig = {};
+        }
+        toolConfig.retrievalConfig = structuredClone(retrievalConfig);
+      }
+
       // Cannot use tools and function calling at the same time
       const jsonMode =
         (request.output?.format === 'json' || !!request.output?.schema) &&

js/plugins/google-genai/src/vertexai/types.ts

@@ -25,6 +25,8 @@ import {
   GenerateContentRequest,
   GenerateContentResponse,
   GenerateContentStreamResult,
+  GoogleMaps,
+  GoogleMapsTool,
   GoogleSearchRetrieval,
   GoogleSearchRetrievalTool,
   GroundingMetadata,
@@ -42,6 +44,7 @@ import {
   ToolConfig,
   isCodeExecutionTool,
   isFunctionDeclarationsTool,
+  isGoogleMapsTool,
   isGoogleSearchRetrievalTool,
   isObject,
   isRetrievalTool,
@@ -55,6 +58,7 @@ export {
   TaskTypeSchema,
   isCodeExecutionTool,
   isFunctionDeclarationsTool,
+  isGoogleMapsTool,
   isGoogleSearchRetrievalTool,
   isObject,
   isRetrievalTool,
@@ -66,6 +70,8 @@ export {
   type GenerateContentRequest,
   type GenerateContentResponse,
   type GenerateContentStreamResult,
+  type GoogleMaps,
+  type GoogleMapsTool,
   type GoogleSearchRetrieval,
   type GoogleSearchRetrievalTool,
   type GroundingMetadata,

js/plugins/google-genai/tests/vertexai/gemini_test.ts

@@ -34,6 +34,7 @@ import {
   HarmBlockThreshold,
   HarmCategory,
   isFunctionDeclarationsTool,
+  isGoogleMapsTool,
   isGoogleSearchRetrievalTool,
   isRetrievalTool,
 } from '../../src/vertexai/types.js';
@@ -347,6 +348,34 @@ describe('Vertex AI Gemini', () => {
         assert.deepStrictEqual(apiRequest.labels, myLabels);
       });
 
+      it('handles retrievalConfig', async () => {
+        mockFetchResponse(defaultApiResponse);
+        const request: GenerateRequest<typeof GeminiConfigSchema> = {
+          ...minimalRequest,
+          config: {
+            retrievalConfig: {
+              latLng: {
+                latitude: 37.7749,
+                longitude: -122.4194,
+              },
+              languageCode: 'en-US',
+            },
+          },
+        };
+        const model = defineModel('gemini-2.5-flash', clientOptions);
+        await model.run(request);
+        const apiRequest: GenerateContentRequest = JSON.parse(
+          fetchStub.lastCall.args[1].body
+        );
+        assert.deepStrictEqual(apiRequest.toolConfig?.retrievalConfig, {
+          latLng: {
+            latitude: 37.7749,
+            longitude: -122.4194,
+          },
+          languageCode: 'en-US',
+        });
+      });
+
       it('constructs tools array with functionDeclarations', async () => {
         mockFetchResponse(defaultApiResponse);
         const request: GenerateRequest<typeof GeminiConfigSchema> = {
@@ -402,6 +431,29 @@ describe('Vertex AI Gemini', () => {
         }
       });
 
+      it('handles googleMaps tool', async () => {
+        mockFetchResponse(defaultApiResponse);
+        const request: GenerateRequest<typeof GeminiConfigSchema> = {
+          ...minimalRequest,
+          config: {
+            tools: [{ googleMaps: { enableWidget: true } } as any],
+          },
+        };
+        const model = defineModel('gemini-2.5-flash', clientOptions);
+        await model.run(request);
+        const apiRequest: GenerateContentRequest = JSON.parse(
+          fetchStub.lastCall.args[1].body
+        );
+        const mapsTool = apiRequest.tools?.find(isGoogleMapsTool);
+        assert.ok(mapsTool, 'Expected GoogleMapsTool');
+        if (mapsTool) {
+          assert.ok(mapsTool.googleMaps, 'Expected googleMaps property');
+          assert.deepStrictEqual(mapsTool, {
+            googleMaps: { enableWidget: true },
+          });
+        }
+      });
+
       if (clientOptions.kind === 'regional') {
         it('handles vertexRetrieval tool', async () => {
           mockFetchResponse(defaultApiResponse);

js/testapps/basic-gemini/package.json

@@ -8,7 +8,7 @@
     "start": "node lib/index.js",
     "build": "tsc",
     "build:watch": "tsc --watch",
-    "genkit:dev": "genkit start -- npx tsx --watch src/index.ts"
+    "genkit:dev": "genkit start -- npx tsx --watch src/index-vertexai.ts"
   },
   "keywords": [],
   "author": "",

js/testapps/basic-gemini/src/index-vertexai.ts

@@ -133,6 +133,34 @@ ai.defineFlow('streaming', async (_, { sendChunk }) => {
   return poem;
 });
 
+// Google maps grounding
+ai.defineFlow('maps-grounding', async () => {
+  const { text, raw } = await ai.generate({
+    model: vertexAI.model('gemini-2.5-flash'),
+    prompt: 'Describe some sights near me',
+    config: {
+      tools: [
+        {
+          googleMaps: {
+            enableWidget: true,
+          },
+        },
+      ],
+      retrievalConfig: {
+        latLng: {
+          latitude: 43.0896,
+          longitude: -79.0849,
+        },
+      },
+    },
+  });
+
+  return {
+    text,
+    groundingMetadata: (raw as any)?.candidates[0]?.groundingMetadata,
+  };
+});
+
 // Search grounding
 ai.defineFlow('search-grounding', async () => {
   const { text, raw } = await ai.generate({