feat: throw for unknown CLI commands, traceId in eval:* (#3693)

Author: ssbushi

genkit-tools/cli/src/cli.ts

@@ -138,10 +138,11 @@ export async function startCLI(): Promise<void> {
       logger.info(program.help());
     })
   );
-  // Default action to catch unknown commands.
-  program.action(() => {
-    // print help
+  // Handle unknown commands.
+  program.on('command:*', (operands) => {
+    logger.error(`error: unknown command '${operands[0]}'`);
     logger.info(program.help());
+    process.exit(1);
   });
 
   await program.parseAsync();

genkit-tools/common/src/eval/evaluate.ts

@@ -193,6 +193,9 @@ export async function runEvaluation(params: {
       },
     });
     scores[name] = response.result;
+    logger.info(
+      `Finished evaluator '${action.name}'. Trace ID: ${response.telemetry?.traceId}`
+    );
   }
 
   const scoredResults = enrichResultsWithScoring(scores, evalDataset);

js/testapps/evals/src/genkit.ts

@@ -16,12 +16,7 @@
 
 import { devLocalVectorstore } from '@genkit-ai/dev-local-vectorstore';
 import { GenkitMetric, genkitEval } from '@genkit-ai/evaluator';
-import {
-  gemini15Flash,
-  gemini15Pro,
-  googleAI,
-  textEmbeddingGecko001,
-} from '@genkit-ai/googleai';
+import { googleAI } from '@genkit-ai/googleai';
 import { genkit } from 'genkit';
 import { langchain } from 'genkitx-langchain';
 
@@ -55,27 +50,27 @@ export const ai = genkit({
       metrics: [
         {
           type: GenkitMetric.MALICIOUSNESS,
-          judge: gemini15Pro,
+          judge: googleAI.model('gemini-2.5-pro'),
           judgeConfig: PERMISSIVE_SAFETY_SETTINGS,
         },
         {
           type: GenkitMetric.ANSWER_ACCURACY,
-          judge: gemini15Pro,
+          judge: googleAI.model('gemini-2.5-pro'),
           judgeConfig: PERMISSIVE_SAFETY_SETTINGS,
         },
       ],
     }),
     devLocalVectorstore([
       {
         indexName: 'pdfQA',
-        embedder: textEmbeddingGecko001,
+        embedder: googleAI.embedder('text-embedding-004'),
       },
     ]),
     langchain({
       evaluators: {
         criteria: ['coherence'],
         labeledCriteria: ['correctness'],
-        judge: gemini15Flash,
+        judge: googleAI.model('gemini-2.5-pro'),
       },
     }),
   ],

js/testapps/evals/src/pdf-rag.ts

@@ -18,7 +18,7 @@ import {
   devLocalIndexerRef,
   devLocalRetrieverRef,
 } from '@genkit-ai/dev-local-vectorstore';
-import { gemini15Flash } from '@genkit-ai/googleai';
+import googleAI from '@genkit-ai/googleai';
 import { z } from 'genkit';
 import { EvalStatusEnum, type BaseEvalDataPoint } from 'genkit/evaluator';
 import { Document } from 'genkit/retriever';
@@ -70,7 +70,7 @@ export const pdfQA = ai.defineFlow(
       context: docs.map((d) => d.text).join('\n\n'),
     });
     const llmResponse = await ai.generate({
-      model: gemini15Flash,
+      model: googleAI.model('gemini-2.5-flash'),
       prompt: augmentedPrompt,
     });
     return llmResponse.text;
@@ -86,7 +86,7 @@ export const simpleStructured = ai.defineFlow(
   },
   async (i) => {
     const llmResponse = await ai.generate({
-      model: gemini15Flash,
+      model: googleAI.model('gemini-2.5-flash'),
       prompt: i.query,
     });
     return { response: llmResponse.text };
@@ -102,7 +102,7 @@ export const simpleEcho = ai.defineFlow(
   },
   async (i) => {
     const llmResponse = await ai.generate({
-      model: gemini15Flash,
+      model: googleAI.model('gemini-2.5-flash'),
       prompt: i,
     });
     return llmResponse.text;