feat(cli): added dev:test-model cli command (#4064)

Author: pavelgj

bin/check_license

@@ -63,6 +63,7 @@ $HOME/go/bin/addlicense \
   -ignore '.trunk/**/*' \
   -ignore '**/*.toml' \
   -ignore '**/*.nix' \
+  -ignore '**/*.yaml' \
   "$TOP_DIR"
 
 uv run --directory "${PY_DIR}" liccheck

genkit-tools/cli/package.json

@@ -32,16 +32,17 @@
   "dependencies": {
     "@genkit-ai/telemetry-server": "workspace:*",
     "@genkit-ai/tools-common": "workspace:*",
+    "@inquirer/prompts": "^7.8.0",
     "@modelcontextprotocol/sdk": "^1.13.1",
     "axios": "^1.7.7",
     "colorette": "^2.0.20",
     "commander": "^11.1.0",
     "extract-zip": "^2.0.1",
     "get-port": "5.1.1",
-    "@inquirer/prompts": "^7.8.0",
     "open": "^6.3.0",
     "ora": "^5.4.1",
-    "semver": "^7.7.2"
+    "semver": "^7.7.2",
+    "yaml": "^2.8.0"
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",

genkit-tools/cli/src/cli.ts

@@ -23,6 +23,7 @@ import {
 } from '@genkit-ai/tools-common/utils';
 import { Command, program } from 'commander';
 import { config } from './commands/config';
+import { devTestModel } from './commands/dev-test-model';
 import { evalExtractData } from './commands/eval-extract-data';
 import { evalFlow } from './commands/eval-flow';
 import { evalRun } from './commands/eval-run';
@@ -59,6 +60,7 @@ const commands: Command[] = [
   initAiTools,
   config,
   start,
+  devTestModel,
   mcp,
 ];
 

genkit-tools/cli/src/commands/dev-test-model.ts

@@ -0,0 +1,74 @@
+- model: googleai/imagen-4.0-generate-001
+  supports:
+    - output-image
+- model: googleai/gemini-2.5-flash-preview-tts
+  tests:
+    - name: TTS Test
+      input:
+        messages:
+          - role: user
+            content:
+              - text: 'Hello world'
+        config:
+          responseModalities: ['AUDIO']
+      validators:
+        - valid-media:audio
+- model: googleai/gemini-2.5-pro
+  supports:
+    - tool-request
+    - structured-output
+    - multiturn
+    - system-role
+    - input-image-base64
+    - input-image-url
+    - input-video-youtube
+- model: googleai/gemini-3-pro-preview
+  supports:
+    - tool-request
+    - structured-output
+    - multiturn
+    - system-role
+    - input-image-base64
+    - input-image-url
+    - input-video-youtube
+  tests:
+    - name: Tool Response Conformance
+      input:
+        messages:
+          - role: user
+            content:
+              - text: 'What is the weather in New York? Use the tool.'
+          - role: model
+            content:
+              - toolRequest:
+                  name: weather
+                  input:
+                    city: New York
+                metadata:
+                  thoughtSignature: CvABAXLI2nxTZfKU3MkzLiGBrX62oq77vN2kHjT8pwwXRjtzbCqC07pPhIZ31sS+2kUFDh/kUY4SOvZzjjtP8UxI5GSFRWlX8yVDrDFo17RN/urwc1QuaMMzy66eQubpPRDEwfi6S5IKxZq0kRX6cSceB4NVCQAAAU8sYJwqWFL9CIaGac4lzF+34VvMWFLqdb40oe7/gw/KK1fqAeqDs+FJLksA+Q5qpHn3BETcqT0AuFe01IB2EVA7Us+/N3VGonw61F5cFNjHXO1jIYDybl3MXR9M5T5QB1a3EyicYXSX5/+bCmny1ka4kInbtzEqMMuv
+          - role: tool
+            content:
+              - toolResponse:
+                  name: weather
+                  output: '21C'
+        tools:
+          - name: weather
+            description: Get the weather for a city
+            inputSchema:
+              type: object
+              properties:
+                city:
+                  type: string
+              required:
+                - city
+      validators:
+        - text-includes:21
+- model: googleai/gemini-2.5-flash
+  supports:
+    - tool-request
+    - structured-output
+    - multiturn
+    - system-role
+    - input-image-base64
+    - input-image-url
+    - input-video-youtube