fix(cli): wait for model actions before dispatching conformance tests

genkit dev:test-model dispatched test requests immediately after the
runtime appeared, without checking whether the target model actions
were actually registered. This caused 404 errors on /api/runAction
for models that hadn't finished registering yet (especially with
non-JS runtimes that may have slower startup).

Add waitForActions() that polls /api/actions after the runtime is
detected and waits until all model keys referenced in the test
suites are present before starting tests. Polls every 500ms for
up to 30 seconds, with periodic progress logging.

Fixes #4599

Author: yesudeep

genkit-tools/cli/src/commands/dev-test-model.ts

@@ -484,6 +484,57 @@ async function waitForRuntime(manager: RuntimeManager) {
   logger.warn('Runtime not detected after 10 seconds.');
 }
 
+/**
+ * Waits until all model actions referenced by the test suites are registered
+ * with the runtime's reflection server. This prevents 404 errors caused by
+ * dispatching tests before the runtime has finished registering actions.
+ */
+async function waitForActions(
+  manager: RuntimeManager,
+  suites: TestSuite[]
+): Promise<void> {
+  const requiredKeys = new Set<string>();
+  for (const suite of suites) {
+    if (suite.model) {
+      const key = suite.model.startsWith('/')
+        ? suite.model
+        : `/model/${suite.model}`;
+      requiredKeys.add(key);
+    }
+  }
+  if (requiredKeys.size === 0) return;
+
+  const maxAttempts = 60; // 30 seconds total
+  const delayMs = 500;
+
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    try {
+      const actions = await manager.listActions();
+      const registeredKeys = new Set(Object.keys(actions));
+      const missing = [...requiredKeys].filter(
+        (k) => !registeredKeys.has(k)
+      );
+      if (missing.length === 0) {
+        logger.info(
+          `All ${requiredKeys.size} model actions registered. Starting tests.`
+        );
+        return;
+      }
+      if (attempt % 10 === 0 && attempt > 0) {
+        logger.info(
+          `Waiting for ${missing.length} model action(s) to register: ${missing.join(', ')}...`
+        );
+      }
+    } catch {
+      // Actions endpoint not ready yet, keep polling.
+    }
+    await new Promise((r) => setTimeout(r, delayMs));
+  }
+  logger.warn(
+    'Not all model actions registered after 30 seconds. Proceeding anyway.'
+  );
+}
+
 async function runTest(
   manager: RuntimeManager,
   model: string,
@@ -504,8 +555,8 @@ async function runTest(
       },
       shouldStream
         ? (chunk) => {
-            collectedChunks.push(chunk);
-          }
+          collectedChunks.push(chunk);
+        }
         : undefined
     );
 
@@ -527,8 +578,7 @@ async function runTest(
   } catch (e) {
     if (e instanceof GenkitToolsError) {
       logger.error(
-        `❌ Failed: ${testCase.name} - ${
-          e.data?.stack || JSON.stringify(e.data?.details) || e
+        `❌ Failed: ${testCase.name} - ${e.data?.stack || JSON.stringify(e.data?.details) || e
         }`
       );
     } else if (e instanceof Error) {
@@ -655,6 +705,10 @@ export const devTestModel = new Command('dev:test-model')
           .split(',')
           .map((s) => s.trim());
 
+        // Wait for all model actions to be registered before dispatching tests.
+        // This prevents 404 errors when the runtime is still initializing.
+        await waitForActions(manager, suites);
+
         for (const suite of suites) {
           if (!suite.model) {
             logger.error('Model name required in test suite.');