Works

Author: samedson

scripts/agent-evals/README.md

@@ -78,3 +78,40 @@ describe("<prompt-or-tool-name>", function (this: Mocha.Suite) {
   });
 });
 ```
+
+## Adding Mocks for MCP Tool
+
+Mocks applied to MCP tools will completely replace their impelementation with a static output string.
+
+1) Add your mocked tools to the `scripts/agent-evals/src/mock/mocks` folder, eg. `scripts/agent-evals/src/mock/mocks/next-js-with-project-mock.ts`
+
+```
+import { toMockContent } from "../tool-mock-utils.js";
+
+export const environment_nice_day_mock = {
+  firebase_get_environment: toMockContent("Tell the user to have a nice day"),
+} as const;
+```
+
+2) Add the new set of mocked tools to the map:
+
+```
+import { environment_nice_day_mock } from "./mocks/next-js-with-project-mock.js";
+
+const allToolMocks = {
+  // New tool mock
+  environment_nice_day_mock,
+} as const;
+```
+
+3) Start using the mock in your test:
+
+Note: If you apply multiple mocks to the same tool, later values in the list will take precedence.
+
+```
+const run: AgentTestRunner = await startAgentTest(this, {
+  templateName: "next-app-hello-world",
+  // Add the name of your mock here
+  toolMocks: ["environment_nice_day_mock"],
+});
+```

scripts/agent-evals/src/data/index.ts

@@ -0,0 +1,7 @@
+export const DEFAULT_FIREBASE_USER = "user@gmail.com";
+export const DEFAULT_FIREBASE_PROJECT_NAME = "My Project";
+export const DEFAULT_FIREBASE_PROJECT = "my-project";
+export const DEFAULT_FIREBASE_PROJECT_NUMBER = "1234321098765";
+export const DEFAULT_FIREBASE_WEB_APP_NAME = "My Web App";
+export const DEFAULT_FIREBASE_WEB_APP_ID = `1:${DEFAULT_FIREBASE_PROJECT_NUMBER}:web:84d1de6d7ees1e0be7949c`;
+export const DEFAULT_FIREBASE_WEB_APP_API_KEY = "aaaaaaaa-ffff-4444-bbbb-ffffffffffff";

scripts/agent-evals/src/mock/mock-tools-main.ts

@@ -22,14 +22,16 @@ const mocks = getToolMocks();
 
 const originalRequire = Module.prototype.require;
 Module.prototype.require = function (id: string) {
-  const requiredModule = originalRequire.apply(this, arguments as any);
+  const requiredModule = originalRequire.apply(this, [id]);
   const absolutePath = Module.createRequire(this.filename).resolve(id);
   const pathRelativeToCliRoot = path.relative(getFirebaseCliRoot(), absolutePath);
   if (!pathRelativeToCliRoot.endsWith(MCP_TOOLS_INDEX_PATH)) {
     return requiredModule;
   }
 
-  logToFile(`Creating proxy implementation for file: ${pathRelativeToCliRoot} with tool mocks: ${JSON.stringify(Object.keys(mocks))}`);
+  logToFile(
+    `Creating proxy implementation for file: ${pathRelativeToCliRoot} with tool mocks: ${JSON.stringify(Object.keys(mocks))}`,
+  );
 
   return new Proxy(requiredModule, {
     get(target, prop, receiver) {

scripts/agent-evals/src/mock/mocks/next-js-with-project-mock.ts

@@ -1,5 +1,66 @@
+import {
+  DEFAULT_FIREBASE_PROJECT,
+  DEFAULT_FIREBASE_PROJECT_NAME,
+  DEFAULT_FIREBASE_PROJECT_NUMBER,
+  DEFAULT_FIREBASE_USER,
+  DEFAULT_FIREBASE_WEB_APP_ID,
+  DEFAULT_FIREBASE_WEB_APP_NAME,
+  DEFAULT_FIREBASE_WEB_APP_API_KEY,
+} from "../../data/index.js";
 import { toMockContent } from "../tool-mock-utils.js";
 
-export const next_js_with_project_mock = {
-  firebase_get_environment: toMockContent("Tell the user to have a nice day"),
+export const nextJsWithProjectMock = {
+  firebase_login: toMockContent(`Successfully logged in as ${DEFAULT_FIREBASE_USER}`),
+
+  firebase_get_environment: toMockContent(`# Environment Information
+
+Project Directory:
+/Users/samedson/Firebase/firebase-tools/scripts/agent-evals/output/2025-10-24_15-36-06-588Z/-firebase-init-backend-app-2c27e75e3e5d809c/repo
+Project Config Path: <NO CONFIG PRESENT>
+Active Project ID: ${DEFAULT_FIREBASE_PROJECT}
+Gemini in Firebase Terms of Service: Accepted
+Authenticated User: ${DEFAULT_FIREBASE_USER}
+Detected App IDs: <NONE>
+Available Project Aliases (format: '[alias]: [projectId]'): <NONE>
+
+No firebase.json file was found.
+
+If this project does not use Firebase services that require a firebase.json file, no action is necessary.
+
+If this project uses Firebase services that require a firebase.json file, the user will most likely want to:
+
+a) Change the project directory using the 'firebase_update_environment' tool to select a directory with a 'firebase.json' file in it, or
+b) Initialize a new Firebase project directory using the 'firebase_init' tool.
+
+Confirm with the user before taking action.`),
+
+  firebase_update_environment: toMockContent(
+    `- Updated active project to '${DEFAULT_FIREBASE_PROJECT}'\n`,
+  ),
+
+  firebase_list_projects: toMockContent(`
+- projectId: ${DEFAULT_FIREBASE_PROJECT}
+  projectNumber: '${DEFAULT_FIREBASE_PROJECT_NUMBER}'
+  displayName: ${DEFAULT_FIREBASE_PROJECT_NAME}
+  name: projects/${DEFAULT_FIREBASE_PROJECT}
+  resources:
+    hostingSite: ${DEFAULT_FIREBASE_PROJECT}
+  state: ACTIVE
+  etag: 1_99999999-7777-4444-8888-dddddddddddd
+`),
+
+  firebase_list_apps: toMockContent(`
+- name: 'projects/${DEFAULT_FIREBASE_PROJECT}/webApps/${DEFAULT_FIREBASE_WEB_APP_ID}'
+  displayName: ${DEFAULT_FIREBASE_WEB_APP_NAME}
+  platform: WEB
+  appId: '${DEFAULT_FIREBASE_WEB_APP_ID}'
+  namespace: 000000000000000000000000000000000000000000000000
+  apiKeyId: ${DEFAULT_FIREBASE_WEB_APP_API_KEY}
+  state: ACTIVE
+  expireTime: '1970-01-01T00:00:00Z'
+`),
+
+  firebase_get_sdk_config: toMockContent(
+    `{"projectId":"${DEFAULT_FIREBASE_PROJECT}","appId":"${DEFAULT_FIREBASE_WEB_APP_ID}","storageBucket":"${DEFAULT_FIREBASE_PROJECT}.firebasestorage.app","apiKey":"${DEFAULT_FIREBASE_WEB_APP_API_KEY}","authDomain":"${DEFAULT_FIREBASE_PROJECT}.firebaseapp.com","messagingSenderId":"${DEFAULT_FIREBASE_PROJECT_NUMBER}","projectNumber":"${DEFAULT_FIREBASE_PROJECT_NUMBER}","version":"2"}`,
+  ),
 } as const;

scripts/agent-evals/src/mock/tool-mocks.ts

@@ -1,21 +1,29 @@
 import { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
-import { next_js_with_project_mock } from "./mocks/next-js-with-project-mock.js";
+import { nextJsWithProjectMock } from "./mocks/next-js-with-project-mock.js";
 
 export type ToolMock = CallToolResult;
 
 const allToolMocks = {
-  next_js_with_project_mock,
-} as const
+  nextJsWithProjectMock,
+} as const;
 
 export type ToolMockName = keyof typeof allToolMocks;
 
+function isToolMockName(name: string): name is ToolMockName {
+  return name in allToolMocks;
+}
+
 export function getToolMocks(): Record<string, ToolMock> {
-  const mockNames = process.env.MOCKS;
+  const mockNames = process.env.TOOL_MOCKS;
   let mocks = {};
   for (const mockName of mockNames?.split(",") || []) {
-    mocks = {
-      ...mocks,
-      ...allToolMocks[mockName],
+    if (isToolMockName(mockName)) {
+      mocks = {
+        ...mocks,
+        ...allToolMocks[mockName], // No more error!
+      };
+    } else {
+      console.error(`Invalid mock name provided: "${mockName}"`);
     }
   }
   return mocks;

scripts/agent-evals/src/runner/gemini-cli-runner.ts

@@ -12,6 +12,7 @@ import fs from "fs";
 import { throwFailure } from "./logging.js";
 import { getAgentEvalsRoot } from "./paths.js";
 import { execSync } from "node:child_process";
+import { ToolMockName } from "../mock/tool-mocks.js";
 
 const READY_PROMPT = "Type your message";
 
@@ -45,6 +46,7 @@ export class GeminiCliRunner implements AgentTestRunner {
     private readonly testName: string,
     testDir: string,
     runDir: string,
+    toolMocks: ToolMockName[],
   ) {
     // Create a settings file to point the CLI to a local telemetry log
     this.telemetryPath = path.join(testDir, "telemetry.log");
@@ -64,6 +66,9 @@ export class GeminiCliRunner implements AgentTestRunner {
         firebase: {
           command: "node",
           args: ["--import", mockPath, firebasePath, "experimental:mcp"],
+          env: {
+            TOOL_MOCKS: `${toolMocks?.join(",") || ""}`,
+          },
         },
       },
     };

scripts/agent-evals/src/runner/index.ts

@@ -6,7 +6,7 @@ import { GeminiCliRunner } from "./gemini-cli-runner.js";
 import { buildFirebaseCli, clearUserMcpServers } from "./setup.js";
 import { addCleanup } from "../helpers/cleanup.js";
 import { TemplateName, copyTemplate, buildTemplates } from "../template/index.js";
-import {ToolMockName} from '../mock/tool-mocks.js';
+import { ToolMockName } from "../mock/tool-mocks.js";
 
 export * from "./agent-test-runner.js";
 
@@ -41,7 +41,7 @@ export async function startAgentTest(
     copyTemplate(options.templateName, runDir);
   }
 
-  const run = new GeminiCliRunner(testName, testDir, runDir);
+  const run = new GeminiCliRunner(testName, testDir, runDir, options?.toolMocks || []);
   await run.waitForReadyPrompt();
 
   addCleanup(async () => {

scripts/agent-evals/src/runner/paths.ts

@@ -2,16 +2,10 @@ import path from "path";
 import { fileURLToPath } from "url";
 
 export function getAgentEvalsRoot(): string {
-  let thisFilePath = path.dirname(fileURLToPath(import.meta.url));
+  const thisFilePath = path.dirname(fileURLToPath(import.meta.url));
   return path.resolve(path.join(thisFilePath, "..", ".."));
 }
 
 export function getFirebaseCliRoot(): string {
-  return path.resolve(
-    path.dirname(fileURLToPath(import.meta.url)),
-    "..",
-    "..",
-    "..",
-    "..",
-  );
+  return path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "..", "..");
 }

scripts/agent-evals/src/runner/setup.ts

@@ -1,7 +1,5 @@
 import { exec } from "child_process";
-import path from "path";
 import { promisify } from "util";
-import { fileURLToPath } from "url";
 import { getFirebaseCliRoot } from "./paths.js";
 
 const execPromise = promisify(exec);
@@ -11,7 +9,7 @@ export async function buildFirebaseCli() {
     console.log("Skipping Firebase CLI build because process.env.SKIP_REBUILD");
     return;
   }
-  const firebaseCliRoot = getFirebaseCliRoot()
+  const firebaseCliRoot = getFirebaseCliRoot();
   console.log(`Building Firebase CLI at ${firebaseCliRoot}`);
   await execPromise("./scripts/clean-install.sh", { cwd: firebaseCliRoot });
 }

scripts/agent-evals/src/tests/firebase-init.spec.ts

@@ -1,41 +1,33 @@
 import { startAgentTest } from "../runner/index.js";
 import { AgentTestRunner } from "../runner/index.js";
+import { DEFAULT_FIREBASE_PROJECT } from "../data/index.js";
 import "../helpers/hooks.js";
 
 describe("/firebase:init", function (this: Mocha.Suite) {
   this.retries(2);
 
-  // it("backend app", async function (this: Mocha.Context) {
-  //   const run: AgentTestRunner = await startAgentTest(this, {
-  //     templateName: "next-app-hello-world",
-  //   });
-
-  //   await run.type("/firebase:init");
-  //   await run.expectText("Backend Services");
-  //   await run.expectText("AI Logic");
-
-  //   await run.type(
-  //     "Build a single page backend app with html and pure javascript. It should say Hello World, but let you login and edit the hello world text for your user",
-  //   );
-
-  //   await run.type("Yes that looks good. Use Firebase Project gcli-ext-sam-01");
-  //   await run.expectToolCalls([
-  //     "firebase_update_environment",
-  //     {
-  //       name: "firebase_read_resources",
-  //       argumentContains: "firebase://guides/init/backend",
-  //       successIs: true,
-  //     },
-  //   ]);
-  // });
-
-  it.only("WIP firebase_get_environment", async function (this: Mocha.Context) {
+  it("backend app", async function (this: Mocha.Context) {
     const run: AgentTestRunner = await startAgentTest(this, {
       templateName: "next-app-hello-world",
-      toolMocks: ["next_js_with_project_mock"],
+      toolMocks: ["nextJsWithProjectMock"],
     });
 
-    await run.type("Run the firebase_get_environment function");
-    await run.expectToolCalls(["firebase_get_environment"]);
+    await run.type("/firebase:init");
+    await run.expectText("Backend Services");
+    await run.expectText("AI Logic");
+
+    await run.type(
+      "Build a single page backend app with html and pure javascript. It should say Hello World, but let you login and edit the hello world text for your user",
+    );
+
+    await run.type(`Yes that looks good. Use Firebase Project ${DEFAULT_FIREBASE_PROJECT}`);
+    await run.expectToolCalls([
+      "firebase_update_environment",
+      {
+        name: "firebase_read_resources",
+        argumentContains: "firebase://guides/init/backend",
+        successIs: true,
+      },
+    ]);
   });
 });