Update agent-evals to be a commonjs library

Author: schnecle

scripts/agent-evals/.mocharc.yml

@@ -1,5 +1,5 @@
 import:
-  - ./lib/helpers/mocha-bootstrap.js
+  - ./lib/scripts/agent-evals/src/helpers/mocha-bootstrap.js
 timeout: 120000
 recursive: true
 node-options:

scripts/agent-evals/package-lock.json

@@ -2,18 +2,20 @@
   "name": "agent-evals",
   "version": "1.0.0",
   "description": "",
-  "type": "module",
   "scripts": {
     "build": "tsc",
-    "test": "npm run build && mocha 'lib/**/*.spec.js' --reporter spec",
+    "test": "npm run build && mocha 'lib/**/*.spec.js' --reporter spec --require lib/scripts/agent-evals/src/helpers/mocha-bootstrap.js",
     "test:dev": "SKIP_REBUILD=true COPY_FIREBASE_CLI_CONFIG=true npm run test"
   },
   "keywords": [],
   "author": "",
   "license": "ISC",
   "devDependencies": {
     "@types/mocha": "^9.0.0",
-    "@types/node": "^24.9.0"
+    "@types/node": "18.19.1",
+    "@typescript-eslint/eslint-plugin": "^5.9.0",
+    "@typescript-eslint/parser": "^5.9.0",
+    "typescript": "^4.5.4"
   },
   "dependencies": {
     "mocha": "^11.7.1",

scripts/agent-evals/package.json

@@ -1,3 +1,5 @@
-import { setupEnvironment } from "../runner/index.js";
+import { setupEnvironment } from "../runner/index";
 
-await setupEnvironment();
+export async function mochaGlobalSetup() {
+  await setupEnvironment();
+}

scripts/agent-evals/src/helpers/mocha-bootstrap.ts

@@ -1,9 +1,8 @@
 import { Module } from "module";
-import path from "path";
-import fs from "fs";
-import os from "os";
+import * as path from "path";
+import * as fs from "fs";
+import * as os from "os";
 import { getFirebaseCliRoot } from "../runner/paths.js";
-import { getToolMocks } from "./tool-mocks.js";
 
 //
 // This file is run as a node --import parameter before the Firebase CLI to
@@ -13,18 +12,31 @@ import { getToolMocks } from "./tool-mocks.js";
 // Path to the built MCP Tools implementation in the Firebase CLI, relative to
 // the repo's root
 const MCP_TOOLS_INDEX_PATH = "lib/mcp/tools/index.js";
+const CONFIGSTORE_INDEX_PATH = "lib/mcp/configstore.js";
 const LOG_FILE_PATH = path.join(os.homedir(), "Desktop", "agent_evals_mock_logs.txt");
 // Enable this to turn on file logging. This can be helpful for debugging
 // because console logs get swallowed
 const ENABLE_FILE_LOGGING = false;
 
-const mocks = getToolMocks();
-
 const originalRequire = Module.prototype.require;
-Module.prototype.require = function (id: string) {
+(Module.prototype as any).require = function (id: string) {
   const requiredModule = originalRequire.apply(this, [id]);
   const absolutePath = Module.createRequire(this.filename).resolve(id);
   const pathRelativeToCliRoot = path.relative(getFirebaseCliRoot(), absolutePath);
+  console.log(`[DEBUG] Requiring: ${pathRelativeToCliRoot} (Absolute: ${absolutePath})`);
+
+  // Mock configstore to avoid "Cannot find module ../package.json" error and side effects
+  if (pathRelativeToCliRoot.endsWith(CONFIGSTORE_INDEX_PATH)) {
+    logToFile(`Mocking configstore for: ${pathRelativeToCliRoot}`);
+    return {
+      configstore: {
+        get: () => undefined,
+        set: () => { },
+        delete: () => { },
+      },
+    };
+  }
+
   if (!pathRelativeToCliRoot.endsWith(MCP_TOOLS_INDEX_PATH)) {
     return requiredModule;
   }
@@ -67,6 +79,10 @@ Module.prototype.require = function (id: string) {
   });
 };
 
+import { getToolMocks } from "./tool-mocks.js";
+
+const mocks = getToolMocks();
+
 function logToFile(message: string) {
   if (!ENABLE_FILE_LOGGING) {
     return;

scripts/agent-evals/src/mock/mock-tools-main.ts

@@ -1,5 +1,5 @@
 import { DEFAULT_FIREBASE_PROJECT, DEFAULT_FIREBASE_USER } from "../../data/index.js";
-import { renderTemplate } from "../../../../../src/mcp/tools/core/get_environment.js";
+import { renderTemplate } from "../../../../../src/mcp/tools/core/get_environment";
 import { toMockContent } from "../tool-mock-utils.js";
 
 const PROJECT_DIR = "/Users/fakeuser/develop/fake-project";

scripts/agent-evals/src/mock/mocks/get-environment-mock.ts

@@ -7,7 +7,7 @@ import {
   DEFAULT_FIREBASE_WEB_APP_NAME,
   DEFAULT_FIREBASE_WEB_APP_API_KEY,
 } from "../../data/index.js";
-import { renderTemplate } from "../../../../../src/mcp/tools/core/get_environment.js";
+import { renderTemplate } from "../../../../../src/mcp/tools/core/get_environment";
 import { toMockContent } from "../tool-mock-utils.js";
 
 const PROJECT_DIR = "/Users/fakeuser/develop/fake-project";

scripts/agent-evals/src/mock/mocks/next-js-with-project-mock.ts

@@ -1,10 +1,10 @@
-import { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
-import { nextJsWithProjectMock } from "./mocks/next-js-with-project-mock.js";
-import {
+import type { CallToolResult } from "@modelcontextprotocol/sdk/types";
+const { nextJsWithProjectMock } = require("./mocks/next-js-with-project-mock");
+const {
   getEnvironmentWithIosApp,
   getEnvironmentWithAndroidApp,
-  getEnvironmentWithFlutterApp
-} from "./mocks/get-environment-mock.js";
+  getEnvironmentWithFlutterApp,
+} = require("./mocks/get-environment-mock");
 
 export type ToolMock = CallToolResult;
 
@@ -36,3 +36,5 @@ export function getToolMocks(): Record<string, ToolMock> {
   }
   return mocks;
 }
+
+module.exports = { getToolMocks };

scripts/agent-evals/src/mock/tool-mocks.ts

@@ -1,12 +1,6 @@
 import { ToolDef } from "./tool-matcher.js";
 
-export interface AgentTestRunner {
-  /**
-   * Simulates typing a string and waits for the turn to complete. It types one
-   * character at a time to avoid paste detection that the Gemini CLI has
-   */
-  type(text: string): Promise<void>;
-
+export interface AgentTestMatchers {
   /**
    * Looks for a specific string or regex to in the agent's output since the
    * last time the user typed and pressed enter.
@@ -26,3 +20,16 @@ export interface AgentTestRunner {
    */
   expectMemory(text: string | RegExp): Promise<void>;
 }
+
+export interface AgentTestRunner extends AgentTestMatchers {
+  /**
+   * Simulates typing a string and waits for the turn to complete. It types one
+   * character at a time to avoid paste detection that the Gemini CLI has
+   */
+  type(text: string): Promise<void>;
+
+  /**
+   * Negated assertions
+   */
+  dont: AgentTestMatchers;
+}