chore: integrate capturing accuracy snapshots

Author: himanshusinghs

package.json

@@ -29,7 +29,8 @@
     "check:types": "tsc --noEmit --project tsconfig.json",
     "reformat": "prettier --write .",
     "generate": "./scripts/generate.sh",
-    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage"
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage --testPathIgnorePatterns=/tests/accuracy/",
+    "test:accuracy": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testPathPattern tests/accuracy"
   },
   "license": "Apache-2.0",
   "devDependencies": {

tests/accuracy/sdk/accuracy-snapshot.ts

@@ -0,0 +1,54 @@
+import fs from "fs/promises";
+import path from "path";
+import { z } from "zod";
+
+export const SNAPSHOT_FILE_PATH = path.resolve(process.cwd(), "accuracy-snapshot.json");
+
+export const AccuracySnapshotEntrySchema = z.object({
+    datetime: z.string(),
+    commit: z.string(),
+    model: z.string(),
+    suite: z.string(),
+    test: z.string(),
+    toolCallingAccuracy: z.number(),
+    parameterAccuracy: z.number(),
+});
+
+export type AccuracySnapshotEntry = z.infer<typeof AccuracySnapshotEntrySchema>;
+
+export async function readSnapshot(): Promise<AccuracySnapshotEntry[]> {
+    try {
+        const raw = await fs.readFile(SNAPSHOT_FILE_PATH, "utf8");
+        return AccuracySnapshotEntrySchema.array().parse(JSON.parse(raw));
+    } catch (e: unknown) {
+        if ((e as { code: string }).code === "ENOENT") {
+            return [];
+        }
+        throw e;
+    }
+}
+
+function waitFor(ms: number) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export async function appendAccuracySnapshot(entry: AccuracySnapshotEntry): Promise<void> {
+    AccuracySnapshotEntrySchema.parse(entry);
+
+    for (let attempt = 0; attempt < 5; attempt++) {
+        try {
+            const snapshot = await readSnapshot();
+            snapshot.unshift(entry);
+            const tmp = `${SNAPSHOT_FILE_PATH}~${Date.now()}`;
+            await fs.writeFile(tmp, JSON.stringify(snapshot, null, 2));
+            await fs.rename(tmp, SNAPSHOT_FILE_PATH);
+            return;
+        } catch (e) {
+            if (attempt < 4) {
+                await waitFor(100 + Math.random() * 200);
+            } else {
+                throw e;
+            }
+        }
+    }
+}

tests/accuracy/sdk/describe-accuracy-tests.ts

@@ -3,6 +3,7 @@ import { discoverMongoDBTools, TestTools, MockedTools } from "./test-tools.js";
 import { TestableModels } from "./models.js";
 import { ExpectedToolCall, parameterMatchingAccuracyScorer, toolCallingAccuracyScorer } from "./accuracy-scorers.js";
 import { Agent, getVercelToolCallingAgent } from "./agent.js";
+import { appendAccuracySnapshot } from "./accuracy-snapshot.js";
 
 interface AccuracyTestConfig {
     prompt: string;
@@ -15,6 +16,20 @@ export function describeAccuracyTests(
     models: TestableModels,
     accuracyTestConfigs: AccuracyTestConfig[]
 ) {
+    const accuracyDatetime = process.env.ACCURACY_DATETIME;
+    if (!accuracyDatetime) {
+        throw new Error("ACCURACY_DATETIME environment variable is not set");
+    }
+    const accuracyCommit = process.env.ACCURACY_COMMIT;
+    if (!accuracyCommit) {
+        throw new Error("ACCURACY_COMMIT environment variable is not set");
+    }
+
+    if (!models.length) {
+        console.warn(`No models available to test ${suiteName}`);
+        return;
+    }
+
     const eachModel = describe.each(models);
     const eachTest = it.each(accuracyTestConfigs);
 
@@ -35,15 +50,30 @@ export function describeAccuracyTests(
         eachTest("$prompt", async function (testConfig) {
             testTools.mockTools(testConfig.mockedTools);
             const conversation = await agent.prompt(testConfig.prompt, model, testTools.vercelAiTools());
-            console.log("conversation", conversation);
             const toolCalls = testTools.getToolCalls();
-            console.log("?????? toolCalls", toolCalls);
-            console.log("???? expected", testConfig.expectedToolCalls);
             const toolCallingAccuracy = toolCallingAccuracyScorer(testConfig.expectedToolCalls, toolCalls);
             const parameterMatchingAccuracy = parameterMatchingAccuracyScorer(testConfig.expectedToolCalls, toolCalls);
+            await appendAccuracySnapshot({
+                datetime: accuracyDatetime,
+                commit: accuracyCommit,
+                model: model.modelName,
+                suite: suiteName,
+                test: testConfig.prompt,
+                toolCallingAccuracy,
+                parameterAccuracy: parameterMatchingAccuracy,
+            });
 
-            expect(toolCallingAccuracy).not.toEqual(0);
-            expect(parameterMatchingAccuracy).toBeGreaterThanOrEqual(0.5);
+            try {
+                expect(toolCallingAccuracy).not.toEqual(0);
+                expect(parameterMatchingAccuracy).toBeGreaterThanOrEqual(0.5);
+            } catch (error) {
+                console.warn(`Accuracy test failed for ${model.modelName} - ${suiteName} - ${testConfig.prompt}`);
+                console.warn(`Conversation`, JSON.stringify(conversation, null, 2));
+                console.warn(`Tool calls`, JSON.stringify(toolCalls, null, 2));
+                console.warn(`Tool calling accuracy`, toolCallingAccuracy);
+                console.warn(`Parameter matching accuracy`, parameterMatchingAccuracy);
+                throw error;
+            }
         });
     });
 }

tests/accuracy/sdk/models.ts

@@ -3,6 +3,7 @@ import { createGoogleGenerativeAI } from "@himanshusinghs/google";
 import { ollama } from "ollama-ai-provider";
 
 export interface Model<P extends LanguageModelV1 = LanguageModelV1> {
+    readonly modelName: string;
     isAvailable(): boolean;
     getModel(): P;
 }
@@ -25,7 +26,7 @@ export class OllamaModel implements Model {
     constructor(readonly modelName: string) {}
 
     isAvailable(): boolean {
-        return true;
+        return false;
     }
 
     getModel() {
@@ -35,8 +36,8 @@ export class OllamaModel implements Model {
 
 const ALL_TESTABLE_MODELS = [
     new GeminiModel("gemini-1.5-flash"),
-    // new GeminiModel("gemini-2.0-flash"),
-    // new OllamaModel("qwen3:latest"),
+    new GeminiModel("gemini-2.0-flash"),
+    new OllamaModel("qwen3:latest"),
 ];
 
 export type TestableModels = ReturnType<typeof getAvailableModels>;

tests/accuracy/sdk/test-tools.ts

@@ -132,9 +132,6 @@ export async function discoverMongoDBTools(): Promise<Tool[]> {
         await mcpClient.connect(clientTransport);
 
         return (await mcpClient.listTools()).tools;
-    } catch (error: unknown) {
-        console.error("Unexpected error occured", error);
-        return [];
     } finally {
         await mcpClient?.close();
         await mcpServer?.session?.close();