chore: rebase fixes and move to vitest

Author: himanshusinghs

package-lock.json

@@ -16,18 +16,12 @@ export MDB_ACCURACY_RUN_ID=$(npx uuid v4)
 # By default we run all the tests under tests/accuracy folder unless a path is
 # specified in the command line. Such as:
 # npm run test:accuracy -- tests/accuracy/some-test.test.ts
-if [ $# -gt 0 ]; then
-    TEST_PATH_PATTERN="$1"
-    shift
-else
-    TEST_PATH_PATTERN="tests/accuracy"
-fi
-echo "Running accuracy tests with MDB_ACCURACY_RUN_ID '$MDB_ACCURACY_RUN_ID' and TEST_PATH_PATTERN '$TEST_PATH_PATTERN'"
-node --experimental-vm-modules node_modules/jest/bin/jest.js --bail --testPathPatterns "$TEST_PATH_PATTERN" "$@"
+echo "Running accuracy tests with MDB_ACCURACY_RUN_ID '$MDB_ACCURACY_RUN_ID'"
+vitest --config vitest.config.ts --project=accuracy --coverage=false --run --testTimeout=3600000 "$@"
 
 # Preserving the exit code from test run to correctly notify in the CI
 # environments when the tests fail.
-JEST_EXIT_CODE=$?
+TEST_EXIT_CODE=$?
 
 # Each test run submits an accuracy result with the accuracyRunStatus:
 # "in-progress". When all the tests are done and jest exits with an exit code of
@@ -42,10 +36,10 @@ JEST_EXIT_CODE=$?
 
 # This is necessary when comparing one accuracy run with another as we wouldn't
 # want to compare against an incomplete run.
-export MDB_ACCURACY_RUN_STATUS=$([ $JEST_EXIT_CODE -eq 0 ] && echo "done" || echo "failed")
+export MDB_ACCURACY_RUN_STATUS=$([ $TEST_EXIT_CODE -eq 0 ] && echo "done" || echo "failed")
 npx tsx scripts/accuracy/update-accuracy-run-status.ts || echo "Warning: Failed to update accuracy run status to '$MDB_ACCURACY_RUN_STATUS'"
 
 # This is optional but we do it anyways to generate a readable summary of report.
 npx tsx scripts/accuracy/generate-test-summary.ts || echo "Warning: Failed to generate test summary HTML report"
 
-exit $JEST_EXIT_CODE
+exit $TEST_EXIT_CODE

scripts/accuracy/run-accuracy-tests.sh

@@ -52,6 +52,7 @@ export class DiskBasedResultStorage implements AccuracyResultStorage {
         // commit so that we can use that during baseline comparison.
         if (status === AccuracyRunStatus.Done) {
             const latestResultFilePath = this.getLatestResultFilePath(commitSHA);
+            await this.ensureFileWithInitialData(latestResultFilePath, JSON.stringify({}));
             await this.withFileLock(latestResultFilePath, async () => {
                 await fs.unlink(latestResultFilePath);
                 await fs.link(resultFilePath, latestResultFilePath);
@@ -86,7 +87,7 @@ export class DiskBasedResultStorage implements AccuracyResultStorage {
             ],
         };
         const resultFilePath = this.getAccuracyResultFilePath(commitSHA, runId);
-        const { fileCreatedWithInitialData } = await this.ensureAccuracyResultFile(
+        const { fileCreatedWithInitialData } = await this.ensureFileWithInitialData(
             resultFilePath,
             JSON.stringify(initialData, null, 2)
         );
@@ -143,7 +144,7 @@ export class DiskBasedResultStorage implements AccuracyResultStorage {
         }
     }
 
-    private async ensureAccuracyResultFile(
+    private async ensureFileWithInitialData(
         filePath: string,
         initialData: string
     ): Promise<{

tests/accuracy/sdk/accuracy-result-storage/disk-storage.ts

@@ -1,3 +1,4 @@
+import { describe, it, beforeAll, beforeEach, afterAll } from "vitest";
 import { TestableModels } from "./models.js";
 import { calculateToolCallingAccuracy } from "./accuracy-scorer.js";
 import { getVercelToolCallingAgent, VercelAgent } from "./agent.js";

tests/accuracy/sdk/describe-accuracy-tests.ts

@@ -1,3 +1,4 @@
+import { describe, expect, it } from "vitest";
 import { calculateToolCallingAccuracy } from "../accuracy/sdk/accuracy-scorer.js";
 import { ExpectedToolCall, LLMToolCall } from "../accuracy/sdk/accuracy-result-storage/result-storage.js";
 

tests/unit/accuracy-scorer.test.ts

@@ -1,15 +1,39 @@
 import { defineConfig } from "vitest/config";
 
+// Shared exclusions for all projects
+// Ref: https://vitest.dev/config/#exclude
+const vitestDefaultExcludes = [
+    "**/node_modules/**",
+    "**/dist/**",
+    "**/cypress/**",
+    "**/.{idea,git,cache,output,temp}/**",
+    "**/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress,tsup,build,eslint,prettier}.config.*",
+];
+
 export default defineConfig({
     test: {
         environment: "node",
         testTimeout: 3600000,
         hookTimeout: 3600000,
-        include: ["**/*.test.ts"],
         setupFiles: ["./tests/setup.ts"],
         coverage: {
             exclude: ["node_modules", "tests", "dist"],
             reporter: ["lcov"],
         },
+        projects: [
+            {
+                test: {
+                    name: "unit-and-integration",
+                    include: ["**/*.test.ts"],
+                    exclude: [...vitestDefaultExcludes, "tests/accuracy/**"],
+                },
+            },
+            {
+                test: {
+                    name: "accuracy",
+                    include: ["**/accuracy/*.test.ts"],
+                },
+            },
+        ],
     },
 });