chore: PR feedback

himanshusinghs · himanshusinghs · commit e6f426dc8f7d · 2025-07-16T18:29:46.000+02:00
- well defined Model types
- move getAllAvailableModels() inside the test setup
diff --git a/tests/accuracy/aggregate.test.ts b/tests/accuracy/aggregate.test.ts
@@ -1,7 +1,6 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     {
         prompt: "Group all the movies in 'mflix.movies' namespace by 'release_year' and give me a count of them",
         expectedToolCalls: [
diff --git a/tests/accuracy/collection-indexes.test.ts b/tests/accuracy/collection-indexes.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 
 function callsCollectionIndexes(prompt: string): AccuracyTestConfig {
@@ -17,7 +16,7 @@ function callsCollectionIndexes(prompt: string): AccuracyTestConfig {
     };
 }
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     callsCollectionIndexes("How many indexes do I have in 'mflix.movies' namespace?"),
     callsCollectionIndexes("List all the indexes in movies collection in mflix database"),
     callsCollectionIndexes(
diff --git a/tests/accuracy/collection-schema.test.ts b/tests/accuracy/collection-schema.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 
 function callsCollectionSchema(prompt: string): AccuracyTestConfig {
@@ -17,7 +16,7 @@ function callsCollectionSchema(prompt: string): AccuracyTestConfig {
     };
 }
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     callsCollectionSchema("Is there a title field in 'db1.coll1' namespace?"),
     callsCollectionSchema("What is the type of value stored in title field in coll1 collection in db1 database?"),
 ]);
diff --git a/tests/accuracy/collection-storage-size.test.ts b/tests/accuracy/collection-storage-size.test.ts
@@ -1,7 +1,6 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     {
         prompt: "What is the size of 'mflix.movies' namespace",
         expectedToolCalls: [
diff --git a/tests/accuracy/count.test.ts b/tests/accuracy/count.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 
 function callsCountToolWithEmptyQuery(prompt: string, database = "mflix", collection = "movies"): AccuracyTestConfig {
@@ -38,7 +37,7 @@ function callsCountToolWithQuery(
     };
 }
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     callsCountToolWithEmptyQuery("Count number of documents in 'mflix.movies' namespace."),
     callsCountToolWithEmptyQuery(
         "How many documents are there in 'characters' collection in 'comics' database?",
diff --git a/tests/accuracy/create-collection.test.ts b/tests/accuracy/create-collection.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 import { ExpectedToolCall } from "./sdk/accuracy-result-storage/result-storage.js";
 
@@ -27,7 +26,7 @@ function callsCreateCollectionWithListCollections(prompt: string, expectedToolCa
     };
 }
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     callsCreateCollection("Create a new namespace 'mflix.documentaries'", "mflix", "documentaries"),
     callsCreateCollection("Create a new collection villains in comics database", "comics", "villains"),
     callsCreateCollectionWithListCollections(
diff --git a/tests/accuracy/create-index.test.ts b/tests/accuracy/create-index.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 
 function callsCreateIndex(prompt: string, indexKeys: Record<string, unknown>): AccuracyTestConfig {
@@ -18,7 +17,7 @@ function callsCreateIndex(prompt: string, indexKeys: Record<string, unknown>): A
     };
 }
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     callsCreateIndex(
         "Create an index that covers the following query on 'mflix.movies' namespace - { \"release_year\": 1992 }",
         {
diff --git a/tests/accuracy/db-stats.test.ts b/tests/accuracy/db-stats.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 
 function callsListDatabases(prompt: string, database = "mflix"): AccuracyTestConfig {
@@ -16,4 +15,4 @@ function callsListDatabases(prompt: string, database = "mflix"): AccuracyTestCon
     };
 }
 
-describeAccuracyTests(getAvailableModels(), [callsListDatabases("What is the size occupied by database mflix?")]);
+describeAccuracyTests([callsListDatabases("What is the size occupied by database mflix?")]);
diff --git a/tests/accuracy/delete-many.test.ts b/tests/accuracy/delete-many.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 
 function callsDeleteManyWithEmptyFilters(prompt: string): AccuracyTestConfig {
@@ -33,7 +32,7 @@ function callsDeleteManyWithFilters(prompt: string): AccuracyTestConfig {
     };
 }
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     callsDeleteManyWithEmptyFilters("Delete all the documents from 'mflix.movies' namespace"),
     callsDeleteManyWithEmptyFilters("Purge the collection 'movies' in database 'mflix'"),
     callsDeleteManyWithFilters("Remove all the documents from namespace 'mflix.movies' where runtime is less than 100"),
diff --git a/tests/accuracy/drop-collection.test.ts b/tests/accuracy/drop-collection.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 import { ExpectedToolCall } from "./sdk/accuracy-result-storage/result-storage.js";
 
@@ -25,7 +24,7 @@ function callsDropCollection(prompt: string, expectedToolCalls: ExpectedToolCall
     };
 }
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     onlyCallsDropCollection("Remove mflix.movies namespace from my cluster."),
     onlyCallsDropCollection("Drop movies collection from mflix database."),
     callsDropCollection("Remove books collection from which ever database contains it.", [
diff --git a/tests/accuracy/drop-database.test.ts b/tests/accuracy/drop-database.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 import { ExpectedToolCall } from "./sdk/accuracy-result-storage/result-storage.js";
 
@@ -24,7 +23,7 @@ function callsDropDatabase(prompt: string, expectedToolCalls: ExpectedToolCall[]
     };
 }
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     onlyCallsDropDatabase("Remove mflix database from my cluster."),
     onlyCallsDropDatabase("Drop database named mflix."),
     callsDropDatabase("If there is a mflix database in my cluster then drop it.", [
diff --git a/tests/accuracy/explain.test.ts b/tests/accuracy/explain.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 
 function callsExplain(prompt: string, method: Record<string, unknown>): AccuracyTestConfig {
@@ -51,7 +50,7 @@ const callsExplainWithCount = (prompt: string) =>
  * because we are using Zod.union, when we probably should've used
  * Zod.discriminatedUnion
  */
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     callsExplainWithFind(
         `Will fetching documents, where release_year is 2020, from 'mflix.movies' namespace perform a collection scan?`
     ),
diff --git a/tests/accuracy/find.test.ts b/tests/accuracy/find.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 
 function callsFindNoFilter(prompt: string, database = "mflix", collection = "movies"): AccuracyTestConfig {
@@ -93,7 +92,7 @@ function callsFindWithFilterSortAndLimit(
     };
 }
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     callsFindNoFilter("List all the movies in 'mflix.movies' namespace."),
     callsFindNoFilter("List all the documents in 'comics.books' namespace.", "comics", "books"),
     callsFindWithFilter("Find all the movies in 'mflix.movies' namespace with runtime less than 100.", {
diff --git a/tests/accuracy/insert-many.test.ts b/tests/accuracy/insert-many.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 
 function callsInsertMany(prompt: string): AccuracyTestConfig {
@@ -47,7 +46,7 @@ function callsEmptyInsertMany(prompt: string) {
     };
 }
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     callsInsertMany(
         [
             "In my namespace 'mflix.movies', insert 3 documents each with the following fields:",
diff --git a/tests/accuracy/list-collections.test.ts b/tests/accuracy/list-collections.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 
 function callsListCollections(prompt: string): AccuracyTestConfig {
@@ -48,7 +47,7 @@ function callsListDatabasesAndListCollections(prompt: string): AccuracyTestConfi
     };
 }
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     callsListCollections("How many collections do I have in database mflix?"),
     callsListCollections("List all the collections in my MongoDB database mflix."),
     callsListCollections("Is there a shows collection in my MongoDB database mflix?"),
diff --git a/tests/accuracy/list-databases.test.ts b/tests/accuracy/list-databases.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 
 function callsListDatabases(prompt: string): AccuracyTestConfig {
@@ -14,7 +13,7 @@ function callsListDatabases(prompt: string): AccuracyTestConfig {
     };
 }
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     callsListDatabases("How many databases do I have?"),
     callsListDatabases("List all the databases that I have in my clusters"),
     callsListDatabases("Is there a mflix database in my cluster?"),
diff --git a/tests/accuracy/logs.test.ts b/tests/accuracy/logs.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 import { ExpectedToolCall } from "./sdk/accuracy-result-storage/result-storage.js";
 
@@ -10,7 +9,7 @@ function callsLogsTool(prompt: string, toolCall: ExpectedToolCall): AccuracyTest
     };
 }
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     callsLogsTool("Were there any startup warnings for my MongoDB server?", {
         toolName: "mongodb-logs",
         parameters: {
diff --git a/tests/accuracy/rename-collection.test.ts b/tests/accuracy/rename-collection.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 
 function callsRenameCollection(prompt: string): AccuracyTestConfig {
@@ -35,7 +34,7 @@ function callsRenameCollectionWithDropTarget(prompt: string): AccuracyTestConfig
     };
 }
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     callsRenameCollection("Rename my 'mflix.movies' namespace to 'mflix.new_movies'"),
     callsRenameCollectionWithDropTarget(
         "Rename my 'mflix.movies' namespace to 'mflix.new_movies' while removing the old namespace."
diff --git a/tests/accuracy/sdk/describe-accuracy-tests.ts b/tests/accuracy/sdk/describe-accuracy-tests.ts
@@ -1,5 +1,5 @@
 import { describe, it, beforeAll, beforeEach, afterAll } from "vitest";
-import { TestableModels } from "./models.js";
+import { getAvailableModels } from "./models.js";
 import { calculateToolCallingAccuracy } from "./accuracy-scorer.js";
 import { getVercelToolCallingAgent, VercelAgent } from "./agent.js";
 import { prepareTestData, setupMongoDBIntegrationTest } from "../../integration/tools/mongodb/mongodbHelpers.js";
@@ -41,11 +41,12 @@ export interface AccuracyTestConfig {
     mockedTools?: MockedTools;
 }
 
-export function describeAccuracyTests(models: TestableModels, accuracyTestConfigs: AccuracyTestConfig[]) {
+export function describeAccuracyTests(accuracyTestConfigs: AccuracyTestConfig[]) {
     if (!process.env.MDB_ACCURACY_RUN_ID) {
         throw new Error("MDB_ACCURACY_RUN_ID env variable is required for accuracy test runs!");
     }
 
+    const models = getAvailableModels();
     if (!models.length) {
         throw new Error("No models available to test. Ensure that the API keys are properly setup!");
     }
diff --git a/tests/accuracy/sdk/models.ts b/tests/accuracy/sdk/models.ts
@@ -4,12 +4,12 @@ import { createAzure } from "@ai-sdk/azure";
 import { createOpenAI } from "@ai-sdk/openai";
 import { ollama } from "ollama-ai-provider";
 
-export interface Model<P extends LanguageModelV1 = LanguageModelV1> {
+export interface Model<VercelModel extends LanguageModelV1 = LanguageModelV1> {
     readonly modelName: string;
     readonly provider: string;
     readonly displayName: string;
     isAvailable(): boolean;
-    getModel(): P;
+    getModel(): VercelModel;
 }
 
 export class OpenAIModel implements Model {
@@ -88,10 +88,8 @@ export class OllamaModel implements Model {
     }
 }
 
-const ALL_TESTABLE_MODELS = [new AzureOpenAIModel("gpt-4o")];
+const ALL_TESTABLE_MODELS: Model[] = [new AzureOpenAIModel("gpt-4o")];
 
-export type TestableModels = ReturnType<typeof getAvailableModels>;
-
-export function getAvailableModels() {
+export function getAvailableModels(): Model[] {
     return ALL_TESTABLE_MODELS.filter((model) => model.isAvailable());
 }
diff --git a/tests/accuracy/update-many.test.ts b/tests/accuracy/update-many.test.ts
@@ -1,5 +1,4 @@
 import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";
-import { getAvailableModels } from "./sdk/models.js";
 import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";
 
 function callsUpdateManyWithEmptyFilters(prompt: string): AccuracyTestConfig {
@@ -43,7 +42,7 @@ function callsUpdateManyWithFilters(prompt: string, filter: Record<string, unkno
     };
 }
 
-describeAccuracyTests(getAvailableModels(), [
+describeAccuracyTests([
     callsUpdateManyWithEmptyFilters(
         "Update all the documents in 'mflix.movies' namespace with a new field 'new_field' set to 1"
     ),

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,6 @@`
`1`	`1`	`import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";`
`2`		`-import { getAvailableModels } from "./sdk/models.js";`
`3`	`2`
`4`		`-describeAccuracyTests(getAvailableModels(), [`
	`3`	`+describeAccuracyTests([`
`5`	`4`	`{`
`6`	`5`	`prompt: "Group all the movies in 'mflix.movies' namespace by 'release_year' and give me a count of them",`
`7`	`6`	`expectedToolCalls: [`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,4 @@`
`1`	`1`	`import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js";`
`2`		`-import { getAvailableModels } from "./sdk/models.js";`
`3`	`2`	`import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js";`
`4`	`3`
`5`	`4`	`function callsCreateIndex(prompt: string, indexKeys: Record<string, unknown>): AccuracyTestConfig {`
`@@ -18,7 +17,7 @@ function callsCreateIndex(prompt: string, indexKeys: Record<string, unknown>): A`
`18`	`17`	`};`
`19`	`18`	`}`
`20`	`19`
`21`		`-describeAccuracyTests(getAvailableModels(), [`
	`20`	`+describeAccuracyTests([`
`22`	`21`	`callsCreateIndex(`
`23`	`22`	`"Create an index that covers the following query on 'mflix.movies' namespace - { \"release_year\": 1992 }",`
`24`	`23`	`{`