chore(tests): accuracy tests for MongoDB tools exposed by MCP server MCP-39

tests/accuracy/aggregate.test.ts

@@ -1,14 +1,21 @@
 import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
+import { ParameterScorers, withParameterScorer } from "./sdk/parameterScorer.js";
 
 describeAccuracyTests([
     {
         prompt: "Group all the movies in 'mflix.movies' namespace by 'release_year' and give me a count of them",
         expectedToolCalls: [
             {
                 toolName: "aggregate",
-                parameters: {
-                    pipeline: { $group: { _id: "$release_year", count: { $sum: 1 } } },
-                },
+                parameters: withParameterScorer(
+                    {
+                        database: "mflix",
+                        collection: "movies",
+                        pipeline: [{ $group: { _id: "$release_year", count: { $sum: 1 } } }],
+                    },
+                    // There should not be a $match at all hence the custom matcher
+                    ParameterScorers.noAdditionsAllowedForPaths(["pipeline.0.$match"])
+                ),
             },
         ],
     },

tests/accuracy/collectionSchema.test.ts

@@ -2,25 +2,25 @@ import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
 
 describeAccuracyTests([
     {
-        prompt: "Is there a title field in 'db1.coll1' namespace?",
+        prompt: "Is there a title field in 'mflix.movies' namespace?",
         expectedToolCalls: [
             {
                 toolName: "collection-schema",
                 parameters: {
-                    database: "db1",
-                    collection: "coll1",
+                    database: "mflix",
+                    collection: "movies",
                 },
             },
         ],
     },
     {
-        prompt: "What is the type of value stored in title field in coll1 collection in db1 database?",
+        prompt: "What is the type of value stored in title field in movies collection in mflix database?",
         expectedToolCalls: [
             {
                 toolName: "collection-schema",
                 parameters: {
-                    database: "db1",
-                    collection: "coll1",
+                    database: "mflix",
+                    collection: "movies",
                 },
             },
         ],

tests/accuracy/count.test.ts

@@ -1,15 +1,19 @@
 import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
+import { ParameterScorers, withParameterScorer } from "./sdk/parameterScorer.js";
 
 describeAccuracyTests([
     {
         prompt: "Count number of documents in 'mflix.movies' namespace.",
         expectedToolCalls: [
             {
                 toolName: "count",
-                parameters: {
-                    database: "mflix",
-                    collection: "movies",
-                },
+                parameters: withParameterScorer(
+                    {
+                        database: "mflix",
+                        collection: "movies",
+                    },
+                    ParameterScorers.emptyAdditionsAllowedForPaths(["query"])
+                ),
             },
         ],
     },
@@ -18,10 +22,13 @@ describeAccuracyTests([
         expectedToolCalls: [
             {
                 toolName: "count",
-                parameters: {
-                    database: "comics",
-                    collection: "characters",
-                },
+                parameters: withParameterScorer(
+                    {
+                        database: "comics",
+                        collection: "characters",
+                    },
+                    ParameterScorers.emptyAdditionsAllowedForPaths(["query"])
+                ),
             },
         ],
     },
@@ -30,11 +37,14 @@ describeAccuracyTests([
         expectedToolCalls: [
             {
                 toolName: "count",
-                parameters: {
-                    database: "mflix",
-                    collection: "movies",
-                    query: { runtime: { $lt: 100 } },
-                },
+                parameters: withParameterScorer(
+                    {
+                        database: "mflix",
+                        collection: "movies",
+                        query: { runtime: { $lt: 100 } },
+                    },
+                    ParameterScorers.noAdditionsAllowedForPaths(["query"])
+                ),
             },
         ],
     },

tests/accuracy/createIndex.test.ts

@@ -1,18 +1,22 @@
 import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
+import { ParameterScorers, withParameterScorer } from "./sdk/parameterScorer.js";
 
 describeAccuracyTests([
     {
         prompt: "Create an index that covers the following query on 'mflix.movies' namespace - { \"release_year\": 1992 }",
         expectedToolCalls: [
             {
                 toolName: "create-index",
-                parameters: {
-                    database: "mflix",
-                    collection: "movies",
-                    keys: {
-                        release_year: 1,
+                parameters: withParameterScorer(
+                    {
+                        database: "mflix",
+                        collection: "movies",
+                        keys: {
+                            release_year: 1,
+                        },
                     },
-                },
+                    ParameterScorers.noAdditionsAllowedForPaths(["keys"])
+                ),
             },
         ],
     },
@@ -21,13 +25,16 @@ describeAccuracyTests([
         expectedToolCalls: [
             {
                 toolName: "create-index",
-                parameters: {
-                    database: "mflix",
-                    collection: "movies",
-                    keys: {
-                        title: "text",
+                parameters: withParameterScorer(
+                    {
+                        database: "mflix",
+                        collection: "movies",
+                        keys: {
+                            title: "text",
+                        },
                     },
-                },
+                    ParameterScorers.noAdditionsAllowedForPaths(["keys"])
+                ),
             },
         ],
     },

tests/accuracy/deleteMany.test.ts

@@ -1,15 +1,19 @@
 import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
+import { ParameterScorers, withParameterScorer } from "./sdk/parameterScorer.js";
 
 describeAccuracyTests([
     {
         prompt: "Delete all the documents from 'mflix.movies' namespace",
         expectedToolCalls: [
             {
                 toolName: "delete-many",
-                parameters: {
-                    database: "mflix",
-                    collection: "movies",
-                },
+                parameters: withParameterScorer(
+                    {
+                        database: "mflix",
+                        collection: "movies",
+                    },
+                    ParameterScorers.emptyAdditionsAllowedForPaths(["filter"])
+                ),
             },
         ],
     },
@@ -18,10 +22,13 @@ describeAccuracyTests([
         expectedToolCalls: [
             {
                 toolName: "delete-many",
-                parameters: {
-                    database: "mflix",
-                    collection: "movies",
-                },
+                parameters: withParameterScorer(
+                    {
+                        database: "mflix",
+                        collection: "movies",
+                    },
+                    ParameterScorers.emptyAdditionsAllowedForPaths(["filter"])
+                ),
             },
         ],
     },
@@ -30,10 +37,14 @@ describeAccuracyTests([
         expectedToolCalls: [
             {
                 toolName: "delete-many",
-                parameters: {
-                    database: "mflix",
-                    collection: "movies",
-                },
+                parameters: withParameterScorer(
+                    {
+                        database: "mflix",
+                        collection: "movies",
+                        filter: { runtime: { $lt: 100 } },
+                    },
+                    ParameterScorers.noAdditionsAllowedForPaths(["filter"])
+                ),
             },
         ],
     },

tests/accuracy/explain.test.ts

@@ -1,4 +1,5 @@
 import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
+import { ParameterScorers, withParameterScorer } from "./sdk/parameterScorer.js";
 
 /**
  * None of these tests score a parameter match on any of the models, likely
@@ -11,62 +12,72 @@ describeAccuracyTests([
         expectedToolCalls: [
             {
                 toolName: "explain",
-                parameters: {
-                    database: "mflix",
-                    collection: "movies",
-                    method: [
-                        {
-                            name: "find",
-                            arguments: {
-                                filter: { release_year: 2020 },
+                parameters: withParameterScorer(
+                    {
+                        database: "mflix",
+                        collection: "movies",
+                        method: [
+                            {
+                                name: "find",
+                                arguments: {
+                                    filter: { release_year: 2020 },
+                                },
                             },
-                        },
-                    ],
-                },
+                        ],
+                    },
+                    // Any addition to method itself will essentially change the explain output
+                    ParameterScorers.noAdditionsAllowedForPaths(["method"])
+                ),
             },
         ],
     },
     {
-        prompt: `Will fetching documents, where release_year is 2020, from 'mflix.movies' namespace perform a collection scan?`,
+        prompt: `Will aggregating documents, where release_year is 2020, from 'mflix.movies' namespace perform a collection scan?`,
         expectedToolCalls: [
             {
                 toolName: "explain",
-                parameters: {
-                    database: "mflix",
-                    collection: "movies",
-                    method: [
-                        {
-                            name: "aggregate",
-                            arguments: {
-                                pipeline: [
-                                    {
-                                        $match: { release_year: 2020 },
-                                    },
-                                ],
+                parameters: withParameterScorer(
+                    {
+                        database: "mflix",
+                        collection: "movies",
+                        method: [
+                            {
+                                name: "aggregate",
+                                arguments: {
+                                    pipeline: [
+                                        {
+                                            $match: { release_year: 2020 },
+                                        },
+                                    ],
+                                },
                             },
-                        },
-                    ],
-                },
+                        ],
+                    },
+                    ParameterScorers.noAdditionsAllowedForPaths(["method"])
+                ),
             },
         ],
     },
     {
-        prompt: `Will fetching documents, where release_year is 2020, from 'mflix.movies' namespace perform a collection scan?`,
+        prompt: `Will counting documents, where release_year is 2020, from 'mflix.movies' namespace perform a collection scan?`,
         expectedToolCalls: [
             {
                 toolName: "explain",
-                parameters: {
-                    database: "mflix",
-                    collection: "movies",
-                    method: [
-                        {
-                            name: "count",
-                            arguments: {
-                                query: { release_year: 2020 },
+                parameters: withParameterScorer(
+                    {
+                        database: "mflix",
+                        collection: "movies",
+                        method: [
+                            {
+                                name: "count",
+                                arguments: {
+                                    query: { release_year: 2020 },
+                                },
                             },
-                        },
-                    ],
-                },
+                        ],
+                    },
+                    ParameterScorers.noAdditionsAllowedForPaths(["method"])
+                ),
             },
         ],
     },