From 194f6c42a2aabbec218537b0f3397752ba0f395c Mon Sep 17 00:00:00 2001 From: gagik Date: Wed, 10 Sep 2025 14:32:04 +0200 Subject: [PATCH 1/5] chore: add all eval cases and CSV conversion script --- package-lock.json | 96 ++++++ packages/compass-assistant/.gitignore | 1 + packages/compass-assistant/package.json | 4 +- .../scripts/convert-csv-to-eval-cases.ts | 163 ++++++++++ .../test/eval-cases/aggregation-pipeline.ts | 4 +- .../test/eval-cases/atlas-search.ts | 4 +- .../test/eval-cases/generated-cases.ts | 303 ++++++++++++++++++ .../test/eval-cases/index.ts | 14 +- .../test/eval-cases/model-data.ts | 4 +- 9 files changed, 577 insertions(+), 16 deletions(-) create mode 100644 packages/compass-assistant/.gitignore create mode 100644 packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts create mode 100644 packages/compass-assistant/test/eval-cases/generated-cases.ts diff --git a/package-lock.json b/package-lock.json index c4ecc5b2beb..98af0bce29a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6031,6 +6031,21 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@fast-csv/parse": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-5.0.5.tgz", + "integrity": "sha512-M0IbaXZDbxfOnpVE5Kps/a6FGlILLhtLsvWd9qNH3d2TxNnpbNkFf3KD26OmJX6MHq7PdQAl5htStDwnuwHx6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, "node_modules/@fastify/busboy": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", @@ -32160,6 +32175,13 @@ "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", "integrity": "sha1-gteb/zCmfEAF/9XiUVMArZyk168=" }, + "node_modules/lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", + "dev": true, + "license": "MIT" + }, "node_modules/lodash.flattendeep": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", @@ -32180,12 +32202,33 @@ "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA=" }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==", + "dev": true, + "license": "MIT" + }, "node_modules/lodash.ismatch": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz", "integrity": "sha512-fPMfXjGQEV9Xsq/8MTSgUf255gawYRbjwMyDbcvDhXgV7enSZA0hynz6vMPnpAb5iONEzBHBPsT+0zes5Z301g==", "dev": true }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==", + "dev": true, + "license": "MIT" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -32210,6 +32253,13 @@ "lodash._reinterpolate": "^3.0.0" } }, + "node_modules/lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==", + "dev": true, + "license": "MIT" + }, "node_modules/lodash.zip": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.zip/-/lodash.zip-4.2.0.tgz", @@ -47471,6 +47521,7 @@ "use-sync-external-store": "^1.5.0" }, "devDependencies": { + "@fast-csv/parse": "^5.0.5", "@mongodb-js/eslint-config-compass": "^1.4.8", "@mongodb-js/mocha-config-compass": "^1.7.1", "@mongodb-js/prettier-config-compass": "^1.2.8", @@ -57964,6 +58015,20 @@ "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz", "integrity": "sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==" }, + "@fast-csv/parse": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-5.0.5.tgz", + "integrity": "sha512-M0IbaXZDbxfOnpVE5Kps/a6FGlILLhtLsvWd9qNH3d2TxNnpbNkFf3KD26OmJX6MHq7PdQAl5htStDwnuwHx6w==", + "dev": true, + "requires": { + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, "@fastify/busboy": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", @@ -61015,6 +61080,7 @@ "version": "file:packages/compass-assistant", "requires": { "@ai-sdk/openai": "^2.0.4", + "@fast-csv/parse": "^5.0.5", "@mongodb-js/atlas-service": "^0.57.1", "@mongodb-js/compass-app-registry": "^9.4.21", "@mongodb-js/compass-components": "^1.50.1", @@ -83854,6 +83920,12 @@ "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", "integrity": "sha1-gteb/zCmfEAF/9XiUVMArZyk168=" }, + "lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", + "dev": true + }, "lodash.flattendeep": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", @@ -83874,12 +83946,30 @@ "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA=" }, + "lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==", + "dev": true + }, "lodash.ismatch": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz", "integrity": "sha512-fPMfXjGQEV9Xsq/8MTSgUf255gawYRbjwMyDbcvDhXgV7enSZA0hynz6vMPnpAb5iONEzBHBPsT+0zes5Z301g==", "dev": true }, + "lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==", + "dev": true + }, + "lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==", + "dev": true + }, "lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -83904,6 +83994,12 @@ "lodash._reinterpolate": "^3.0.0" } }, + "lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==", + "dev": true + }, "lodash.zip": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.zip/-/lodash.zip-4.2.0.tgz", diff --git a/packages/compass-assistant/.gitignore b/packages/compass-assistant/.gitignore new file mode 100644 index 00000000000..e6a21cdb339 --- /dev/null +++ b/packages/compass-assistant/.gitignore @@ -0,0 +1 @@ +test/eval-cases/eval_cases.csv \ No newline at end of file diff --git a/packages/compass-assistant/package.json b/packages/compass-assistant/package.json index 32e9dc4b1bb..6c8c19deb9b 100644 --- a/packages/compass-assistant/package.json +++ b/packages/compass-assistant/package.json @@ -48,7 +48,8 @@ "test-watch": "npm run test -- --watch", "test-ci": "npm run test-cov", "test-ci-electron": "npm run test-electron", - "reformat": "npm run eslint . -- --fix && npm run prettier -- --write ." + "reformat": "npm run eslint . -- --fix && npm run prettier -- --write .", + "convert-eval-cases": "ts-node scripts/convert-csv-to-eval-cases.ts" }, "dependencies": { "@ai-sdk/openai": "^2.0.4", @@ -67,6 +68,7 @@ "use-sync-external-store": "^1.5.0" }, "devDependencies": { + "@fast-csv/parse": "^5.0.5", "@mongodb-js/eslint-config-compass": "^1.4.8", "@mongodb-js/mocha-config-compass": "^1.7.1", "@mongodb-js/prettier-config-compass": "^1.2.8", diff --git a/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts b/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts new file mode 100644 index 00000000000..dabcac604d6 --- /dev/null +++ b/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts @@ -0,0 +1,163 @@ +#!/usr/bin/env ts-node +/* eslint-disable no-console */ +// eslint-disable-next-line @typescript-eslint/no-restricted-imports +import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'fs'; +// eslint-disable-next-line @typescript-eslint/no-restricted-imports +import { join, resolve } from 'path'; +import { parse } from '@fast-csv/parse'; + +/** This is copied from the Compass Assistant PD Eval Cases */ +type CSVRow = { + 'Your Name': string; + 'Interaction Type\n(can add other types)': string; + 'Input\nHighlighting key: \nHardcoded\n\nContextual passed from client to assistant\n\nUser-entered': string; + 'Expected Output\n(target 100-200 words, okay to go over if needed)': string; + 'Expected Links\n(comma separated please)': string; + Notes: string; +}; + +type SimpleEvalCase = { + name?: string; + input: string; + expected: string; + expectedSources?: string[]; +}; + +function escapeString(str: string): string { + return str + .replace(/\\/g, '\\\\') + .replace(/`/g, '\\`') + .replace(/\${/g, '\\${') + .replace(/\r?\n/g, '\\n') // Handle newlines + .replace(/[\u200B-\u200D\uFEFF\u2028\u2029]/g, '') // Remove zero-width spaces and other invisible characters + .replace(/[^\S ]/g, ' ') // Replace all whitespace except normal spaces with spaces + .replace(/\s+/g, ' ') // Collapse multiple spaces + .trim(); // Remove leading/trailing whitespace +} + +function generateEvalCaseFile(cases: SimpleEvalCase[]): string { + const caseDefinitions = cases + .map((evalCase) => { + const sourcesPart = + evalCase.expectedSources && evalCase.expectedSources.length > 0 + ? ` expectedSources: [\n ${evalCase.expectedSources + .map((source) => `'${escapeString(source)}'`) + .join(',\n ')},\n ],` + : ''; + + return ` { + input: \`${escapeString(evalCase.input)}\`, + expected: \`${escapeString(evalCase.expected)}\`,${ + sourcesPart ? '\n' + sourcesPart : '' + } + }`; + }) + .join(',\n'); + + return `/** This file is auto-generated by the convert-csv-to-eval-cases script. +Do not modify this file manually. */ +import type { SimpleEvalCase } from '../assistant.eval'; + +export const generatedEvalCases: SimpleEvalCase[] = [ +${caseDefinitions}, +]; +`; +} + +async function convertCSVToEvalCases() { + const scriptDir = __dirname; + const csvFilePath = resolve(scriptDir, '../test/eval-cases/eval_cases.csv'); + // Check that the CSV file exists + if (!existsSync(csvFilePath)) { + console.error( + `The CSV file does not exist: ${csvFilePath}. Please import it and try again.` + ); + process.exit(1); + } + const outputDir = resolve(scriptDir, '../test/eval-cases'); + + console.log('Converting CSV to eval cases...'); + console.log(`Reading from: ${csvFilePath}`); + console.log(`Output directory: ${outputDir}`); + + // Ensure output directory exists + mkdirSync(outputDir, { recursive: true }); + + const allCases: SimpleEvalCase[] = []; + + // Read and parse CSV using async/await + const csvContent = readFileSync(csvFilePath, 'utf8'); + + const rows = await new Promise((resolve, reject) => { + const results: CSVRow[] = []; + const stream = parse({ + headers: true, + }) + .on('data', (row: CSVRow) => results.push(row)) + .on('end', () => resolve(results)) + .on('error', reject); + + stream.write(csvContent); + stream.end(); + }); + + // Process rows + for (const row of rows) { + // Skip empty rows or header-like rows + const input = + row[ + 'Input\nHighlighting key: \nHardcoded\n\nContextual passed from client to assistant\n\nUser-entered' + ]?.trim(); + const expected = + row[ + 'Expected Output\n(target 100-200 words, okay to go over if needed)' + ]?.trim(); + const yourName = row['Your Name']?.trim(); + const interactionType = + row['Interaction Type\n(can add other types)']?.trim(); + + if (!input || !expected || !yourName || !interactionType) { + continue; // Skip incomplete rows + } + + // Parse expected sources + const expectedLinksRaw = + row['Expected Links\n(comma separated please)']?.trim(); + let expectedSources: string[] = []; + + if (expectedLinksRaw) { + expectedSources = expectedLinksRaw + .replace(/\r?\n/g, ' ') // Replace newlines with spaces first + .split(',') + .map((link) => link.trim()) + .filter((link) => link && link.startsWith('http')); + } + + const evalCase: SimpleEvalCase = { + input, + expected, + ...(expectedSources.length > 0 && { expectedSources }), + }; + + allCases.push(evalCase); + } + + console.log(`\nProcessed ${allCases.length} cases`); + + // Generate single file with all cases + const filename = 'generated-cases'; + const filepath = join(outputDir, `${filename}.ts`); + const content = generateEvalCaseFile(allCases); + + writeFileSync(filepath, content, 'utf8'); + console.log(`✓ Generated ${filename}.ts with ${allCases.length} cases`); + + console.log('\n✅ Conversion completed successfully!'); +} + +convertCSVToEvalCases().catch((error) => { + console.error('❌ Conversion failed:', error); + process.exit(1); +}); + +export { convertCSVToEvalCases }; diff --git a/packages/compass-assistant/test/eval-cases/aggregation-pipeline.ts b/packages/compass-assistant/test/eval-cases/aggregation-pipeline.ts index a3d7590bf70..dd962519b1c 100644 --- a/packages/compass-assistant/test/eval-cases/aggregation-pipeline.ts +++ b/packages/compass-assistant/test/eval-cases/aggregation-pipeline.ts @@ -1,6 +1,6 @@ import type { SimpleEvalCase } from '../assistant.eval'; -const evalCases: SimpleEvalCase[] = [ +export const aggregationPipelineEvalCases: SimpleEvalCase[] = [ { input: 'What is an aggregation pipeline?', expected: `The aggregation pipeline in MongoDB is a framework for data processing and transformation. It consists of a sequence of stages, where each stage performs an operation on the input documents and passes the results to the next stage. Common operations include filtering, grouping, projecting, joining, and calculating values. Aggregation pipelines are powerful for data analysis, reporting, and transformation tasks in MongoDB. @@ -37,5 +37,3 @@ db.orders.aggregate([ ], }, ]; - -export default evalCases; diff --git a/packages/compass-assistant/test/eval-cases/atlas-search.ts b/packages/compass-assistant/test/eval-cases/atlas-search.ts index 75210fb4934..f6b63c81fbf 100644 --- a/packages/compass-assistant/test/eval-cases/atlas-search.ts +++ b/packages/compass-assistant/test/eval-cases/atlas-search.ts @@ -1,6 +1,6 @@ import type { SimpleEvalCase } from '../assistant.eval'; -const evalCases: SimpleEvalCase[] = [ +export const atlasSearchEvalCases: SimpleEvalCase[] = [ { input: 'How can I filter docs before running a $search query?', expected: @@ -11,5 +11,3 @@ const evalCases: SimpleEvalCase[] = [ ], }, ]; - -export default evalCases; diff --git a/packages/compass-assistant/test/eval-cases/generated-cases.ts b/packages/compass-assistant/test/eval-cases/generated-cases.ts new file mode 100644 index 00000000000..c46ae1c311f --- /dev/null +++ b/packages/compass-assistant/test/eval-cases/generated-cases.ts @@ -0,0 +1,303 @@ +/** This file is auto-generated by the convert-csv-to-eval-cases script. +Do not modify this file manually. */ +import type { SimpleEvalCase } from '../assistant.eval'; + +export const generatedEvalCases: SimpleEvalCase[] = [ + { + input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nConnection string (password redacted): \nmongodb+srv://betsy:*****@cluster0.pndqllj.mongodb.net/\n\nError message:\nAuthentication failed.\n\nThere was a problem connecting to cluster0.pndqllj.mongodb.net`, + expected: `To help troubleshoot the error, please explore the following solutions:\nEnsure the correct Username and Password are provided.\n\nEnsure that the MongoDB database user you are authenticating as exists.\n\nVerify the Authentication Database and authentication mechanism.\n\nVerify that your selected authentication mechanism is supported by your MongoDB database.\n\nIf connecting to an Atlas deployment, check the Connect modal in the Atlas UI for connection details and code snippets.\n\n\nThe Compass logs can provide additional information on connection errors. You may find more detailed error messages to help diagnose your issue.\n\nOnce a successful connection has been established, please Save this connection as a Favorite to ease the connection process in the future.`, + expectedSources: [ + 'https://www.mongodb.com/docs/compass/current/troubleshooting/connection-errors', + 'https://www.mongodb.com/docs/atlas/troubleshoot-connection', + 'https://www.mongodb.com/docs/atlas/compass-connection', + 'https://www.mongodb.com/docs/compass/current/connect', + 'https://www.mongodb.com/docs/compass/current/troubleshooting/logs https://www.mongodb.com/docs/compass/current/troubleshooting/logs/', + ], + }, + { + input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nConnection string (password redacted): \nmongodb+srv://betsy:*****@cluster0.pndqllj.mongodb.net/\n\nError message:\nquerySrv ENOTFOUND _mongodb._tcp.cluster0.pndollj.mongodb.net\n\nThere was a problem connecting to cluster0.pndollj.mongodb.net`, + expected: `This connection error is often the result of a DNS resolution issue. To help troubleshoot the error, please explore the following solutions: \nDouble-check that you entered the correct cluster address in Compass.\n\nOpen a terminal or command prompt and run “nslookup” on the DNS SRV record for your cluster. If this fails, your DNS server may be blocking or unable to resolve the address. Also ensure DNS TXT results are allowed to be returned.\n\nEnsure your cluster still exists and is not paused. DNS resolution will fail if the cluster you’re trying to connect to has been deleted.\n\nEnsure your computer has a stable internet connection. \n\nCheck the Compass log for additional information. \n\n\nThe Compass logs can provide additional information on connection errors. You may find more detailed error messages to help diagnose your issue.`, + expectedSources: [ + 'https://www.mongodb.com/docs/compass/current/troubleshooting/connection-errors', + 'https://www.mongodb.com/docs/atlas/troubleshoot-connection', + 'https://www.mongodb.com/docs/atlas/compass-connection', + 'https://www.mongodb.com/docs/compass/current/connect', + 'https://www.mongodb.com/docs/compass/current/troubleshooting/logs', + ], + }, + { + input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nConnection string (password redacted): \nmongodb+srv://betsy:*****@cluster0.pndqllj.mongodb.net/\n\nError message:\nconnect ENETUNREACH 89.193.144.189:27017\n\nThere was a problem connecting to App`, + expected: `This connection error indicates that the destination network is unreachable. To help troubleshoot the error, please explore the following solutions: \nCheck the Compass log\n\nEnsure your computer has a stable internet connection. \n\nDouble-check that you entered the correct cluster address in Compass.\n\nCheck your VPN, firewall, and network settings. It’s possible that your configuration is blocking the request. \n\nEnsure your cluster still exists and is not paused.\n\n\nThe Compass logs can provide additional information on connection errors. You may find more detailed error messages to help diagnose your issue.`, + expectedSources: [ + 'https://www.mongodb.com/docs/compass/current/troubleshooting/logs/', + ], + }, + { + input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nConnection string (password redacted): \nmongodb+srv://betsy:*****@cluster0.pndqllj.mongodb.net/\n\nError message:\nquerySrv ECONNREFUSED`, + expected: `This is possibly a DNS (network) or MongoDB Driver issue. Given that DNS SRV queries sometimes behave differently than other DNS lookup types, we suggest the following next steps:\n\nLocate your "legacy" connection string in Atlas\n\nThe legacy connection string will start with mongodb:// instead of mongodb+srv://\n\nAttempt to connect using the legacy connection string\n\nIf the legacy connection string works, ensure you are using the latest version of MongoDB Compass, share the findings of what works and what does not work with your network administrator\n\nIf the legacy connection string does not work, follow troubleshooting steps for the next error (the error message is expected to change)`, + expectedSources: [ + 'https://www.mongodb.com/docs/compass/current/troubleshooting/connection-errors', + 'https://www.mongodb.com/docs/atlas/troubleshoot-connection', + 'https://www.mongodb.com/docs/atlas/compass-connection', + 'https://www.mongodb.com/docs/compass/current/connect', + 'https://www.mongodb.com/docs/compass/current/troubleshooting/logs', + ], + }, + { + input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nAuth mechanism: \n\nError message:\nconnect ECONNREFUSED`, + expected: `This is typically an authentication issue.\nCheck the Compass log\n\nConfirm that the connection string is correct (username, password, hostname, port, connection options including TLS/SSL or X.509 certificates, etc).\n\nConfirm the correct authentication method is being used (TLS/SSL, LDAP, X.509, Kerberos, OIDC)\n\nConfirm the authentication server on the customer side has been set up properly for the user (LDAP server, Kerberos server, etc).\n\nFor Atlas connections using the Standard Connection, ensure the IP address is added to the Network Access list.`, + expectedSources: [ + 'https://www.mongodb.com/docs/compass/current/troubleshooting/logs/', + ], + }, + { + input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nAuth mechanism: \n\nError message:`, + expected: `Cannot connect in an Atlas multi-region deployment setup\nCheck the Compass log\n\nConfirm that the connection string is correct (username, password, hostname, port, connection options including TLS/SSL certificates, etc).\n\nMake sure the number of connections are not at the maximum number for the tier.\n\nMake sure the Cloud provider on the customer side has peering VPCs set up. These settings are made outside of Atlas and may need the help of the customer's IT team.\n\nNetwork packet inspection and traceroute may be needed.`, + }, + { + input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n \n\nAuth mechanism: \n\nError message:`, + expected: `Cannot connect to an individual Atlas host when using Private Endpoint.\nCheck the Compass log\n\nConfirm that the connection string is correct (username, password, hostname, port, connection options including TLS/SSL certificates, etc). Atlas Private Endpoint connections use ports starting at 1024.\n\nVerify VPN and firewalls are not preventing a connection from the IP address and ports.\n\nMake sure the number of connections on the Server are less than the maximum number for the tier; otherwise, connections will not be successful.\n\nEnsure DNS SRV and DNS TXT records are successful and fast, say, 0.1 seconds.`, + }, + { + input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\nAuth mechanism: \n\nError message:`, + expected: `Cannot connect to Atlas using VPC connections.\nCheck the Compass log\n\nConfirm that the connection string is correct (username, password, hostname, port, connection options including TLS/SSL certificates, etc). Atlas Private Endpoint connections use ports starting at 1024.\n\nVerify VPN and firewalls are not preventing a connection from the IP address and ports.\n\nMake sure the number of connections on the Server are less than the maximum number for the tier; otherwise, connections will not be successful.\n\nEnsure DNS SRV and DNS TXT records are successful and fast, say, 0.1 seconds.`, + }, + { + input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nAuth mechanism: \n\nError message:`, + expected: `Cannot connect with an AWS Lambda application.\nCheck the Compass log\n\nUse the Reachability Analyzer for troubleshooting\n\n[FH: I think the KB article should be AI ingested. The other troubleshooting steps are extensive]`, + }, + { + input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\nAuth mechanism: \nUsername/password\n\nError message:\nserverSelectionTimeout\n\nThere was a problem connecting to cluster0.pndollj.mongodb.net`, + expected: `Check the Compass log\n\nConfirm that the connection string is correct (username, password, hostname, port, connection options including TLS/SSL or X.509 certificates, etc).\n\nOpen a terminal or command prompt and run “nslookup” on the DNS SRV record for your cluster. If this fails, your DNS server may be blocking or unable to resolve the address. Also ensure DNS TXT results are allowed to be returned.\n\nConfirm the correct authentication method is being used (TLS/SSL, LDAP, X.509, Kerberos, OIDC)\n\nConfirm the authentication server on the customer side has been set up properly for the user (LDAP server, Kerberos server, etc).\n\nFor Atlas connections using the Standard Connection, ensure the IP address is added to the Network Access list.\n\n\nThis error is generic and not specific enough to identify the connection issue. Further explanation here:\nHow to Troubleshoot and Resolve the 'MongoServerSelectionError' in MongoDB`, + }, + { + input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\nAuth mechanism: \nOIDC\n\nError message:\n\nAuthentication failed. There was a problem connecting to `, + expected: `It sounds like you successfully authenticated with your identity provider, but the MongoDB server is misconfigured and did not accept your authentication request. To resolve this issue:\nCheck the Compass log\n\nCheck whether you are able to connect to the cluster using the MongoDB Shell\n\nGather information using mongosh --oidcDumpTokens \n\nReach out to the cluster administrator with the Compass log, output from mongosh, and the time at which you connected, and the versions of Compass and mongosh you used`, + }, + { + input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\nAuth mechanism: \nOIDC\n\nError message:\n\nCompass hangs and opens a browser windows that says “400 Bad Request”.`, + expected: `It sounds like your MongoDB server or identity provider is not configured correctly. To resolve this issue:\nCheck the Compass log\n\nCheck whether you are able to connect to the cluster using the MongoDB Shell\n\nReach out to the cluster administrator with the Compass log, output from mongosh, and the time at which you connected, and the versions of Compass and mongosh you used`, + }, + { + input: `Compass hardcoded prompt: \nGiven the MongoDB explain plan output below, provide a concise human readable explanation that explains the query execution plan and highlights aspects of the plan that might impact query performance. Respond with as much concision and clarity as possible. \n\nIf a clear optimization should be made, please suggest the optimization and describe how it can be accomplished in MongoDB Compass. Do not advise users to create indexes without weighing the pros and cons. \n\nExplain output: \n{"explainVersion":"1","stages":[{"$cursor":{"queryPlanner":{"namespace":"restaurants.nyc","indexFilterSet":false,"parsedQuery":{"cuisine":{"$eq":"Italian"}},"queryHash":"0D9A721D","planCacheKey":"0BC4F8A9","optimizationTimeMillis":0,"maxIndexedOrSolutionsReached":false,"maxIndexedAndSolutionsReached":false,"maxScansToExplodeReached":false,"winningPlan":{"stage":"PROJECTION_SIMPLE","transformBy":{"borough":1,"_id":0},"inputStage":{"stage":"FETCH","inputStage":{"stage":"IXSCAN","keyPattern":{"cuisine":-1},"indexName":"cuisine_-1","isMultiKey":false,"multiKeyPaths":{"cuisine":[]},"isUnique":false,"isSparse":false,"isPartial":false,"indexVersion":2,"direction":"forward","indexBounds":{"cuisine":["[\\"Italian\\", \\"Italian\\"]"]}}}},"rejectedPlans":[]},"executionStats":{"executionSuccess":true,"nReturned":43207,"executionTimeMillis":1028,"totalKeysExamined":43207,"totalDocsExamined":43207,"executionStages":{"stage":"PROJECTION_SIMPLE","nReturned":43207,"executionTimeMillisEstimate":888,"works":43208,"advanced":43207,"needTime":0,"needYield":0,"saveState":80,"restoreState":80,"isEOF":1,"transformBy":{"borough":1,"_id":0},"inputStage":{"stage":"FETCH","nReturned":43207,"executionTimeMillisEstimate":885,"works":43208,"advanced":43207,"needTime":0,"needYield":0,"saveState":80,"restoreState":80,"isEOF":1,"docsExamined":43207,"alreadyHasObj":0,"inputStage":{"stage":"IXSCAN","nReturned":43207,"executionTimeMillisEstimate":48,"works":43208,"advanced":43207,"needTime":0,"needYield":0,"saveState":80,"restoreState":80,"isEOF":1,"keyPattern":{"cuisine":-1},"indexName":"cuisine_-1","isMultiKey":false,"multiKeyPaths":{"cuisine":[]},"isUnique":false,"isSparse":false,"isPartial":false,"indexVersion":2,"direction":"forward","indexBounds":{"cuisine":["[\\"Italian\\", \\"Italian\\"]"]},"keysExamined":43207,"seeks":1,"dupsTested":0,"dupsDropped":0}}}}}}, "nReturned":43207,"executionTimeMillisEstimate":1004},{"$group":{"_id":"$borough","count":{"$sum":{"$const":1}}},"maxAccumulatorMemoryUsageBytes":{"count":480},"totalOutputDataSizeBytes":1254,"usedDisk":false,"spills":0,"spilledDataStorageSize":0,"numBytesSpilledEstimate":0,"spilledRecords":0,"nReturned":5,"executionTimeMillisEstimate":1019}],"serverInfo":{"host":"atlas-mk8saw-shard-00-01.pndqllj.mongodb.net","port":27017,"version":"7.0.21","gitVersion":"a47b62aff2bae1914085c3ef1d90fc099acf000c"},"serverParameters":{"internalQueryFacetBufferSizeBytes":104857600,"internalQueryFacetMaxOutputDocSizeBytes":104857600,"internalLookupStageIntermediateDocumentMaxSizeBytes":104857600,"internalDocumentSourceGroupMaxMemoryBytes":104857600,"internalQueryMaxBlockingSortMemoryUsageBytes":104857600,"internalQueryProhibitBlockingMergeOnMongoS":0,"internalQueryMaxAddToSetBytes":104857600,"internalDocumentSourceSetWindowFieldsMaxMemoryBytes":104857600,"internalQueryFrameworkControl":"forceClassicEngine"},"command":{"aggregate":"nyc","pipeline":[{"$match":{"cuisine":"Italian"}},{"$group":{"_id":"$borough","count":{"$sum":1}}}],"cursor":{},"maxTimeMS":60000,"$db":"restaurants"},"ok":1,"$clusterTime":{"clusterTime":{"$timestamp":"7522966816189054977"},"signature":{"hash":"dhZ8X27p5jRJo1x/U9ZpF8L4eSc=","keyId":{"low":1,"high":1737746411,"unsigned":false}}},"operationTime":{"$timestamp":"7522966816189054977"}}`, + expected: `Aggregation Pipeline Recap \nThis pipeline filters documents where cuisine is "Italian", then groups them by borough, counting how many Italian restaurants exist in each borough.\nExecution Plan Breakdown\n1. Stage 1 – $match: { cuisine: "Italian" }\nMongoDB uses the cuisine_-1 index to identify matching documents (IXSCAN).\n\nHowever, it performs a FETCH stage afterward, meaning it reads full documents from disk—43,207 in total.\n\nWhy? Because the index does not contain the borough field, which is needed for grouping.\n\n2. Stage 2 – $group by borough\nPerformed in-memory, no disk spill (usedDisk: false).\n\nOnly 5 groups returned, minimal memory used (480 bytes).\n\nPerformance Observations \nKeys examined: 43,207\n\nDocuments examined: 43,207\n\nExecution time: ~1s\n\nAlthough the IXSCAN is efficient, the full document FETCH is a performance bottleneck in this case. The query is not covered because not all of the fields used in the aggregation are part of an index.\nOptimization Suggestion\nTo eliminate the FETCH stage and significantly reduce I/O, you can create a compound index that includes both cuisine and borough. If you choose to create the compound index, ensure that the index fields are selected in this order. The order of fields in a compound index is very important. Here, cuisine is first used to filter the documents, so it must come before borough. \nAlong the same lines, if you create this compound index, you no longer need a separate index on just the cuisine field. This is because MongoDB can use the prefixes of a compound index to optimize query execution. A query filtering only on cuisine can efficiently use the prefix of the compound index, making a standalone index on cuisine redundant. \nPlease note that creating an unnecessary index may be detrimental to your application. New indexes require additional index storage and increase the overhead for write operations. \nConsider the pros and cons carefully when deciding whether or not to create a new index. Also, think through which index makes the most sense for your application in the context of other queries frequently run. Please ask follow up questions if you’d like further guidance.`, + expectedSources: [ + 'https://www.mongodb.com/docs/manual/tutorial/analyze-query-plan/', + 'https://www.mongodb.com/docs/manual/reference/explain-results', + 'https://www.mongodb.com/docs/manual/faq/indexes', + 'https://www.mongodb.com/docs/manual/tutorial/equality-sort-range-guideline/#std-label-esr-indexing-guideline', + 'https://www.mongodb.com/docs/manual/core/indexes/index-types/index-compound/#index-prefixes', + ], + }, + { + input: `Compass hardcoded prompt: \nGiven the MongoDB explain plan output below, provide a concise human readable explanation that explains the query execution plan and highlights aspects of the plan that might impact query performance. Respond with as much concision and clarity as possible. \n\nIf a clear optimization should be made, please suggest the optimization and describe how it can be accomplished in MongoDB Compass. Do not advise users to create indexes without weighing the pros and cons. \n\nExplain output: \n{"explainVersion":"1","queryPlanner":{"namespace":"restaurants.nyc","indexFilterSet":false,"parsedQuery":{"category":{"$eq":"low_rated"}},"queryHash":"2EBC9A51","planCacheKey":"2EBC9A51","optimizationTimeMillis":0,"maxIndexedOrSolutionsReached":false,"maxIndexedAndSolutionsReached":false,"maxScansToExplodeReached":false,"winningPlan":{"stage":"COLLSCAN","filter":{"category":{"$eq":"low_rated"}},"direction":"forward"},"rejectedPlans":[]},"executionStats":{"executionSuccess":true,"nReturned":759349,"executionTimeMillis":1233,"totalKeysExamined":0,"totalDocsExamined":998014,"executionStages":{"stage":"COLLSCAN","filter":{"category":{"$eq":"low_rated"}},"nReturned":759349,"executionTimeMillisEstimate":232,"works":998015,"advanced":759349,"needTime":238665,"needYield":0,"saveState":998,"restoreState":998,"isEOF":1,"direction":"forward","docsExamined":998014}},"command":{"find":"nyc","filter":{"category":"low_rated"},"skip":0,"limit":0,"maxTimeMS":60000,"$db":"restaurants"},"serverInfo":{"host":"atlas-mk8saw-shard-00-01.pndqllj.mongodb.net","port":27017,"version":"7.0.22","gitVersion":"6660d93e97cdb0dc35d8ceb699c511f5850278a5"},"serverParameters":{"internalQueryFacetBufferSizeBytes":104857600,"internalQueryFacetMaxOutputDocSizeBytes":104857600,"internalLookupStageIntermediateDocumentMaxSizeBytes":104857600,"internalDocumentSourceGroupMaxMemoryBytes":104857600,"internalQueryMaxBlockingSortMemoryUsageBytes":104857600,"internalQueryProhibitBlockingMergeOnMongoS":0,"internalQueryMaxAddToSetBytes":104857600,"internalDocumentSourceSetWindowFieldsMaxMemoryBytes":104857600,"internalQueryFrameworkControl":"forceClassicEngine"},"ok":1,"$clusterTime":{"clusterTime":{"$timestamp":"7537442131581861889"},"signature":{"hash":"vkFwzpNfNq+HU/KcFnxAhplnBzA=","keyId":{"low":1,"high":1745504880,"unsigned":false}}},"operationTime":{"$timestamp":"7537442131581861889"}}`, + expected: `This query is slow because it scans every single document in your collection.\nQuery Execution Analysis\nYour query on the restaurants.nyc collection searched for documents where the category field equals "low_rated".\nThe execution plan shows a COLLSCAN, which means a Collection Scan. This is the least efficient way to run a query. MongoDB had to load and inspect every one of the 998,014 documents in your collection to find the ones that matched. This process took 1,233 ms (over 1.2 seconds).\nThink of it like trying to find every mention of a word in a book by reading it from cover to cover instead of using the index in the back.\nOptimization Suggestion: Add an Index\nTo speed this up, you could create an index on the category field. \nPros:\n\nFaster Reads: Queries filtering by category will be dramatically faster.\n\nBetter Scaling: The query will stay fast even as your collection grows.\n\nCons:\n\nSlightly Slower Writes: Every time you add, update, or delete a document, the index must also be updated. This adds overhead.\n\nMore Storage: The index itself will take up additional disk space.\n\nConsider the pros and cons carefully when deciding whether or not to create a new index. Also, think through which index makes the most sense for your application in the context of other queries frequently run. Please ask follow up questions if you’d like further guidance.\nHow to Create the Index in MongoDB Compass\nGo to the restaurants.nyc collection.\n\nClick the Indexes tab.\n\nClick the Create button.\n\nIn the Fields section, enter category as the field name.\n\nClick Create.\n\nAfter creating the index, running the query again will be much faster and the explain plan will show an IXSCAN (Index Scan), confirming it's working correctly.`, + expectedSources: [ + 'https://www.mongodb.com/docs/manual/tutorial/analyze-query-plan/', + 'https://www.mongodb.com/docs/manual/reference/explain-results', + 'https://www.mongodb.com/docs/manual/faq/indexes', + ], + }, + { + input: `How can I filter docs before running a $search query?`, + expected: `Because the $search stage must be the first stage in an \naggregation pipeline, you cannot pre-filter documents with a preceding $match stage. Instead, filtering should be performed within the $search stage using the filter clause of the compound operator. This allows you to apply predicate queries (e.g., on ranges, dates, or specific terms) to narrow down the dataset before the main query clauses (must or should) are executed. Alternatively, you can filter documents by creating a View—a partial index of your collection that pre-queries and filters out unwanted documents. Note that users need createCollection privileges to build views.`, + expectedSources: [ + 'https://www.mongodb.com/docs/atlas/atlas-search/compound/#options https://www.mongodb.com/docs/atlas/atlas-search/transform-documents-collections/#example--filter-documents', + ], + }, + { + input: `What is the $search stage?`, + expected: `The $search stage is a part of MongoDB Atlas Search, allowing users to perform full-text searches and sophisticated query expressions on text data, numerical fields, metadata, and more. It integrates seamlessly with MongoDB's aggregation framework as the first stage in a pipeline, enabling complex querying and transformation workflows.`, + expectedSources: [ + 'https://www.mongodb.com/docs/atlas/atlas-search/#what-is-fts-', + ], + }, + { + input: `Can $search work with regular MongoDB indexes?`, + expected: `No, $search cannot work with standard MongoDB indexes. It requires a dedicated Atlas Search Index that is specifically optimized for search operations and supports an array of features like text, autocomplete, and facet searches.`, + expectedSources: [ + 'https://www.mongodb.com/docs/atlas/atlas-search/index-definitions/#index-reference', + ], + }, + { + input: `How do I sort query results using $search?`, + expected: `To sort query results using $search, you include the sort parameter within the $search stage. Numeric, date, ObjectId, boolean and UUID field types support sorting natively. For string fields, you must map the field as the token type in your Atlas Search Index to enable proper sorting.`, + expectedSources: [ + 'https://www.mongodb.com/docs/atlas/atlas-search/sort/#sort-fts-results', + ], + }, + { + input: `What is token type mapping in Atlas Search?`, + expected: `In Atlas Search, mapping a field with the token type indexes it as a single term, without applying tokenization or lowercasing. This is particularly useful for enabling sorting, faceting, and exact matching on string fields. If you need to perform case-insensitive operations, you can apply the normalizer: lowercase.`, + expectedSources: [ + 'https://www.mongodb.com/docs/atlas/atlas-search/field-types/token-type/#how-to-index-string-fields-for-efficient-sorting-and-faceting', + ], + }, + { + input: `Why doesn’t $search return expected results for substrings of a token?`, + expected: `$search operates based on tokenized fields, meaning that substrings of tokens may not match unless explicitly configured with features like the autocomplete data type or n-gram and edgeNgram tokenizers. Ensure your Atlas Search Index settings align with your query requirements for partial matching.`, + expectedSources: [ + 'https://www.mongodb.com/docs/atlas/atlas-search/field-types/token-type/#how-to-index-string-fields-for-efficient-sorting-and-faceting', + ], + }, + { + input: `How can I use $search for simple keyword search across multiple fields?`, + expected: `To apply $search across multiple fields, use the text operator with a path value that is an array of field names. Alternatively, for more complex scenarios, you can use the compound operator with should or must clauses to specify the search terms and target fields.`, + expectedSources: [ + 'https://www.mongodb.com/docs/atlas/atlas-search/text/#text-operator https://www.mongodb.com/docs/atlas/atlas-search/compound/#compound-operator', + ], + }, + { + input: `Can I apply fuzzy matching with $search?`, + expected: `Yes, $search supports fuzzy matching for text and autocomplete fields, which returns results for close matches (e.g., misspelled words). You configure this by adding a fuzzy object to your query, specifying the maxEdits (the maximum number of single-character edits) allowed. An edit can be an insertion, deletion, or substitution. Example:\n{\n "$search": {\n "text": {\n "query": "monodb",\n "path": "title",\n "fuzzy": {\n "maxEdits": 2\n }\n }\n }\n}`, + expectedSources: [ + 'https://www.mongodb.com/docs/atlas/atlas-search/text/#text-operator', + ], + }, + { + input: `How do I debug $search queries that return no results?`, + expected: `If $search queries return no results, first ensure that your Atlas Search Index is active and has finished building. Confirm your query targets a field that is correctly mapped in the index definition. Check the query syntax, field types, and analyzer settings.`, + }, + { + input: `How can I combine multiple conditions in a single $search query?`, + expected: `To combine multiple conditions in a single $search query, use the compound operator. The must clause acts as an AND operator (all conditions must be met), should increases the relevancy score for documents that match (acting as a preferential OR), and filter applies conditions without affecting the score. For example:\n{\n "$search": {\n "compound": {\n "must": [\n { "text": { "query": "mongodb", "path": "title" } }\n ],\n "should": [\n { "text": { "query": "database", "path": "description" } },\n { "text": { "query": "flexibility", "path": "features" } }\n ],\n "filter": [\n { "range": { "path": "year", "gte": 2020 } }\n ]\n }\n }\n}`, + expectedSources: [ + 'https://www.mongodb.com/docs/atlas/atlas-search/compound/#compound-operator', + ], + }, + { + input: `Can $search support wildcard or regex queries?`, + expected: `Yes, $search supports both wildcard and regex queries. The wildcard operator allows for simple pattern matching using * (matches zero or more characters) and ? (matches a single character). The regex operator supports more complex patterns using the Lucene regular expression syntax. Example using wildcard:\n{\n "$search": {\n "wildcard": {\n "query": "mongo*",\n "path": "title",\n "allowAnalyzedField": true\n }\n }\n}\n\n\nFor optimal query performance when searching for substrings, the autocomplete type and operator is recommended. Using the autocomplete type requires more storage resources, so performance vs storage tradeoffs must be considered.`, + expectedSources: [ + 'https://www.mongodb.com/docs/atlas/atlas-search/wildcard/#wildcard-operator', + ], + }, + { + input: `How do I analyze why certain documents have a higher search score than others?`, + expected: `To analyze search scores, you can include "scoreDetails": true in the $search stage and $meta: "searchScoreDetails" in a subsequent $project stage to return a detailed breakdown of the score calculation for each document. For a full execution plan, you can prepend db.collection.explain().aggregate(...) to your query in mongosh. This returns a detailed JSON document explaining how the query was parsed and executed, including timing and scoring details.`, + expectedSources: [ + 'https://www.mongodb.com/docs/atlas/atlas-search/scoring/#score-the-documents-in-the-results', + ], + }, + { + input: `Can I use $search to return facets for query results?`, + expected: `Yes, you can calculate facets (aggregated counts of search matches grouped by specific criteria) by using the facet collector within the $search stage. This is typically done alongside a query. For faceting on string fields, the field must be mapped with the stringFacet or token type in the index.\n{\n "$search": {\n "facet": {\n "operator": {\n "text": {\n "query": "database",\n "path": "title"\n }\n },\n "facets": {\n "categoryFacet": {\n "type": "string",\n "path": "category"\n }\n }\n }\n }\n}`, + expectedSources: [ + 'https://www.mongodb.com/docs/atlas/atlas-search/facet/#facet-collector', + ], + }, + { + input: `Why does $search fail when querying on a field without an Atlas Search Index?`, + expected: `$search queries will fail or return no results if the queried field is not included in the Atlas Search Index definition. Unlike standard MongoDB queries which can perform collection scans, $search relies exclusively on its pre-built search index for all operations.`, + }, + { + input: `What is the difference between $search and $searchMeta?`, + expected: `Both $search and $searchMeta execute a search query, but $search returns the matching documents, while $searchMeta returns only the metadata about the search results, such as the total count of matching documents (count) or facet results. Use $searchMeta when you only need the metadata and want to avoid the overhead of retrieving documents. If you need both the metadata and documents, you can access the $$SEARCH_META aggregation variable in a subsequent stage after $search.\n{\n "$search": {\n "facet": {\n "operator": {\n "near": {\n "path": "released",\n "origin": ISODate("1999-07-01T00:00:00.000+00:00"),\n "pivot": 7776000000\n }\n },\n "facets": {\n "genresFacet": {\n "type": "string",\n "path": "genres"\n }\n }\n }\n }\n },\n { "$limit": 100 },\n {\n "$facet": {\n "docs": [],\n "meta": [\n {"$replaceWith": "$$SEARCH_META"},\n {"$limit": 1}\n ]\n }\n },\n {\n "$set": {\n "meta": {\n "$arrayElemAt": ["$meta", 0]\n }\n }\n }\n])`, + expectedSources: [ + 'https://www.mongodb.com/docs/atlas/atlas-search/query-syntax/#choose-the-aggregation-pipeline-stage', + ], + }, + { + input: `How can I highlight terms in $search results?`, + expected: `To highlight matched terms, add a highlight object to your $search stage. Specify the path of the field to highlight in. The query results will include an extra highlights field containing snippets with the matched terms wrapped in tags by default.\n{\n "$search": {\n "text": {\n "query": "mongodb",\n "path": "description"\n },\n "highlight": {\n "path": "description"\n }\n }\n}`, + expectedSources: [ + 'https://www.mongodb.com/docs/atlas/atlas-search/highlighting/#highlight-search-terms-in-results', + ], + }, + { + input: `Can $search handle multi-language text queries?`, + expected: `Yes, $search supports multi-language queries by configuring language-specific analyzers in the Atlas Search Index. You can select from a variety of analyzers (e.g., lucene.english, lucene.french, lucene.spanish) to correctly apply language-specific rules for stemming, tokenization, and stop words, which improves search relevance.`, + expectedSources: [ + 'https://www.mongodb.com/docs/atlas/atlas-search/analyzers/language/#language-analyzers', + ], + }, + { + input: `What is an aggregation pipeline?`, + expected: `The aggregation pipeline in MongoDB is a framework for data processing and transformation. It consists of a sequence of stages, where each stage performs an operation on the input documents and passes the results to the next stage. Common operations include filtering, grouping, projecting, joining, and calculating values. Aggregation pipelines are powerful for data analysis, reporting, and transformation tasks in MongoDB.\n\nCompass makes it easy to create and run aggregation pipelines under the Aggregations tab. You may generate an aggregation pipeline with natural language, utilize the visual stage editor, or edit aggregations in the text view. \n\nExample aggregation pipeline: \ndb.orders.aggregate([\n // Stage 1: Unwind the array of products\n { $unwind: { path: "$products" } },\n\n // Stage 2: Match products that cost more than $15\n { $match: { "products.price": { $gt: 15 } } },\n\n // Stage 3: Group products by product ID\n { $group: {\n _id: "$products.prod_id",\n product: { $first: "$products.name" },\n total_value: { $sum: "$products.price" },\n quantity: { $sum: 1 }\n }\n },\n\n // Stage 4: Add a product_id field\n { $set: { product_id: "$_id" } },\n\n // Stage 5: Remove the _id field\n { $unset: ["_id"] }\n])`, + expectedSources: [ + 'https://www.mongodb.com/docs/manual/core/aggregation-pipeline/', + 'https://www.mongodb.com/docs/compass/create-agg-pipeline/', + ], + }, + { + input: `How do I model data with MongoDB?`, + expected: `Data modeling in MongoDB is highly dependent on how you access your data. To ensure that your data model has a logical structure and achieves optimal performance, plan your schema prior to using your database at a production scale. To determine your data model, use the following schema design process:\n\n\nIdentify your workload: Identify the operations that your application runs most frequently\n\nMap relationships: Identify the relationships in your application's data and decide whether to link or embed related data.\n\nApply design patterns: Apply schema design patterns to optimize reads and writes.\n\nCreate indexes: Create indexes to support common query patterns.`, + expectedSources: [ + 'https://www.mongodb.com/docs/manual/data-modeling/#plan-your-schema', + 'https://www.mongodb.com/docs/manual/data-modeling/schema-design-process/#designing-your-schema', + ], + }, + { + input: `How is MongoDB data modeling different from data modeling with a relational database?`, + expected: `When using a relational database: \nYou must determine a table's schema before you insert data.\n\nYou often need to join data from several different tables to return the data needed by your application.\n\n\nIn MongoDB: \nYour schema can change over time as the needs of your application change.\n\n\nThe flexible data model lets you store data to match the way your application returns data, and avoid joins. Avoiding joins across multiple collections improves performance and reduces your deployment's workload.`, + expectedSources: [ + 'https://www.mongodb.com/docs/manual/data-modeling/#schema-design--differences-between-relational-and-document-databases', + ], + }, + { + input: `Is MongoDB schemaless?`, + expected: `No, MongoDB has a flexible schema model, which means:\n\nDocuments within a single collection are not required to have the same set of fields.\n\nA field's data type can differ between documents within a collection.\n\nGenerally, documents in a collection share a similar structure. To ensure consistency in your data model, you can create schema validation rules.`, + expectedSources: [ + 'https://www.mongodb.com/docs/manual/data-modeling/#data-modeling', + ], + }, + { + input: `Should I embed my data in an existing collection or put it in a new collection?`, + expected: `Consider embedding data when it: \nSimplifies your code \n\nPieces of information have a “has-a,” “contains,” or similar relationship \n\nPieces of information are queried together \n\nPieces of information are updated together \n\nPieces of information are archived at the same time`, + }, + { + input: `Does MongoDB have foreign keys?`, + expected: `MongoDB does not store foreign keys in the database; however you can reference data stored in another collection by using the $lookup aggregation stage. There, you define foreign and local fields to reference.`, + expectedSources: [ + 'https://www.mongodb.com/docs/manual/reference/operator/aggregation/lookup/#-lookup--aggregation-', + ], + }, + { + input: `How can I set schema validation rules in MongoDB?`, + expected: `You can set validation rules in MongoDB using JSON schema validation. They can be added in Compass under the Validation tab or using the collMod database command. \n\nSome users may prefer to use an ODM to perform validation at the application level. MongoDB does not have an official ODM but they are still popular amongst MongoDB users.`, + expectedSources: [ + 'https://www.mongodb.com/docs/manual/core/schema-validation/specify-validation-level/', + 'https://www.mongodb.com/docs/compass/validation/', + ], + }, + { + input: `How can I reduce the size of my documents? \n\n[in response to “bloated documents” insight]`, + expected: `To prevent bloated documents, restructure your schema with smaller documents and use document references to separate fields that aren't returned together. This approach reduces the working set size and improves performance.\n\nConsider referencing data that: \nIs currently embedded but frequently changes.\n\nIs part of a complex many-to-many relationships or large hierarchical data sets.\n\nIs currently embedded but is frequently queried on its own.`, + expectedSources: [ + 'https://www.mongodb.com/docs/manual/data-modeling/design-antipatterns/bloated-documents/', + 'https://www.mongodb.com/docs/manual/data-modeling/concepts/embedding-vs-references/#std-label-data-modeling-referencing', + ], + }, + { + input: `What is the best index for this query?\n\nQuery or agg written, indexes on the collection \n\n[in response to “query executed without index” insight]`, + expected: `In isolation, the index would support the inputted query . \n\nIndexes optimize reads but can slow down writes. Only index fields that appear frequently in queries or return sorted results. You may be able to modify rather than create a new index to support your query, if you anticipate running it frequently.`, + expectedSources: [ + 'https://www.mongodb.com/docs/manual/core/data-model-operations/', + 'https://www.mongodb.com/docs/manual/data-modeling/schema-design-process/create-indexes/', + ], + }, + { + input: `When should I index a query? \n\n[in response to “query executed without index” insight]`, + expected: `If your application is repeatedly running queries on the same fields, you can create an index on those fields to improve performance.\n\nFor collections with a high write-to-read ratio, indexes are expensive because each insert must also update any indexes. Only create indexes on fields that are frequently accessed or return sorted, to maintain write efficiency.`, + expectedSources: [ + 'https://www.mongodb.com/docs/manual/core/data-model-operations/', + 'https://www.mongodb.com/docs/manual/data-modeling/schema-design-process/create-indexes/', + 'https://www.mongodb.com/docs/manual/indexes/', + ], + }, + { + input: `How can I reduce the number of indexes on my collection? \n\nIndexes on the collection \n\n[in response to “high number of indexes on collection” insight]`, + expected: `Identify indexes that are rarely used, redundant due to coverage by another compound index, or not used at all. \n\nBased on the indexes you have on the collection, the following indexes are redundant: . These can be replaced by \n\nCompass can not reliably evaluate which indexes are rarely used or not used at all. Use your knowledge of your workload to assess whether additional indexes can be removed, or visit Atlas Performance Advisor.`, + expectedSources: [ + 'https://www.mongodb.com/docs/manual/data-modeling/design-antipatterns/unnecessary-indexes/#std-label-unnecessary-indexes-antipattern', + 'https://www.mongodb.com/docs/atlas/performance-advisor/drop-indexes/', + ], + }, + { + input: `How do I avoid using $lookup? \n\n$lookup stage in agg builder \n\n[in response to “$lookup usage” insight]`, + expected: `Avoid using $lookup by storing data that is accessed together in the same document. \n\nUse embedded data models in the following scenarios:\nYou have "contains" relationships between entities. For example, a contacts document that contains an address. See Model One-to-One Relationships with Embedded Documents.\n\nYou have one-to-many relationships between entities. In these relationships, the "many" or child documents are viewed in the context of the "one" or parent documents. See Model One-to-Many Relationships with Embedded Documents.\n\n\nEmbedding provides the following benefits:\nBetter performance for read operations\n\nThe ability to retrieve related data in a single database operation\n\nThe ability to to update related data in a single atomic write operation\n\n\nTo query data within embedded documents, use dot notation.`, + expectedSources: [ + 'https://www.mongodb.com/docs/manual/data-modeling/concepts/embedding-vs-references/', + ], + }, +]; diff --git a/packages/compass-assistant/test/eval-cases/index.ts b/packages/compass-assistant/test/eval-cases/index.ts index 03270a041db..d04287feda2 100644 --- a/packages/compass-assistant/test/eval-cases/index.ts +++ b/packages/compass-assistant/test/eval-cases/index.ts @@ -1,10 +1,12 @@ import type { SimpleEvalCase } from '../assistant.eval'; -import atlasSearch from './atlas-search'; -import aggregationPipeline from './aggregation-pipeline'; -import modelData from './model-data'; +import { modelDataEvalCases } from './model-data'; +import { atlasSearchEvalCases } from './atlas-search'; +import { aggregationPipelineEvalCases } from './aggregation-pipeline'; +import { generatedEvalCases } from './generated-cases'; export const evalCases: SimpleEvalCase[] = [ - ...atlasSearch, - ...aggregationPipeline, - ...modelData, + ...atlasSearchEvalCases, + ...generatedEvalCases, + ...aggregationPipelineEvalCases, + ...modelDataEvalCases, ]; diff --git a/packages/compass-assistant/test/eval-cases/model-data.ts b/packages/compass-assistant/test/eval-cases/model-data.ts index 44d71af6da5..e48ff921da1 100644 --- a/packages/compass-assistant/test/eval-cases/model-data.ts +++ b/packages/compass-assistant/test/eval-cases/model-data.ts @@ -1,6 +1,6 @@ import type { SimpleEvalCase } from '../assistant.eval'; -const evalCases: SimpleEvalCase[] = [ +export const modelDataEvalCases: SimpleEvalCase[] = [ { input: 'How do I model data with MongoDB?', expected: `Data modeling in MongoDB is highly dependent on how you access your data. To ensure that your data model has a logical structure and achieves optimal performance, plan your schema prior to using your database at a production scale. To determine your data model, use the following schema design process: @@ -16,5 +16,3 @@ Create indexes: Create indexes to support common query patterns. ], }, ]; - -export default evalCases; From 921ad45331b3558acb9d5bac06b25da3adc45e2c Mon Sep 17 00:00:00 2001 From: gagik Date: Wed, 10 Sep 2025 14:55:43 +0200 Subject: [PATCH 2/5] chore: add tags --- .../scripts/convert-csv-to-eval-cases.ts | 35 ++++++++++++++- .../compass-assistant/test/assistant.eval.ts | 16 +++++++ .../test/entrypoints/explain-plan.ts | 1 + .../test/eval-cases/aggregation-pipeline.ts | 1 + .../test/eval-cases/atlas-search.ts | 1 + .../test/eval-cases/generated-cases.ts | 43 +++++++++++++++++++ .../test/eval-cases/model-data.ts | 1 + 7 files changed, 97 insertions(+), 1 deletion(-) diff --git a/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts b/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts index dabcac604d6..1e856810d4f 100644 --- a/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts +++ b/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts @@ -21,6 +21,19 @@ type SimpleEvalCase = { input: string; expected: string; expectedSources?: string[]; + tags?: string[]; +}; + +const interactionTypeTags = { + 'End-User Input Only': 'end-user-input', + 'Connection Error': 'connection-error', + 'DNS Error': 'dns-error', + 'Explain Plan': 'explain-plan', + 'Proactive Perf': 'proactive-performance-insights', + 'General network error': 'general-network-error', + OIDC: 'oidc', + TSL: 'tsl-ssl', + SSL: 'tsl-ssl', }; function escapeString(str: string): string { @@ -45,11 +58,18 @@ function generateEvalCaseFile(cases: SimpleEvalCase[]): string { .join(',\n ')},\n ],` : ''; + const tagsPart = + evalCase.tags && evalCase.tags.length > 0 + ? ` tags: [\n ${evalCase.tags + .map((tag) => `'${escapeString(tag)}'`) + .join(',\n ')},\n ],` + : ''; + return ` { input: \`${escapeString(evalCase.input)}\`, expected: \`${escapeString(evalCase.expected)}\`,${ sourcesPart ? '\n' + sourcesPart : '' - } + }${tagsPart ? '\n' + tagsPart : ''} }`; }) .join(',\n'); @@ -133,10 +153,23 @@ async function convertCSVToEvalCases() { .filter((link) => link && link.startsWith('http')); } + const tags: string[] = []; + + if (interactionType) { + for (const tag of Object.keys(interactionTypeTags)) { + if (interactionType.includes(tag)) { + tags.push( + interactionTypeTags[tag as keyof typeof interactionTypeTags] + ); + } + } + } + const evalCase: SimpleEvalCase = { input, expected, ...(expectedSources.length > 0 && { expectedSources }), + ...(tags.length > 0 && { tags }), }; allCases.push(evalCase); diff --git a/packages/compass-assistant/test/assistant.eval.ts b/packages/compass-assistant/test/assistant.eval.ts index d36327f69df..e44fd3bd7a0 100644 --- a/packages/compass-assistant/test/assistant.eval.ts +++ b/packages/compass-assistant/test/assistant.eval.ts @@ -23,6 +23,20 @@ export type SimpleEvalCase = { input: string; expected: string; expectedSources?: string[]; + tags: ( + | 'end-user-input' + | 'connection-error' + | 'dns-error' + | 'explain-plan' + | 'proactive-performance-insights' + | 'general-network-error' + | 'oidc' + | 'tsl-ssl' + | 'ssl' + | 'model-data' + | 'aggregation-pipeline' + | 'atlas-search' + )[]; }; type Message = { @@ -96,6 +110,7 @@ function makeEvalCases(): ConversationEvalCase[] { expected: { messages: [{ text: c.expected, sources: c.expectedSources || [] }], }, + tags: c.tags || [], metadata: {}, }; } @@ -111,6 +126,7 @@ function makeEvalCases(): ConversationEvalCase[] { expected: { messages: [{ text: c.expected, sources: c.expectedSources || [] }], }, + tags: c.tags || [], metadata: {}, }; }); diff --git a/packages/compass-assistant/test/entrypoints/explain-plan.ts b/packages/compass-assistant/test/entrypoints/explain-plan.ts index 6f8a233ffcc..5ab296cc49f 100644 --- a/packages/compass-assistant/test/entrypoints/explain-plan.ts +++ b/packages/compass-assistant/test/entrypoints/explain-plan.ts @@ -3949,6 +3949,7 @@ function buildPrompt(explainCase: ExplainCase): SimpleEvalCase { }).prompt, expected: explainCase.expected, expectedSources: explainCase.expectedsources, + tags: ['explain-plan'], }; } diff --git a/packages/compass-assistant/test/eval-cases/aggregation-pipeline.ts b/packages/compass-assistant/test/eval-cases/aggregation-pipeline.ts index dd962519b1c..14b345a88c3 100644 --- a/packages/compass-assistant/test/eval-cases/aggregation-pipeline.ts +++ b/packages/compass-assistant/test/eval-cases/aggregation-pipeline.ts @@ -35,5 +35,6 @@ db.orders.aggregate([ 'https://www.mongodb.com/docs/manual/core/aggregation-pipeline/', 'https://www.mongodb.com/docs/compass/create-agg-pipeline/', ], + tags: ['aggregation-pipeline'], }, ]; diff --git a/packages/compass-assistant/test/eval-cases/atlas-search.ts b/packages/compass-assistant/test/eval-cases/atlas-search.ts index f6b63c81fbf..d69d4b7b6e8 100644 --- a/packages/compass-assistant/test/eval-cases/atlas-search.ts +++ b/packages/compass-assistant/test/eval-cases/atlas-search.ts @@ -9,5 +9,6 @@ export const atlasSearchEvalCases: SimpleEvalCase[] = [ 'https://www.mongodb.com/docs/atlas/atlas-search/compound/#options', 'https://www.mongodb.com/docs/atlas/atlas-search/transform-documents-collections/#example--filter-documents', ], + tags: ['atlas-search'], }, ]; diff --git a/packages/compass-assistant/test/eval-cases/generated-cases.ts b/packages/compass-assistant/test/eval-cases/generated-cases.ts index c46ae1c311f..fa2163d9830 100644 --- a/packages/compass-assistant/test/eval-cases/generated-cases.ts +++ b/packages/compass-assistant/test/eval-cases/generated-cases.ts @@ -13,6 +13,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ 'https://www.mongodb.com/docs/compass/current/connect', 'https://www.mongodb.com/docs/compass/current/troubleshooting/logs https://www.mongodb.com/docs/compass/current/troubleshooting/logs/', ], + tags: ['connection-error'], }, { input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nConnection string (password redacted): \nmongodb+srv://betsy:*****@cluster0.pndqllj.mongodb.net/\n\nError message:\nquerySrv ENOTFOUND _mongodb._tcp.cluster0.pndollj.mongodb.net\n\nThere was a problem connecting to cluster0.pndollj.mongodb.net`, @@ -24,6 +25,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ 'https://www.mongodb.com/docs/compass/current/connect', 'https://www.mongodb.com/docs/compass/current/troubleshooting/logs', ], + tags: ['connection-error'], }, { input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nConnection string (password redacted): \nmongodb+srv://betsy:*****@cluster0.pndqllj.mongodb.net/\n\nError message:\nconnect ENETUNREACH 89.193.144.189:27017\n\nThere was a problem connecting to App`, @@ -31,6 +33,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/compass/current/troubleshooting/logs/', ], + tags: ['connection-error', 'general-network-error'], }, { input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nConnection string (password redacted): \nmongodb+srv://betsy:*****@cluster0.pndqllj.mongodb.net/\n\nError message:\nquerySrv ECONNREFUSED`, @@ -42,6 +45,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ 'https://www.mongodb.com/docs/compass/current/connect', 'https://www.mongodb.com/docs/compass/current/troubleshooting/logs', ], + tags: ['connection-error'], }, { input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nAuth mechanism: \n\nError message:\nconnect ECONNREFUSED`, @@ -49,34 +53,42 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/compass/current/troubleshooting/logs/', ], + tags: ['connection-error'], }, { input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nAuth mechanism: \n\nError message:`, expected: `Cannot connect in an Atlas multi-region deployment setup\nCheck the Compass log\n\nConfirm that the connection string is correct (username, password, hostname, port, connection options including TLS/SSL certificates, etc).\n\nMake sure the number of connections are not at the maximum number for the tier.\n\nMake sure the Cloud provider on the customer side has peering VPCs set up. These settings are made outside of Atlas and may need the help of the customer's IT team.\n\nNetwork packet inspection and traceroute may be needed.`, + tags: ['connection-error'], }, { input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n \n\nAuth mechanism: \n\nError message:`, expected: `Cannot connect to an individual Atlas host when using Private Endpoint.\nCheck the Compass log\n\nConfirm that the connection string is correct (username, password, hostname, port, connection options including TLS/SSL certificates, etc). Atlas Private Endpoint connections use ports starting at 1024.\n\nVerify VPN and firewalls are not preventing a connection from the IP address and ports.\n\nMake sure the number of connections on the Server are less than the maximum number for the tier; otherwise, connections will not be successful.\n\nEnsure DNS SRV and DNS TXT records are successful and fast, say, 0.1 seconds.`, + tags: ['connection-error'], }, { input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\nAuth mechanism: \n\nError message:`, expected: `Cannot connect to Atlas using VPC connections.\nCheck the Compass log\n\nConfirm that the connection string is correct (username, password, hostname, port, connection options including TLS/SSL certificates, etc). Atlas Private Endpoint connections use ports starting at 1024.\n\nVerify VPN and firewalls are not preventing a connection from the IP address and ports.\n\nMake sure the number of connections on the Server are less than the maximum number for the tier; otherwise, connections will not be successful.\n\nEnsure DNS SRV and DNS TXT records are successful and fast, say, 0.1 seconds.`, + tags: ['connection-error'], }, { input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\n\nAuth mechanism: \n\nError message:`, expected: `Cannot connect with an AWS Lambda application.\nCheck the Compass log\n\nUse the Reachability Analyzer for troubleshooting\n\n[FH: I think the KB article should be AI ingested. The other troubleshooting steps are extensive]`, + tags: ['connection-error'], }, { input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\nAuth mechanism: \nUsername/password\n\nError message:\nserverSelectionTimeout\n\nThere was a problem connecting to cluster0.pndollj.mongodb.net`, expected: `Check the Compass log\n\nConfirm that the connection string is correct (username, password, hostname, port, connection options including TLS/SSL or X.509 certificates, etc).\n\nOpen a terminal or command prompt and run “nslookup” on the DNS SRV record for your cluster. If this fails, your DNS server may be blocking or unable to resolve the address. Also ensure DNS TXT results are allowed to be returned.\n\nConfirm the correct authentication method is being used (TLS/SSL, LDAP, X.509, Kerberos, OIDC)\n\nConfirm the authentication server on the customer side has been set up properly for the user (LDAP server, Kerberos server, etc).\n\nFor Atlas connections using the Standard Connection, ensure the IP address is added to the Network Access list.\n\n\nThis error is generic and not specific enough to identify the connection issue. Further explanation here:\nHow to Troubleshoot and Resolve the 'MongoServerSelectionError' in MongoDB`, + tags: ['connection-error'], }, { input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\nAuth mechanism: \nOIDC\n\nError message:\n\nAuthentication failed. There was a problem connecting to `, expected: `It sounds like you successfully authenticated with your identity provider, but the MongoDB server is misconfigured and did not accept your authentication request. To resolve this issue:\nCheck the Compass log\n\nCheck whether you are able to connect to the cluster using the MongoDB Shell\n\nGather information using mongosh --oidcDumpTokens \n\nReach out to the cluster administrator with the Compass log, output from mongosh, and the time at which you connected, and the versions of Compass and mongosh you used`, + tags: ['connection-error', 'oidc'], }, { input: `Compass hardcoded prompt: \nGiven the error message below and connection string, please provide clear instructions to guide the user to debug their connection attempt from MongoDB Compass. If no auth mechanism is specified in the connection string, the default (username/password) is being used. \n\nAuth mechanism: \nOIDC\n\nError message:\n\nCompass hangs and opens a browser windows that says “400 Bad Request”.`, expected: `It sounds like your MongoDB server or identity provider is not configured correctly. To resolve this issue:\nCheck the Compass log\n\nCheck whether you are able to connect to the cluster using the MongoDB Shell\n\nReach out to the cluster administrator with the Compass log, output from mongosh, and the time at which you connected, and the versions of Compass and mongosh you used`, + tags: ['connection-error', 'oidc'], }, { input: `Compass hardcoded prompt: \nGiven the MongoDB explain plan output below, provide a concise human readable explanation that explains the query execution plan and highlights aspects of the plan that might impact query performance. Respond with as much concision and clarity as possible. \n\nIf a clear optimization should be made, please suggest the optimization and describe how it can be accomplished in MongoDB Compass. Do not advise users to create indexes without weighing the pros and cons. \n\nExplain output: \n{"explainVersion":"1","stages":[{"$cursor":{"queryPlanner":{"namespace":"restaurants.nyc","indexFilterSet":false,"parsedQuery":{"cuisine":{"$eq":"Italian"}},"queryHash":"0D9A721D","planCacheKey":"0BC4F8A9","optimizationTimeMillis":0,"maxIndexedOrSolutionsReached":false,"maxIndexedAndSolutionsReached":false,"maxScansToExplodeReached":false,"winningPlan":{"stage":"PROJECTION_SIMPLE","transformBy":{"borough":1,"_id":0},"inputStage":{"stage":"FETCH","inputStage":{"stage":"IXSCAN","keyPattern":{"cuisine":-1},"indexName":"cuisine_-1","isMultiKey":false,"multiKeyPaths":{"cuisine":[]},"isUnique":false,"isSparse":false,"isPartial":false,"indexVersion":2,"direction":"forward","indexBounds":{"cuisine":["[\\"Italian\\", \\"Italian\\"]"]}}}},"rejectedPlans":[]},"executionStats":{"executionSuccess":true,"nReturned":43207,"executionTimeMillis":1028,"totalKeysExamined":43207,"totalDocsExamined":43207,"executionStages":{"stage":"PROJECTION_SIMPLE","nReturned":43207,"executionTimeMillisEstimate":888,"works":43208,"advanced":43207,"needTime":0,"needYield":0,"saveState":80,"restoreState":80,"isEOF":1,"transformBy":{"borough":1,"_id":0},"inputStage":{"stage":"FETCH","nReturned":43207,"executionTimeMillisEstimate":885,"works":43208,"advanced":43207,"needTime":0,"needYield":0,"saveState":80,"restoreState":80,"isEOF":1,"docsExamined":43207,"alreadyHasObj":0,"inputStage":{"stage":"IXSCAN","nReturned":43207,"executionTimeMillisEstimate":48,"works":43208,"advanced":43207,"needTime":0,"needYield":0,"saveState":80,"restoreState":80,"isEOF":1,"keyPattern":{"cuisine":-1},"indexName":"cuisine_-1","isMultiKey":false,"multiKeyPaths":{"cuisine":[]},"isUnique":false,"isSparse":false,"isPartial":false,"indexVersion":2,"direction":"forward","indexBounds":{"cuisine":["[\\"Italian\\", \\"Italian\\"]"]},"keysExamined":43207,"seeks":1,"dupsTested":0,"dupsDropped":0}}}}}}, "nReturned":43207,"executionTimeMillisEstimate":1004},{"$group":{"_id":"$borough","count":{"$sum":{"$const":1}}},"maxAccumulatorMemoryUsageBytes":{"count":480},"totalOutputDataSizeBytes":1254,"usedDisk":false,"spills":0,"spilledDataStorageSize":0,"numBytesSpilledEstimate":0,"spilledRecords":0,"nReturned":5,"executionTimeMillisEstimate":1019}],"serverInfo":{"host":"atlas-mk8saw-shard-00-01.pndqllj.mongodb.net","port":27017,"version":"7.0.21","gitVersion":"a47b62aff2bae1914085c3ef1d90fc099acf000c"},"serverParameters":{"internalQueryFacetBufferSizeBytes":104857600,"internalQueryFacetMaxOutputDocSizeBytes":104857600,"internalLookupStageIntermediateDocumentMaxSizeBytes":104857600,"internalDocumentSourceGroupMaxMemoryBytes":104857600,"internalQueryMaxBlockingSortMemoryUsageBytes":104857600,"internalQueryProhibitBlockingMergeOnMongoS":0,"internalQueryMaxAddToSetBytes":104857600,"internalDocumentSourceSetWindowFieldsMaxMemoryBytes":104857600,"internalQueryFrameworkControl":"forceClassicEngine"},"command":{"aggregate":"nyc","pipeline":[{"$match":{"cuisine":"Italian"}},{"$group":{"_id":"$borough","count":{"$sum":1}}}],"cursor":{},"maxTimeMS":60000,"$db":"restaurants"},"ok":1,"$clusterTime":{"clusterTime":{"$timestamp":"7522966816189054977"},"signature":{"hash":"dhZ8X27p5jRJo1x/U9ZpF8L4eSc=","keyId":{"low":1,"high":1737746411,"unsigned":false}}},"operationTime":{"$timestamp":"7522966816189054977"}}`, @@ -88,6 +100,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ 'https://www.mongodb.com/docs/manual/tutorial/equality-sort-range-guideline/#std-label-esr-indexing-guideline', 'https://www.mongodb.com/docs/manual/core/indexes/index-types/index-compound/#index-prefixes', ], + tags: ['explain-plan'], }, { input: `Compass hardcoded prompt: \nGiven the MongoDB explain plan output below, provide a concise human readable explanation that explains the query execution plan and highlights aspects of the plan that might impact query performance. Respond with as much concision and clarity as possible. \n\nIf a clear optimization should be made, please suggest the optimization and describe how it can be accomplished in MongoDB Compass. Do not advise users to create indexes without weighing the pros and cons. \n\nExplain output: \n{"explainVersion":"1","queryPlanner":{"namespace":"restaurants.nyc","indexFilterSet":false,"parsedQuery":{"category":{"$eq":"low_rated"}},"queryHash":"2EBC9A51","planCacheKey":"2EBC9A51","optimizationTimeMillis":0,"maxIndexedOrSolutionsReached":false,"maxIndexedAndSolutionsReached":false,"maxScansToExplodeReached":false,"winningPlan":{"stage":"COLLSCAN","filter":{"category":{"$eq":"low_rated"}},"direction":"forward"},"rejectedPlans":[]},"executionStats":{"executionSuccess":true,"nReturned":759349,"executionTimeMillis":1233,"totalKeysExamined":0,"totalDocsExamined":998014,"executionStages":{"stage":"COLLSCAN","filter":{"category":{"$eq":"low_rated"}},"nReturned":759349,"executionTimeMillisEstimate":232,"works":998015,"advanced":759349,"needTime":238665,"needYield":0,"saveState":998,"restoreState":998,"isEOF":1,"direction":"forward","docsExamined":998014}},"command":{"find":"nyc","filter":{"category":"low_rated"},"skip":0,"limit":0,"maxTimeMS":60000,"$db":"restaurants"},"serverInfo":{"host":"atlas-mk8saw-shard-00-01.pndqllj.mongodb.net","port":27017,"version":"7.0.22","gitVersion":"6660d93e97cdb0dc35d8ceb699c511f5850278a5"},"serverParameters":{"internalQueryFacetBufferSizeBytes":104857600,"internalQueryFacetMaxOutputDocSizeBytes":104857600,"internalLookupStageIntermediateDocumentMaxSizeBytes":104857600,"internalDocumentSourceGroupMaxMemoryBytes":104857600,"internalQueryMaxBlockingSortMemoryUsageBytes":104857600,"internalQueryProhibitBlockingMergeOnMongoS":0,"internalQueryMaxAddToSetBytes":104857600,"internalDocumentSourceSetWindowFieldsMaxMemoryBytes":104857600,"internalQueryFrameworkControl":"forceClassicEngine"},"ok":1,"$clusterTime":{"clusterTime":{"$timestamp":"7537442131581861889"},"signature":{"hash":"vkFwzpNfNq+HU/KcFnxAhplnBzA=","keyId":{"low":1,"high":1745504880,"unsigned":false}}},"operationTime":{"$timestamp":"7537442131581861889"}}`, @@ -97,6 +110,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ 'https://www.mongodb.com/docs/manual/reference/explain-results', 'https://www.mongodb.com/docs/manual/faq/indexes', ], + tags: ['explain-plan'], }, { input: `How can I filter docs before running a $search query?`, @@ -104,6 +118,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/atlas/atlas-search/compound/#options https://www.mongodb.com/docs/atlas/atlas-search/transform-documents-collections/#example--filter-documents', ], + tags: ['end-user-input'], }, { input: `What is the $search stage?`, @@ -111,6 +126,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/atlas/atlas-search/#what-is-fts-', ], + tags: ['end-user-input'], }, { input: `Can $search work with regular MongoDB indexes?`, @@ -118,6 +134,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/atlas/atlas-search/index-definitions/#index-reference', ], + tags: ['end-user-input'], }, { input: `How do I sort query results using $search?`, @@ -125,6 +142,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/atlas/atlas-search/sort/#sort-fts-results', ], + tags: ['end-user-input'], }, { input: `What is token type mapping in Atlas Search?`, @@ -132,6 +150,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/atlas/atlas-search/field-types/token-type/#how-to-index-string-fields-for-efficient-sorting-and-faceting', ], + tags: ['end-user-input'], }, { input: `Why doesn’t $search return expected results for substrings of a token?`, @@ -139,6 +158,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/atlas/atlas-search/field-types/token-type/#how-to-index-string-fields-for-efficient-sorting-and-faceting', ], + tags: ['end-user-input'], }, { input: `How can I use $search for simple keyword search across multiple fields?`, @@ -146,6 +166,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/atlas/atlas-search/text/#text-operator https://www.mongodb.com/docs/atlas/atlas-search/compound/#compound-operator', ], + tags: ['end-user-input'], }, { input: `Can I apply fuzzy matching with $search?`, @@ -153,10 +174,12 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/atlas/atlas-search/text/#text-operator', ], + tags: ['end-user-input'], }, { input: `How do I debug $search queries that return no results?`, expected: `If $search queries return no results, first ensure that your Atlas Search Index is active and has finished building. Confirm your query targets a field that is correctly mapped in the index definition. Check the query syntax, field types, and analyzer settings.`, + tags: ['end-user-input'], }, { input: `How can I combine multiple conditions in a single $search query?`, @@ -164,6 +187,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/atlas/atlas-search/compound/#compound-operator', ], + tags: ['end-user-input'], }, { input: `Can $search support wildcard or regex queries?`, @@ -171,6 +195,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/atlas/atlas-search/wildcard/#wildcard-operator', ], + tags: ['end-user-input'], }, { input: `How do I analyze why certain documents have a higher search score than others?`, @@ -178,6 +203,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/atlas/atlas-search/scoring/#score-the-documents-in-the-results', ], + tags: ['end-user-input'], }, { input: `Can I use $search to return facets for query results?`, @@ -185,10 +211,12 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/atlas/atlas-search/facet/#facet-collector', ], + tags: ['end-user-input'], }, { input: `Why does $search fail when querying on a field without an Atlas Search Index?`, expected: `$search queries will fail or return no results if the queried field is not included in the Atlas Search Index definition. Unlike standard MongoDB queries which can perform collection scans, $search relies exclusively on its pre-built search index for all operations.`, + tags: ['end-user-input'], }, { input: `What is the difference between $search and $searchMeta?`, @@ -196,6 +224,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/atlas/atlas-search/query-syntax/#choose-the-aggregation-pipeline-stage', ], + tags: ['end-user-input'], }, { input: `How can I highlight terms in $search results?`, @@ -203,6 +232,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/atlas/atlas-search/highlighting/#highlight-search-terms-in-results', ], + tags: ['end-user-input'], }, { input: `Can $search handle multi-language text queries?`, @@ -210,6 +240,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/atlas/atlas-search/analyzers/language/#language-analyzers', ], + tags: ['end-user-input'], }, { input: `What is an aggregation pipeline?`, @@ -218,6 +249,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ 'https://www.mongodb.com/docs/manual/core/aggregation-pipeline/', 'https://www.mongodb.com/docs/compass/create-agg-pipeline/', ], + tags: ['end-user-input'], }, { input: `How do I model data with MongoDB?`, @@ -226,6 +258,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ 'https://www.mongodb.com/docs/manual/data-modeling/#plan-your-schema', 'https://www.mongodb.com/docs/manual/data-modeling/schema-design-process/#designing-your-schema', ], + tags: ['end-user-input'], }, { input: `How is MongoDB data modeling different from data modeling with a relational database?`, @@ -233,6 +266,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/manual/data-modeling/#schema-design--differences-between-relational-and-document-databases', ], + tags: ['end-user-input'], }, { input: `Is MongoDB schemaless?`, @@ -240,10 +274,12 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/manual/data-modeling/#data-modeling', ], + tags: ['end-user-input'], }, { input: `Should I embed my data in an existing collection or put it in a new collection?`, expected: `Consider embedding data when it: \nSimplifies your code \n\nPieces of information have a “has-a,” “contains,” or similar relationship \n\nPieces of information are queried together \n\nPieces of information are updated together \n\nPieces of information are archived at the same time`, + tags: ['end-user-input'], }, { input: `Does MongoDB have foreign keys?`, @@ -251,6 +287,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/manual/reference/operator/aggregation/lookup/#-lookup--aggregation-', ], + tags: ['end-user-input'], }, { input: `How can I set schema validation rules in MongoDB?`, @@ -259,6 +296,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ 'https://www.mongodb.com/docs/manual/core/schema-validation/specify-validation-level/', 'https://www.mongodb.com/docs/compass/validation/', ], + tags: ['end-user-input'], }, { input: `How can I reduce the size of my documents? \n\n[in response to “bloated documents” insight]`, @@ -267,6 +305,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ 'https://www.mongodb.com/docs/manual/data-modeling/design-antipatterns/bloated-documents/', 'https://www.mongodb.com/docs/manual/data-modeling/concepts/embedding-vs-references/#std-label-data-modeling-referencing', ], + tags: ['proactive-performance-insights'], }, { input: `What is the best index for this query?\n\nQuery or agg written, indexes on the collection \n\n[in response to “query executed without index” insight]`, @@ -275,6 +314,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ 'https://www.mongodb.com/docs/manual/core/data-model-operations/', 'https://www.mongodb.com/docs/manual/data-modeling/schema-design-process/create-indexes/', ], + tags: ['proactive-performance-insights'], }, { input: `When should I index a query? \n\n[in response to “query executed without index” insight]`, @@ -284,6 +324,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ 'https://www.mongodb.com/docs/manual/data-modeling/schema-design-process/create-indexes/', 'https://www.mongodb.com/docs/manual/indexes/', ], + tags: ['proactive-performance-insights'], }, { input: `How can I reduce the number of indexes on my collection? \n\nIndexes on the collection \n\n[in response to “high number of indexes on collection” insight]`, @@ -292,6 +333,7 @@ export const generatedEvalCases: SimpleEvalCase[] = [ 'https://www.mongodb.com/docs/manual/data-modeling/design-antipatterns/unnecessary-indexes/#std-label-unnecessary-indexes-antipattern', 'https://www.mongodb.com/docs/atlas/performance-advisor/drop-indexes/', ], + tags: ['proactive-performance-insights'], }, { input: `How do I avoid using $lookup? \n\n$lookup stage in agg builder \n\n[in response to “$lookup usage” insight]`, @@ -299,5 +341,6 @@ export const generatedEvalCases: SimpleEvalCase[] = [ expectedSources: [ 'https://www.mongodb.com/docs/manual/data-modeling/concepts/embedding-vs-references/', ], + tags: ['proactive-performance-insights'], }, ]; diff --git a/packages/compass-assistant/test/eval-cases/model-data.ts b/packages/compass-assistant/test/eval-cases/model-data.ts index e48ff921da1..daea56af33b 100644 --- a/packages/compass-assistant/test/eval-cases/model-data.ts +++ b/packages/compass-assistant/test/eval-cases/model-data.ts @@ -14,5 +14,6 @@ Create indexes: Create indexes to support common query patterns. 'https://www.mongodb.com/docs/manual/data-modeling/#plan-your-schema', 'https://www.mongodb.com/docs/manual/data-modeling/schema-design-process/#designing-your-schema', ], + tags: ['model-data'], }, ]; From 52cdb809c91a778f11f9992a7dab8371e94b00cf Mon Sep 17 00:00:00 2001 From: gagik Date: Wed, 10 Sep 2025 14:58:56 +0200 Subject: [PATCH 3/5] chore: fix tls typo --- packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts b/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts index 1e856810d4f..454705d1fbc 100644 --- a/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts +++ b/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts @@ -32,7 +32,7 @@ const interactionTypeTags = { 'Proactive Perf': 'proactive-performance-insights', 'General network error': 'general-network-error', OIDC: 'oidc', - TSL: 'tsl-ssl', + TLS: 'tls-ssl', SSL: 'tsl-ssl', }; From af841c070f84543fd00bad7be7b32510e1d67f15 Mon Sep 17 00:00:00 2001 From: gagik Date: Wed, 10 Sep 2025 15:05:43 +0200 Subject: [PATCH 4/5] chore: share types --- .../scripts/convert-csv-to-eval-cases.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts b/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts index 454705d1fbc..6b5e69c323e 100644 --- a/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts +++ b/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts @@ -5,6 +5,7 @@ import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'fs'; // eslint-disable-next-line @typescript-eslint/no-restricted-imports import { join, resolve } from 'path'; import { parse } from '@fast-csv/parse'; +import type { SimpleEvalCase } from '../test/assistant.eval'; /** This is copied from the Compass Assistant PD Eval Cases */ type CSVRow = { @@ -16,14 +17,6 @@ type CSVRow = { Notes: string; }; -type SimpleEvalCase = { - name?: string; - input: string; - expected: string; - expectedSources?: string[]; - tags?: string[]; -}; - const interactionTypeTags = { 'End-User Input Only': 'end-user-input', 'Connection Error': 'connection-error', @@ -153,13 +146,15 @@ async function convertCSVToEvalCases() { .filter((link) => link && link.startsWith('http')); } - const tags: string[] = []; + const tags: SimpleEvalCase['tags'][] = []; if (interactionType) { for (const tag of Object.keys(interactionTypeTags)) { if (interactionType.includes(tag)) { tags.push( - interactionTypeTags[tag as keyof typeof interactionTypeTags] + interactionTypeTags[ + tag as keyof typeof interactionTypeTags + ] as unknown as SimpleEvalCase['tags'] ); } } From 4cb7750185f76158474d29afbeacf89dbbe0dd2f Mon Sep 17 00:00:00 2001 From: gagik Date: Wed, 10 Sep 2025 16:23:49 +0200 Subject: [PATCH 5/5] chore: fix check, combine all cases --- packages/compass-assistant/package.json | 3 +- .../scripts/convert-csv-to-eval-cases.ts | 10 ++--- .../test/eval-cases/aggregation-pipeline.ts | 40 ------------------- .../test/eval-cases/atlas-search.ts | 14 ------- .../test/eval-cases/index.ts | 10 +---- .../test/eval-cases/model-data.ts | 19 --------- 6 files changed, 7 insertions(+), 89 deletions(-) delete mode 100644 packages/compass-assistant/test/eval-cases/aggregation-pipeline.ts delete mode 100644 packages/compass-assistant/test/eval-cases/atlas-search.ts delete mode 100644 packages/compass-assistant/test/eval-cases/model-data.ts diff --git a/packages/compass-assistant/package.json b/packages/compass-assistant/package.json index 6c8c19deb9b..00f5b20bc92 100644 --- a/packages/compass-assistant/package.json +++ b/packages/compass-assistant/package.json @@ -49,7 +49,8 @@ "test-ci": "npm run test-cov", "test-ci-electron": "npm run test-electron", "reformat": "npm run eslint . -- --fix && npm run prettier -- --write .", - "convert-eval-cases": "ts-node scripts/convert-csv-to-eval-cases.ts" + "convert-eval-cases": "ts-node scripts/convert-csv-to-eval-cases.ts && npm run reformat", + "eval": "braintrust eval test/assistant.eval.ts --verbose" }, "dependencies": { "@ai-sdk/openai": "^2.0.4", diff --git a/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts b/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts index 6b5e69c323e..20c809b6a6a 100644 --- a/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts +++ b/packages/compass-assistant/scripts/convert-csv-to-eval-cases.ts @@ -26,7 +26,7 @@ const interactionTypeTags = { 'General network error': 'general-network-error', OIDC: 'oidc', TLS: 'tls-ssl', - SSL: 'tsl-ssl', + SSL: 'tls-ssl', }; function escapeString(str: string): string { @@ -146,15 +146,13 @@ async function convertCSVToEvalCases() { .filter((link) => link && link.startsWith('http')); } - const tags: SimpleEvalCase['tags'][] = []; + const tags: SimpleEvalCase['tags'] = []; if (interactionType) { for (const tag of Object.keys(interactionTypeTags)) { if (interactionType.includes(tag)) { tags.push( - interactionTypeTags[ - tag as keyof typeof interactionTypeTags - ] as unknown as SimpleEvalCase['tags'] + interactionTypeTags[tag as keyof typeof interactionTypeTags] as any ); } } @@ -163,8 +161,8 @@ async function convertCSVToEvalCases() { const evalCase: SimpleEvalCase = { input, expected, + tags, ...(expectedSources.length > 0 && { expectedSources }), - ...(tags.length > 0 && { tags }), }; allCases.push(evalCase); diff --git a/packages/compass-assistant/test/eval-cases/aggregation-pipeline.ts b/packages/compass-assistant/test/eval-cases/aggregation-pipeline.ts deleted file mode 100644 index 14b345a88c3..00000000000 --- a/packages/compass-assistant/test/eval-cases/aggregation-pipeline.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { SimpleEvalCase } from '../assistant.eval'; - -export const aggregationPipelineEvalCases: SimpleEvalCase[] = [ - { - input: 'What is an aggregation pipeline?', - expected: `The aggregation pipeline in MongoDB is a framework for data processing and transformation. It consists of a sequence of stages, where each stage performs an operation on the input documents and passes the results to the next stage. Common operations include filtering, grouping, projecting, joining, and calculating values. Aggregation pipelines are powerful for data analysis, reporting, and transformation tasks in MongoDB. - -Compass makes it easy to create and run aggregation pipelines under the Aggregations tab. You may generate an aggregation pipeline with natural language, utilize the visual stage editor, or edit aggregations in the text view. - -Example aggregation pipeline: -db.orders.aggregate([ - // Stage 1: Unwind the array of products - { $unwind: { path: "$products" } }, - - // Stage 2: Match products that cost more than $15 - { $match: { "products.price": { $gt: 15 } } }, - - // Stage 3: Group products by product ID - { $group: { - _id: "$products.prod_id", - product: { $first: "$products.name" }, - total_value: { $sum: "$products.price" }, - quantity: { $sum: 1 } - } - }, - - // Stage 4: Add a product_id field - { $set: { product_id: "$_id" } }, - - // Stage 5: Remove the _id field - { $unset: ["_id"] } -]) -`, - expectedSources: [ - 'https://www.mongodb.com/docs/manual/core/aggregation-pipeline/', - 'https://www.mongodb.com/docs/compass/create-agg-pipeline/', - ], - tags: ['aggregation-pipeline'], - }, -]; diff --git a/packages/compass-assistant/test/eval-cases/atlas-search.ts b/packages/compass-assistant/test/eval-cases/atlas-search.ts deleted file mode 100644 index d69d4b7b6e8..00000000000 --- a/packages/compass-assistant/test/eval-cases/atlas-search.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { SimpleEvalCase } from '../assistant.eval'; - -export const atlasSearchEvalCases: SimpleEvalCase[] = [ - { - input: 'How can I filter docs before running a $search query?', - expected: - 'Because the $search stage must be the first stage in an aggregation pipeline, you cannot pre-filter documents with a preceding $match stage. Instead, filtering should be performed within the $search stage using the filter clause of the compound operator. This allows you to apply predicate queries (e.g., on ranges, dates, or specific terms) to narrow down the dataset before the main query clauses (must or should) are executed. Alternatively, you can filter documents by creating a View—a partial index of your collection that pre-queries and filters out unwanted documents. Note that users need createCollection privileges to build views.', - expectedSources: [ - 'https://www.mongodb.com/docs/atlas/atlas-search/compound/#options', - 'https://www.mongodb.com/docs/atlas/atlas-search/transform-documents-collections/#example--filter-documents', - ], - tags: ['atlas-search'], - }, -]; diff --git a/packages/compass-assistant/test/eval-cases/index.ts b/packages/compass-assistant/test/eval-cases/index.ts index d04287feda2..03ea056c13c 100644 --- a/packages/compass-assistant/test/eval-cases/index.ts +++ b/packages/compass-assistant/test/eval-cases/index.ts @@ -1,12 +1,4 @@ import type { SimpleEvalCase } from '../assistant.eval'; -import { modelDataEvalCases } from './model-data'; -import { atlasSearchEvalCases } from './atlas-search'; -import { aggregationPipelineEvalCases } from './aggregation-pipeline'; import { generatedEvalCases } from './generated-cases'; -export const evalCases: SimpleEvalCase[] = [ - ...atlasSearchEvalCases, - ...generatedEvalCases, - ...aggregationPipelineEvalCases, - ...modelDataEvalCases, -]; +export const evalCases: SimpleEvalCase[] = generatedEvalCases; diff --git a/packages/compass-assistant/test/eval-cases/model-data.ts b/packages/compass-assistant/test/eval-cases/model-data.ts deleted file mode 100644 index daea56af33b..00000000000 --- a/packages/compass-assistant/test/eval-cases/model-data.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { SimpleEvalCase } from '../assistant.eval'; - -export const modelDataEvalCases: SimpleEvalCase[] = [ - { - input: 'How do I model data with MongoDB?', - expected: `Data modeling in MongoDB is highly dependent on how you access your data. To ensure that your data model has a logical structure and achieves optimal performance, plan your schema prior to using your database at a production scale. To determine your data model, use the following schema design process: - -Identify your workload: Identify the operations that your application runs most frequently -Map relationships: Identify the relationships in your application's data and decide whether to link or embed related data. -Apply design patterns: Apply schema design patterns to optimize reads and writes. -Create indexes: Create indexes to support common query patterns. -`, - expectedSources: [ - 'https://www.mongodb.com/docs/manual/data-modeling/#plan-your-schema', - 'https://www.mongodb.com/docs/manual/data-modeling/schema-design-process/#designing-your-schema', - ], - tags: ['model-data'], - }, -];