chore(compass-assistant): add all eval cases, tags and CSV conversion script COMPASS-9823 COMPASS-9758 (#7304)

Author: gagik

package-lock.json

@@ -0,0 +1 @@
+test/eval-cases/eval_cases.csv

packages/compass-assistant/.gitignore

@@ -48,7 +48,9 @@
     "test-watch": "npm run test -- --watch",
     "test-ci": "npm run test-cov",
     "test-ci-electron": "npm run test-electron",
-    "reformat": "npm run eslint . -- --fix && npm run prettier -- --write ."
+    "reformat": "npm run eslint . -- --fix && npm run prettier -- --write .",
+    "convert-eval-cases": "ts-node scripts/convert-csv-to-eval-cases.ts && npm run reformat",
+    "eval": "braintrust eval test/assistant.eval.ts --verbose"
   },
   "dependencies": {
     "@ai-sdk/openai": "^2.0.4",
@@ -67,6 +69,7 @@
     "use-sync-external-store": "^1.5.0"
   },
   "devDependencies": {
+    "@fast-csv/parse": "^5.0.5",
     "@mongodb-js/eslint-config-compass": "^1.4.9",
     "@mongodb-js/mocha-config-compass": "^1.7.1",
     "@mongodb-js/prettier-config-compass": "^1.2.8",

packages/compass-assistant/package.json

@@ -0,0 +1,189 @@
+#!/usr/bin/env ts-node
+/* eslint-disable no-console */
+// eslint-disable-next-line @typescript-eslint/no-restricted-imports
+import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'fs';
+// eslint-disable-next-line @typescript-eslint/no-restricted-imports
+import { join, resolve } from 'path';
+import { parse } from '@fast-csv/parse';
+import type { SimpleEvalCase } from '../test/assistant.eval';
+
+/** This is copied from the Compass Assistant PD Eval Cases */
+type CSVRow = {
+  'Your Name': string;
+  'Interaction Type\n(can add other types)': string;
+  'Input\nHighlighting key: \nHardcoded\n\nContextual passed from client to assistant\n\nUser-entered': string;
+  'Expected Output\n(target 100-200 words, okay to go over if needed)': string;
+  'Expected Links\n(comma separated please)': string;
+  Notes: string;
+};
+
+const interactionTypeTags = {
+  'End-User Input Only': 'end-user-input',
+  'Connection Error': 'connection-error',
+  'DNS Error': 'dns-error',
+  'Explain Plan': 'explain-plan',
+  'Proactive Perf': 'proactive-performance-insights',
+  'General network error': 'general-network-error',
+  OIDC: 'oidc',
+  TLS: 'tls-ssl',
+  SSL: 'tls-ssl',
+};
+
+function escapeString(str: string): string {
+  return str
+    .replace(/\\/g, '\\\\')
+    .replace(/`/g, '\\`')
+    .replace(/\${/g, '\\${')
+    .replace(/\r?\n/g, '\\n') // Handle newlines
+    .replace(/[\u200B-\u200D\uFEFF\u2028\u2029]/g, '') // Remove zero-width spaces and other invisible characters
+    .replace(/[^\S ]/g, ' ') // Replace all whitespace except normal spaces with spaces
+    .replace(/\s+/g, ' ') // Collapse multiple spaces
+    .trim(); // Remove leading/trailing whitespace
+}
+
+function generateEvalCaseFile(cases: SimpleEvalCase[]): string {
+  const caseDefinitions = cases
+    .map((evalCase) => {
+      const sourcesPart =
+        evalCase.expectedSources && evalCase.expectedSources.length > 0
+          ? `    expectedSources: [\n      ${evalCase.expectedSources
+              .map((source) => `'${escapeString(source)}'`)
+              .join(',\n      ')},\n    ],`
+          : '';
+
+      const tagsPart =
+        evalCase.tags && evalCase.tags.length > 0
+          ? `    tags: [\n      ${evalCase.tags
+              .map((tag) => `'${escapeString(tag)}'`)
+              .join(',\n      ')},\n    ],`
+          : '';
+
+      return `  {
+    input: \`${escapeString(evalCase.input)}\`,
+    expected: \`${escapeString(evalCase.expected)}\`,${
+        sourcesPart ? '\n' + sourcesPart : ''
+      }${tagsPart ? '\n' + tagsPart : ''}
+  }`;
+    })
+    .join(',\n');
+
+  return `/** This file is auto-generated by the convert-csv-to-eval-cases script.
+Do not modify this file manually. */
+import type { SimpleEvalCase } from '../assistant.eval';
+
+export const generatedEvalCases: SimpleEvalCase[] = [
+${caseDefinitions},
+];
+`;
+}
+
+async function convertCSVToEvalCases() {
+  const scriptDir = __dirname;
+  const csvFilePath = resolve(scriptDir, '../test/eval-cases/eval_cases.csv');
+  // Check that the CSV file exists
+  if (!existsSync(csvFilePath)) {
+    console.error(
+      `The CSV file does not exist: ${csvFilePath}. Please import it and try again.`
+    );
+    process.exit(1);
+  }
+  const outputDir = resolve(scriptDir, '../test/eval-cases');
+
+  console.log('Converting CSV to eval cases...');
+  console.log(`Reading from: ${csvFilePath}`);
+  console.log(`Output directory: ${outputDir}`);
+
+  // Ensure output directory exists
+  mkdirSync(outputDir, { recursive: true });
+
+  const allCases: SimpleEvalCase[] = [];
+
+  // Read and parse CSV using async/await
+  const csvContent = readFileSync(csvFilePath, 'utf8');
+
+  const rows = await new Promise<CSVRow[]>((resolve, reject) => {
+    const results: CSVRow[] = [];
+    const stream = parse({
+      headers: true,
+    })
+      .on('data', (row: CSVRow) => results.push(row))
+      .on('end', () => resolve(results))
+      .on('error', reject);
+
+    stream.write(csvContent);
+    stream.end();
+  });
+
+  // Process rows
+  for (const row of rows) {
+    // Skip empty rows or header-like rows
+    const input =
+      row[
+        'Input\nHighlighting key: \nHardcoded\n\nContextual passed from client to assistant\n\nUser-entered'
+      ]?.trim();
+    const expected =
+      row[
+        'Expected Output\n(target 100-200 words, okay to go over if needed)'
+      ]?.trim();
+    const yourName = row['Your Name']?.trim();
+    const interactionType =
+      row['Interaction Type\n(can add other types)']?.trim();
+
+    if (!input || !expected || !yourName || !interactionType) {
+      continue; // Skip incomplete rows
+    }
+
+    // Parse expected sources
+    const expectedLinksRaw =
+      row['Expected Links\n(comma separated please)']?.trim();
+    let expectedSources: string[] = [];
+
+    if (expectedLinksRaw) {
+      expectedSources = expectedLinksRaw
+        .replace(/\r?\n/g, ' ') // Replace newlines with spaces first
+        .split(',')
+        .map((link) => link.trim())
+        .filter((link) => link && link.startsWith('http'));
+    }
+
+    const tags: SimpleEvalCase['tags'] = [];
+
+    if (interactionType) {
+      for (const tag of Object.keys(interactionTypeTags)) {
+        if (interactionType.includes(tag)) {
+          tags.push(
+            interactionTypeTags[tag as keyof typeof interactionTypeTags] as any
+          );
+        }
+      }
+    }
+
+    const evalCase: SimpleEvalCase = {
+      input,
+      expected,
+      tags,
+      ...(expectedSources.length > 0 && { expectedSources }),
+    };
+
+    allCases.push(evalCase);
+  }
+
+  console.log(`\nProcessed ${allCases.length} cases`);
+
+  // Generate single file with all cases
+  const filename = 'generated-cases';
+  const filepath = join(outputDir, `${filename}.ts`);
+  const content = generateEvalCaseFile(allCases);
+
+  writeFileSync(filepath, content, 'utf8');
+  console.log(`✓ Generated ${filename}.ts with ${allCases.length} cases`);
+
+  console.log('\n✅ Conversion completed successfully!');
+}
+
+convertCSVToEvalCases().catch((error) => {
+  console.error('❌ Conversion failed:', error);
+  process.exit(1);
+});
+
+export { convertCSVToEvalCases };