refactor(@angular/cli): enhance example search with structured data

clydin · clydin · commit cbfda751f76f · 2025-09-08T12:16:39.000-04:00
This commit refactors the `find_examples` MCP tool and its associated database generator to leverage the structured YAML front matter present in the example markdown files.

Key changes:
- The SQLite database schema is now relational, with dedicated columns for `title`, `summary`, and `keywords`.
- An FTS5 virtual table indexes these structured fields, improving search relevance by allowing queries to target specific metadata.
- The build-time database generator (`tools/example_db_generator.js`) now parses and validates the front matter of each example file using Zod. The build will fail if an example is missing a required field (`title`, `summary`), ensuring data integrity.
- The runtime tool (`packages/.../examples.ts`) uses the same parsing logic but will warn and skip invalid files to be more resilient.

This change provides a more robust and accurate foundation for the example search feature, enabling more precise results and paving the way for future enhancements like semantic search.
diff --git a/package.json b/package.json
@@ -145,6 +145,7 @@
     "verdaccio": "6.1.6",
     "verdaccio-auth-memory": "^10.0.0",
     "yargs-parser": "22.0.0",
+    "zod": "4.1.5",
     "zone.js": "^0.15.0"
   },
   "dependenciesMeta": {
diff --git a/packages/angular/cli/src/commands/mcp/tools/examples.ts b/packages/angular/cli/src/commands/mcp/tools/examples.ts
@@ -114,7 +114,9 @@ async function createFindExampleHandler({ exampleDatabasePath }: McpToolContext)
       db = new DatabaseSync(exampleDatabasePath, { readOnly: true });
     }
     if (!queryStatement) {
-      queryStatement = db.prepare('SELECT * from examples WHERE examples MATCH ? ORDER BY rank;');
+      queryStatement = db.prepare(
+        'SELECT content from examples_fts WHERE examples_fts MATCH ? ORDER BY rank;',
+      );
     }
 
     const sanitizedQuery = escapeSearchQuery(query);
@@ -218,24 +220,128 @@ function suppressSqliteWarning() {
   };
 }
 
+/**
+ * A simple YAML front matter parser.
+ *
+ * This function extracts the YAML block enclosed by `---` at the beginning of a string
+ * and parses it into a JavaScript object. It is not a full YAML parser and only
+ * supports simple key-value pairs and string arrays.
+ *
+ * @param content The string content to parse.
+ * @returns A record containing the parsed front matter data.
+ */
+function parseFrontmatter(content: string): Record<string, unknown> {
+  const match = content.match(/^---\r?\n(.*?)\r?\n---/s);
+  if (!match) {
+    return {};
+  }
+
+  const frontmatter = match[1];
+  const data: Record<string, unknown> = {};
+  const lines = frontmatter.split(/\r?\n/);
+
+  let currentKey = '';
+  let isArray = false;
+  const arrayValues: string[] = [];
+
+  for (const line of lines) {
+    const keyValueMatch = line.match(/^([^:]+):\s*(.*)/);
+    if (keyValueMatch) {
+      if (currentKey && isArray) {
+        data[currentKey] = arrayValues.slice();
+        arrayValues.length = 0;
+      }
+
+      const [, key, value] = keyValueMatch;
+      currentKey = key.trim();
+      isArray = value.trim() === '';
+
+      if (!isArray) {
+        data[currentKey] = value.trim();
+      }
+    } else {
+      const arrayItemMatch = line.match(/^\s*-\s*(.*)/);
+      if (arrayItemMatch && currentKey && isArray) {
+        arrayValues.push(arrayItemMatch[1].trim());
+      }
+    }
+  }
+
+  if (currentKey && isArray) {
+    data[currentKey] = arrayValues;
+  }
+
+  return data;
+}
+
 async function setupRuntimeExamples(
   examplesPath: string,
 ): Promise<import('node:sqlite').DatabaseSync> {
   const { DatabaseSync } = await import('node:sqlite');
   const db = new DatabaseSync(':memory:');
 
-  db.exec(`CREATE VIRTUAL TABLE examples USING fts5(content, tokenize = 'porter ascii');`);
+  // Create a relational table to store the structured example data.
+  db.exec(`
+    CREATE TABLE examples (
+      id INTEGER PRIMARY KEY,
+      title TEXT NOT NULL,
+      summary TEXT NOT NULL,
+      keywords TEXT,
+      content TEXT NOT NULL
+    );
+  `);
+
+  // Create an FTS5 virtual table to provide full-text search capabilities.
+  // It indexes the title, summary, keywords, and the full content.
+  db.exec(`
+    CREATE VIRTUAL TABLE examples_fts USING fts5(
+      title,
+      summary,
+      keywords,
+      content,
+      content='examples',
+      content_rowid='id',
+      tokenize = 'porter ascii'
+    );
+  `);
 
-  const insertStatement = db.prepare('INSERT INTO examples(content) VALUES(?);');
+  // Create triggers to keep the FTS table synchronized with the examples table.
+  db.exec(`
+    CREATE TRIGGER examples_after_insert AFTER INSERT ON examples BEGIN
+      INSERT INTO examples_fts(rowid, title, summary, keywords, content)
+      VALUES (new.id, new.title, new.summary, new.keywords, new.content);
+    END;
+  `);
+
+  const insertStatement = db.prepare(
+    'INSERT INTO examples(title, summary, keywords, content) VALUES(?, ?, ?, ?);',
+  );
+
+  const frontmatterSchema = z.object({
+    title: z.string(),
+    summary: z.string(),
+    keywords: z.array(z.string()).optional(),
+  });
 
   db.exec('BEGIN TRANSACTION');
-  for await (const entry of glob('*.md', { cwd: examplesPath, withFileTypes: true })) {
+  for await (const entry of glob('**/*.md', { cwd: examplesPath, withFileTypes: true })) {
     if (!entry.isFile()) {
       continue;
     }
 
-    const example = await readFile(path.join(entry.parentPath, entry.name), 'utf-8');
-    insertStatement.run(example);
+    const content = await readFile(path.join(entry.parentPath, entry.name), 'utf-8');
+    const frontmatter = parseFrontmatter(content);
+
+    const validation = frontmatterSchema.safeParse(frontmatter);
+    if (!validation.success) {
+      // eslint-disable-next-line no-console
+      console.warn(`Skipping invalid example file ${entry.name}:`, validation.error.issues);
+      continue;
+    }
+
+    const { title, summary, keywords } = validation.data;
+
+    insertStatement.run(title, summary, JSON.stringify(keywords ?? []), content);
   }
   db.exec('END TRANSACTION');
 
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel
@@ -35,6 +35,7 @@ js_binary(
     name = "ng_example_db",
     data = [
         "example_db_generator.js",
+        "//:node_modules/zod",
     ],
     entry_point = "example_db_generator.js",
 )
diff --git a/tools/example_db_generator.js b/tools/example_db_generator.js
@@ -6,22 +6,66 @@
  * found in the LICENSE file at https://angular.dev/license
  */
 
-const { readdirSync, readFileSync, mkdirSync, existsSync, rmSync } = require('node:fs');
-const { resolve, dirname } = require('node:path');
+const { globSync, readdirSync, readFileSync, mkdirSync, existsSync, rmSync } = require('node:fs');
+const { resolve, dirname, join } = require('node:path');
 const { DatabaseSync } = require('node:sqlite');
+const { z } = require('zod');
 
-function generate(inPath, outPath) {
-  const examples = [];
+/**
+ * A simple YAML front matter parser.
+ *
+ * This function extracts the YAML block enclosed by `---` at the beginning of a string
+ * and parses it into a JavaScript object. It is not a full YAML parser and only
+ * supports simple key-value pairs and string arrays.
+ *
+ * @param content The string content to parse.
+ * @returns A record containing the parsed front matter data.
+ */
+function parseFrontmatter(content) {
+  const match = content.match(/^---\r?\n(.*?)\r?\n---/s);
+  if (!match) {
+    return {};
+  }
 
-  const entries = readdirSync(resolve(inPath), { withFileTypes: true });
-  for (const entry of entries) {
-    if (!entry.isFile()) {
-      continue;
+  const frontmatter = match[1];
+  const data = {};
+  const lines = frontmatter.split(/\r?\n/);
+
+  let currentKey = '';
+  let isArray = false;
+  const arrayValues = [];
+
+  for (const line of lines) {
+    const keyValueMatch = line.match(/^([^:]+):\s*(.*)/);
+    if (keyValueMatch) {
+      if (currentKey && isArray) {
+        data[currentKey] = arrayValues.slice();
+        arrayValues.length = 0;
+      }
+
+      const [, key, value] = keyValueMatch;
+      currentKey = key.trim();
+      isArray = value.trim() === '';
+
+      if (!isArray) {
+        data[currentKey] = value.trim();
+      }
+    } else {
+      const arrayItemMatch = line.match(/^\s*-\s*(.*)/);
+      if (arrayItemMatch && currentKey && isArray) {
+        arrayValues.push(arrayItemMatch[1].trim());
+      }
     }
+  }
 
-    examples.push(readFileSync(resolve(inPath, entry.name), 'utf-8'));
+  if (currentKey && isArray) {
+    data[currentKey] = arrayValues;
   }
 
+  return data;
+}
+
+function generate(inPath, outPath) {
   const dbPath = outPath;
   mkdirSync(dirname(outPath), { recursive: true });
 
@@ -30,13 +74,69 @@ function generate(inPath, outPath) {
   }
   const db = new DatabaseSync(dbPath);
 
-  db.exec(`CREATE VIRTUAL TABLE examples USING fts5(content, tokenize = 'porter ascii');`);
+  // Create a relational table to store the structured example data.
+  db.exec(`
+    CREATE TABLE examples (
+      id INTEGER PRIMARY KEY,
+      title TEXT NOT NULL,
+      summary TEXT NOT NULL,
+      keywords TEXT,
+      content TEXT NOT NULL
+    );
+  `);
+
+  // Create an FTS5 virtual table to provide full-text search capabilities.
+  db.exec(`
+    CREATE VIRTUAL TABLE examples_fts USING fts5(
+      title,
+      summary,
+      keywords,
+      content,
+      content='examples',
+      content_rowid='id',
+      tokenize = 'porter ascii'
+    );
+  `);
 
-  const insertStatement = db.prepare('INSERT INTO examples(content) VALUES(?);');
+  // Create triggers to keep the FTS table synchronized with the examples table.
+  db.exec(`
+    CREATE TRIGGER examples_after_insert AFTER INSERT ON examples BEGIN
+      INSERT INTO examples_fts(rowid, title, summary, keywords, content)
+      VALUES (new.id, new.title, new.summary, new.keywords, new.content);
+    END;
+  `);
+
+  const insertStatement = db.prepare(
+    'INSERT INTO examples(title, summary, keywords, content) VALUES(?, ?, ?, ?);',
+  );
+
+  const frontmatterSchema = z.object({
+    title: z.string(),
+    summary: z.string(),
+    keywords: z.array(z.string()).optional(),
+  });
 
   db.exec('BEGIN TRANSACTION');
-  for (const example of examples) {
-    insertStatement.run(example);
+  const entries = globSync
+    ? globSync('**/*.md', { cwd: resolve(inPath), withFileTypes: true })
+    : readdirSync(resolve(inPath), { withFileTypes: true });
+  for (const entry of entries) {
+    if (!entry.isFile() || !entry.name.endsWith('.md')) {
+      continue;
+    }
+
+    const content = readFileSync(join(entry.parentPath, entry.name), 'utf-8');
+    const frontmatter = parseFrontmatter(content);
+
+    const validation = frontmatterSchema.safeParse(frontmatter);
+    if (!validation.success) {
+      console.error(`Validation failed for example file: ${entry.name}`);
+      console.error('Issues:', validation.error.issues);
+      throw new Error(`Invalid front matter in ${entry.name}`);
+    }
+
+    const { title, summary, keywords } = validation.data;
+    insertStatement.run(title, summary, JSON.stringify(keywords ?? []), content);
   }
   db.exec('END TRANSACTION');
 

Original file line number	Diff line number	Diff line change
`@@ -35,6 +35,7 @@ js_binary(`
`35`	`35`	`name = "ng_example_db",`
`36`	`36`	`data = [`
`37`	`37`	`"example_db_generator.js",`
	`38`	`+ "//:node_modules/zod",`
`38`	`39`	`],`
`39`	`40`	`entry_point = "example_db_generator.js",`
`40`	`41`	`)`