diff --git a/package.json b/package.json index a45ea340d64e..1de0091a0ef7 100644 --- a/package.json +++ b/package.json @@ -145,6 +145,7 @@ "verdaccio": "6.1.6", "verdaccio-auth-memory": "^10.0.0", "yargs-parser": "22.0.0", + "zod": "4.1.5", "zone.js": "^0.15.0" }, "dependenciesMeta": { diff --git a/packages/angular/cli/src/commands/mcp/tools/examples.ts b/packages/angular/cli/src/commands/mcp/tools/examples.ts index 21cacd5454c5..2a066535d302 100644 --- a/packages/angular/cli/src/commands/mcp/tools/examples.ts +++ b/packages/angular/cli/src/commands/mcp/tools/examples.ts @@ -8,6 +8,7 @@ import { glob, readFile } from 'node:fs/promises'; import path from 'node:path'; +import type { SQLInputValue } from 'node:sqlite'; import { z } from 'zod'; import { McpToolContext, declareTool } from './tool-registry'; @@ -36,6 +37,15 @@ Examples of queries: - Find lazy loading a route: 'lazy load route' - Find forms with validation: 'form AND (validation OR validator)'`, ), + keywords: z.array(z.string()).optional().describe('Filter examples by specific keywords.'), + required_packages: z + .array(z.string()) + .optional() + .describe('Filter examples by required NPM packages (e.g., "@angular/forms").'), + related_concepts: z + .array(z.string()) + .optional() + .describe('Filter examples by related high-level concepts.'), }); type FindExampleInput = z.infer; @@ -55,7 +65,9 @@ new or evolving features. * **Modern Implementation:** Finding the correct modern syntax for features (e.g., query: 'functional route guard' or 'http client with fetch'). * **Refactoring to Modern Patterns:** Upgrading older code by finding examples of new syntax - (e.g., query: 'built-in control flow' to replace "*ngIf'). + (e.g., query: 'built-in control flow' to replace "*ngIf"). +* **Advanced Filtering:** Combining a full-text search with filters to narrow results. + (e.g., query: 'forms', required_packages: ['@angular/forms'], keywords: ['validation']) * **Tool Selection:** This database primarily contains examples for new and recently updated Angular @@ -64,6 +76,8 @@ new or evolving features. * The examples in this database are the single source of truth for modern Angular coding patterns. * The search query uses a powerful full-text search syntax (FTS5). Refer to the 'query' parameter description for detailed syntax rules and examples. +* You can combine the main 'query' with optional filters like 'keywords', 'required_packages', + and 'related_concepts' to create highly specific searches. `, inputSchema: findExampleInputSchema.shape, outputSchema: { @@ -104,7 +118,7 @@ async function createFindExampleHandler({ exampleDatabasePath }: McpToolContext) suppressSqliteWarning(); - return async ({ query }: FindExampleInput) => { + return async (input: FindExampleInput) => { if (!db) { if (!exampleDatabasePath) { // This should be prevented by the registration logic in mcp-server.ts @@ -113,16 +127,50 @@ async function createFindExampleHandler({ exampleDatabasePath }: McpToolContext) const { DatabaseSync } = await import('node:sqlite'); db = new DatabaseSync(exampleDatabasePath, { readOnly: true }); } - if (!queryStatement) { - queryStatement = db.prepare('SELECT * from examples WHERE examples MATCH ? ORDER BY rank;'); + + const { query, keywords, required_packages, related_concepts } = input; + + // Build the query dynamically + const params: SQLInputValue[] = []; + let sql = 'SELECT content FROM examples_fts'; + const whereClauses = []; + + // FTS query + if (query) { + whereClauses.push('examples_fts MATCH ?'); + params.push(escapeSearchQuery(query)); + } + + // JSON array filters + const addJsonFilter = (column: string, values: string[] | undefined) => { + if (values?.length) { + for (const value of values) { + whereClauses.push(`${column} LIKE ?`); + params.push(`%"${value}"%`); + } + } + }; + + addJsonFilter('keywords', keywords); + addJsonFilter('required_packages', required_packages); + addJsonFilter('related_concepts', related_concepts); + + if (whereClauses.length > 0) { + sql += ` WHERE ${whereClauses.join(' AND ')}`; } - const sanitizedQuery = escapeSearchQuery(query); + // Order the results by relevance using the BM25 algorithm. + // The weights assigned to each column boost the ranking of documents where the + // search term appears in a more important field. + // Column order: title, summary, keywords, required_packages, related_concepts, related_tools, content + sql += ' ORDER BY bm25(examples_fts, 10.0, 5.0, 5.0, 1.0, 2.0, 1.0, 1.0);'; + + const queryStatement = db.prepare(sql); // Query database and return results const examples = []; const textContent = []; - for (const exampleRecord of queryStatement.all(sanitizedQuery)) { + for (const exampleRecord of queryStatement.all(...params)) { const exampleContent = exampleRecord['content'] as string; examples.push({ content: exampleContent }); textContent.push({ type: 'text' as const, text: exampleContent }); @@ -218,24 +266,150 @@ function suppressSqliteWarning() { }; } +/** + * A simple YAML front matter parser. + * + * This function extracts the YAML block enclosed by `---` at the beginning of a string + * and parses it into a JavaScript object. It is not a full YAML parser and only + * supports simple key-value pairs and string arrays. + * + * @param content The string content to parse. + * @returns A record containing the parsed front matter data. + */ +function parseFrontmatter(content: string): Record { + const match = content.match(/^---\r?\n(.*?)\r?\n---/s); + if (!match) { + return {}; + } + + const frontmatter = match[1]; + const data: Record = {}; + const lines = frontmatter.split(/\r?\n/); + + let currentKey = ''; + let isArray = false; + const arrayValues: string[] = []; + + for (const line of lines) { + const keyValueMatch = line.match(/^([^:]+):\s*(.*)/); + if (keyValueMatch) { + if (currentKey && isArray) { + data[currentKey] = arrayValues.slice(); + arrayValues.length = 0; + } + + const [, key, value] = keyValueMatch; + currentKey = key.trim(); + isArray = value.trim() === ''; + + if (!isArray) { + data[currentKey] = value.trim(); + } + } else { + const arrayItemMatch = line.match(/^\s*-\s*(.*)/); + if (arrayItemMatch && currentKey && isArray) { + arrayValues.push(arrayItemMatch[1].trim()); + } + } + } + + if (currentKey && isArray) { + data[currentKey] = arrayValues; + } + + return data; +} + async function setupRuntimeExamples( examplesPath: string, ): Promise { const { DatabaseSync } = await import('node:sqlite'); const db = new DatabaseSync(':memory:'); - db.exec(`CREATE VIRTUAL TABLE examples USING fts5(content, tokenize = 'porter ascii');`); + // Create a relational table to store the structured example data. + db.exec(` + CREATE TABLE examples ( + id INTEGER PRIMARY KEY, + title TEXT NOT NULL, + summary TEXT NOT NULL, + keywords TEXT, + required_packages TEXT, + related_concepts TEXT, + related_tools TEXT, + content TEXT NOT NULL + ); + `); - const insertStatement = db.prepare('INSERT INTO examples(content) VALUES(?);'); + // Create an FTS5 virtual table to provide full-text search capabilities. + db.exec(` + CREATE VIRTUAL TABLE examples_fts USING fts5( + title, + summary, + keywords, + required_packages, + related_concepts, + related_tools, + content, + content='examples', + content_rowid='id', + tokenize = 'porter ascii' + ); + `); + + // Create triggers to keep the FTS table synchronized with the examples table. + db.exec(` + CREATE TRIGGER examples_after_insert AFTER INSERT ON examples BEGIN + INSERT INTO examples_fts(rowid, title, summary, keywords, required_packages, related_concepts, related_tools, content) + VALUES ( + new.id, new.title, new.summary, new.keywords, new.required_packages, new.related_concepts, + new.related_tools, new.content + ); + END; + `); + + const insertStatement = db.prepare( + 'INSERT INTO examples(' + + 'title, summary, keywords, required_packages, related_concepts, related_tools, content' + + ') VALUES(?, ?, ?, ?, ?, ?, ?);', + ); + + const frontmatterSchema = z.object({ + title: z.string(), + summary: z.string(), + keywords: z.array(z.string()).optional(), + required_packages: z.array(z.string()).optional(), + related_concepts: z.array(z.string()).optional(), + related_tools: z.array(z.string()).optional(), + }); db.exec('BEGIN TRANSACTION'); - for await (const entry of glob('*.md', { cwd: examplesPath, withFileTypes: true })) { + for await (const entry of glob('**/*.md', { cwd: examplesPath, withFileTypes: true })) { if (!entry.isFile()) { continue; } - const example = await readFile(path.join(entry.parentPath, entry.name), 'utf-8'); - insertStatement.run(example); + const content = await readFile(path.join(entry.parentPath, entry.name), 'utf-8'); + const frontmatter = parseFrontmatter(content); + + const validation = frontmatterSchema.safeParse(frontmatter); + if (!validation.success) { + // eslint-disable-next-line no-console + console.warn(`Skipping invalid example file ${entry.name}:`, validation.error.issues); + continue; + } + + const { title, summary, keywords, required_packages, related_concepts, related_tools } = + validation.data; + + insertStatement.run( + title, + summary, + JSON.stringify(keywords ?? []), + JSON.stringify(required_packages ?? []), + JSON.stringify(related_concepts ?? []), + JSON.stringify(related_tools ?? []), + content, + ); } db.exec('END TRANSACTION'); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2c9f0e02113d..a0061ea24039 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -316,6 +316,9 @@ importers: yargs-parser: specifier: 22.0.0 version: 22.0.0 + zod: + specifier: 4.1.5 + version: 4.1.5 zone.js: specifier: ^0.15.0 version: 0.15.1 @@ -8950,6 +8953,9 @@ packages: zod@3.25.76: resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + zod@4.1.5: + resolution: {integrity: sha512-rcUUZqlLJgBC33IT3PNMgsCq6TzLQEG/Ei/KTCU0PedSWRMAXoOUN+4t/0H+Q8bdnLPdqUYnvboJT0bn/229qg==} + zone.js@0.15.1: resolution: {integrity: sha512-XE96n56IQpJM7NAoXswY3XRLcWFW83xe0BiAOeMD7K5k5xecOeul3Qcpx6GqEeeHNkW5DWL5zOyTbEfB4eti8w==} @@ -18355,4 +18361,6 @@ snapshots: zod@3.25.76: {} + zod@4.1.5: {} + zone.js@0.15.1: {} diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 65954dc5e6bc..59c817e3aa06 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -35,6 +35,7 @@ js_binary( name = "ng_example_db", data = [ "example_db_generator.js", + "//:node_modules/zod", ], entry_point = "example_db_generator.js", ) diff --git a/tools/example_db_generator.js b/tools/example_db_generator.js index f55303ce6d46..c85b63e497cf 100644 --- a/tools/example_db_generator.js +++ b/tools/example_db_generator.js @@ -6,22 +6,66 @@ * found in the LICENSE file at https://angular.dev/license */ -const { readdirSync, readFileSync, mkdirSync, existsSync, rmSync } = require('node:fs'); -const { resolve, dirname } = require('node:path'); +const { globSync, readdirSync, readFileSync, mkdirSync, existsSync, rmSync } = require('node:fs'); +const { resolve, dirname, join } = require('node:path'); const { DatabaseSync } = require('node:sqlite'); +const { z } = require('zod'); -function generate(inPath, outPath) { - const examples = []; +/** + * A simple YAML front matter parser. + * + * This function extracts the YAML block enclosed by `---` at the beginning of a string + * and parses it into a JavaScript object. It is not a full YAML parser and only + * supports simple key-value pairs and string arrays. + * + * @param content The string content to parse. + * @returns A record containing the parsed front matter data. + */ +function parseFrontmatter(content) { + const match = content.match(/^---\r?\n(.*?)\r?\n---/s); + if (!match) { + return {}; + } - const entries = readdirSync(resolve(inPath), { withFileTypes: true }); - for (const entry of entries) { - if (!entry.isFile()) { - continue; + const frontmatter = match[1]; + const data = {}; + const lines = frontmatter.split(/\r?\n/); + + let currentKey = ''; + let isArray = false; + const arrayValues = []; + + for (const line of lines) { + const keyValueMatch = line.match(/^([^:]+):\s*(.*)/); + if (keyValueMatch) { + if (currentKey && isArray) { + data[currentKey] = arrayValues.slice(); + arrayValues.length = 0; + } + + const [, key, value] = keyValueMatch; + currentKey = key.trim(); + isArray = value.trim() === ''; + + if (!isArray) { + data[currentKey] = value.trim(); + } + } else { + const arrayItemMatch = line.match(/^\s*-\s*(.*)/); + if (arrayItemMatch && currentKey && isArray) { + arrayValues.push(arrayItemMatch[1].trim()); + } } + } - examples.push(readFileSync(resolve(inPath, entry.name), 'utf-8')); + if (currentKey && isArray) { + data[currentKey] = arrayValues; } + return data; +} + +function generate(inPath, outPath) { const dbPath = outPath; mkdirSync(dirname(outPath), { recursive: true }); @@ -30,13 +74,95 @@ function generate(inPath, outPath) { } const db = new DatabaseSync(dbPath); - db.exec(`CREATE VIRTUAL TABLE examples USING fts5(content, tokenize = 'porter ascii');`); + // Create a relational table to store the structured example data. + db.exec(` + CREATE TABLE examples ( + id INTEGER PRIMARY KEY, + title TEXT NOT NULL, + summary TEXT NOT NULL, + keywords TEXT, + required_packages TEXT, + related_concepts TEXT, + related_tools TEXT, + content TEXT NOT NULL + ); + `); + + // Create an FTS5 virtual table to provide full-text search capabilities. + db.exec(` + CREATE VIRTUAL TABLE examples_fts USING fts5( + title, + summary, + keywords, + required_packages, + related_concepts, + related_tools, + content, + content='examples', + content_rowid='id', + tokenize = 'porter ascii' + ); + `); - const insertStatement = db.prepare('INSERT INTO examples(content) VALUES(?);'); + // Create triggers to keep the FTS table synchronized with the examples table. + db.exec(` + CREATE TRIGGER examples_after_insert AFTER INSERT ON examples BEGIN + INSERT INTO examples_fts( + rowid, title, summary, keywords, required_packages, related_concepts, related_tools, + content + ) + VALUES ( + new.id, new.title, new.summary, new.keywords, new.required_packages, + new.related_concepts, new.related_tools, new.content + ); + END; + `); + + const insertStatement = db.prepare( + 'INSERT INTO examples(' + + 'title, summary, keywords, required_packages, related_concepts, related_tools, content' + + ') VALUES(?, ?, ?, ?, ?, ?, ?);', + ); + + const frontmatterSchema = z.object({ + title: z.string(), + summary: z.string(), + keywords: z.array(z.string()).optional(), + required_packages: z.array(z.string()).optional(), + related_concepts: z.array(z.string()).optional(), + related_tools: z.array(z.string()).optional(), + }); db.exec('BEGIN TRANSACTION'); - for (const example of examples) { - insertStatement.run(example); + const entries = globSync + ? globSync('**/*.md', { cwd: resolve(inPath), withFileTypes: true }) + : readdirSync(resolve(inPath), { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith('.md')) { + continue; + } + + const content = readFileSync(join(entry.parentPath, entry.name), 'utf-8'); + const frontmatter = parseFrontmatter(content); + + const validation = frontmatterSchema.safeParse(frontmatter); + if (!validation.success) { + console.error(`Validation failed for example file: ${entry.name}`); + console.error('Issues:', validation.error.issues); + throw new Error(`Invalid front matter in ${entry.name}`); + } + + const { title, summary, keywords, required_packages, related_concepts, related_tools } = + validation.data; + insertStatement.run( + title, + summary, + JSON.stringify(keywords ?? []), + JSON.stringify(required_packages ?? []), + JSON.stringify(related_concepts ?? []), + JSON.stringify(related_tools ?? []), + content, + ); } db.exec('END TRANSACTION');