From 7be32f6086c1f84675a439ab491fae2948dc3d0c Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Sun, 21 Sep 2025 23:57:17 +0200 Subject: [PATCH 01/14] wip --- .cocoignore | 3 +- package.json | 3 + pnpm-lock.yaml | 91 +++++++++ src/lib/cacheDb.ts | 133 ++++++++++++ src/lib/fetchMarkdown.ts | 346 ++++++++++++++++++++++++++++++++ src/lib/log.ts | 32 +++ src/lib/presetCache.ts | 213 ++++++++++++++++++++ src/lib/presets.ts | 267 ++++++++++++++++++++++++ src/lib/utils/pathUtils.test.ts | 335 +++++++++++++++++++++++++++++++ src/lib/utils/pathUtils.ts | 110 ++++++++++ src/lib/utils/prompts.ts | 165 +++++++++++++++ 11 files changed, 1697 insertions(+), 1 deletion(-) create mode 100644 src/lib/cacheDb.ts create mode 100644 src/lib/fetchMarkdown.ts create mode 100644 src/lib/log.ts create mode 100644 src/lib/presetCache.ts create mode 100644 src/lib/presets.ts create mode 100644 src/lib/utils/pathUtils.test.ts create mode 100644 src/lib/utils/pathUtils.ts create mode 100644 src/lib/utils/prompts.ts diff --git a/.cocoignore b/.cocoignore index e28784b..f4e9542 100644 --- a/.cocoignore +++ b/.cocoignore @@ -1,2 +1,3 @@ .claude -.github \ No newline at end of file +.github +*.test.ts \ No newline at end of file diff --git a/package.json b/package.json index 0bbc194..e3b3eaf 100644 --- a/package.json +++ b/package.json @@ -45,16 +45,19 @@ "@types/eslint-scope": "^8.3.2", "@types/estree": "^1.0.8", "@types/node": "^24.3.1", + "@types/tar-stream": "^3.1.4", "@typescript-eslint/types": "^8.43.0", "dotenv": "^17.2.2", "drizzle-kit": "^0.30.2", "drizzle-orm": "^0.40.0", "eslint-config-prettier": "^10.0.1", "globals": "^16.0.0", + "minimatch": "^10.0.3", "prettier": "^3.4.2", "prettier-plugin-svelte": "^3.3.3", "svelte": "^5.0.0", "svelte-check": "^4.0.0", + "tar-stream": "^3.1.7", "typescript": "^5.0.0", "vite": "^7.0.4", "vite-plugin-devtools-json": "^1.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1e51746..26a7865 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -72,6 +72,9 @@ importers: '@types/node': specifier: ^24.3.1 version: 24.5.2 + '@types/tar-stream': + specifier: ^3.1.4 + version: 3.1.4 '@typescript-eslint/types': specifier: ^8.43.0 version: 8.44.0 @@ -90,6 +93,9 @@ importers: globals: specifier: ^16.0.0 version: 16.4.0 + minimatch: + specifier: ^10.0.3 + version: 10.0.3 prettier: specifier: ^3.4.2 version: 3.6.2 @@ -102,6 +108,9 @@ importers: svelte-check: specifier: ^4.0.0 version: 4.3.1(picomatch@4.0.3)(svelte@5.39.2)(typescript@5.9.2) + tar-stream: + specifier: ^3.1.7 + version: 3.1.7 typescript: specifier: ^5.0.0 version: 5.9.2 @@ -636,6 +645,14 @@ packages: resolution: {integrity: sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==} engines: {node: '>=18.18'} + '@isaacs/balanced-match@4.0.1': + resolution: {integrity: sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==} + engines: {node: 20 || >=22} + + '@isaacs/brace-expansion@5.0.0': + resolution: {integrity: sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==} + engines: {node: 20 || >=22} + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -1365,6 +1382,9 @@ packages: '@types/node@24.5.2': resolution: {integrity: sha512-FYxk1I7wPv3K2XBaoyH2cTnocQEu8AOZ60hPbsyukMPLv5/5qr7V1i8PLHdl6Zf87I+xZXFvPCXYjiTFq+YSDQ==} + '@types/tar-stream@3.1.4': + resolution: {integrity: sha512-921gW0+g29mCJX0fRvqeHzBlE/XclDaAG0Ousy1LCghsOhvaKacDeRGEVzQP9IPfKn8Vysy7FEXAIxycpc/CMg==} + '@types/ws@8.18.1': resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} @@ -1541,9 +1561,20 @@ packages: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} engines: {node: '>= 0.4'} + b4a@1.7.1: + resolution: {integrity: sha512-ZovbrBV0g6JxK5cGUF1Suby1vLfKjv4RWi8IxoaO/Mon8BDD9I21RxjHFtgQ+kskJqLAVyQZly3uMBui+vhc8Q==} + peerDependencies: + react-native-b4a: '*' + peerDependenciesMeta: + react-native-b4a: + optional: true + balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + bare-events@2.7.0: + resolution: {integrity: sha512-b3N5eTW1g7vXkw+0CXh/HazGTcO5KYuu/RCNaJbDMPI6LHDi+7qe8EmxKUVe1sUbY2KZOVZFyj62x0OEz9qyAA==} + bindings@1.5.0: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} @@ -2016,6 +2047,9 @@ packages: fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + fast-glob@3.3.3: resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} engines: {node: '>=8.6.0'} @@ -2358,6 +2392,10 @@ packages: resolution: {integrity: sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA==} engines: {node: '>= 0.6'} + minimatch@10.0.3: + resolution: {integrity: sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw==} + engines: {node: 20 || >=22} + minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} @@ -2790,6 +2828,9 @@ packages: std-env@3.9.0: resolution: {integrity: sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw==} + streamx@2.22.1: + resolution: {integrity: sha512-znKXEBxfatz2GBNK02kRnCXjV+AA4kjZIUxeWSr3UGirZMJfTE9uiwKHobnbgxWyL/JWro8tTq+vOqAK1/qbSA==} + string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} @@ -2845,10 +2886,16 @@ packages: tailwind-merge@2.6.0: resolution: {integrity: sha512-P+Vu1qXfzediirmHOC3xKGAYeZtPcV9g76X+xg2FD4tYgR71ewMA35Y3sCz3zhiN/dwefRpJX0yBcgwi1fXNQA==} + tar-stream@3.1.7: + resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} + tar@7.4.3: resolution: {integrity: sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==} engines: {node: '>=18'} + text-decoder@1.2.3: + resolution: {integrity: sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==} + tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -3479,6 +3526,12 @@ snapshots: '@humanwhocodes/retry@0.4.3': {} + '@isaacs/balanced-match@4.0.1': {} + + '@isaacs/brace-expansion@5.0.0': + dependencies: + '@isaacs/balanced-match': 4.0.1 + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -4189,6 +4242,10 @@ snapshots: dependencies: undici-types: 7.12.0 + '@types/tar-stream@3.1.4': + dependencies: + '@types/node': 24.5.2 + '@types/ws@8.18.1': dependencies: '@types/node': 24.5.2 @@ -4407,8 +4464,13 @@ snapshots: axobject-query@4.1.0: {} + b4a@1.7.1: {} + balanced-match@1.0.2: {} + bare-events@2.7.0: + optional: true + bindings@1.5.0: dependencies: file-uri-to-path: 1.0.0 @@ -4879,6 +4941,8 @@ snapshots: fast-deep-equal@3.1.3: {} + fast-fifo@1.3.2: {} + fast-glob@3.3.3: dependencies: '@nodelib/fs.stat': 2.0.5 @@ -5198,6 +5262,10 @@ snapshots: dependencies: mime-db: 1.54.0 + minimatch@10.0.3: + dependencies: + '@isaacs/brace-expansion': 5.0.0 + minimatch@3.1.2: dependencies: brace-expansion: 1.1.12 @@ -5601,6 +5669,15 @@ snapshots: std-env@3.9.0: {} + streamx@2.22.1: + dependencies: + fast-fifo: 1.3.2 + text-decoder: 1.2.3 + optionalDependencies: + bare-events: 2.7.0 + transitivePeerDependencies: + - react-native-b4a + string-width@4.2.3: dependencies: emoji-regex: 8.0.0 @@ -5677,6 +5754,14 @@ snapshots: tailwind-merge@2.6.0: {} + tar-stream@3.1.7: + dependencies: + b4a: 1.7.1 + fast-fifo: 1.3.2 + streamx: 2.22.1 + transitivePeerDependencies: + - react-native-b4a + tar@7.4.3: dependencies: '@isaacs/fs-minipass': 4.0.1 @@ -5686,6 +5771,12 @@ snapshots: mkdirp: 3.0.1 yallist: 5.0.0 + text-decoder@1.2.3: + dependencies: + b4a: 1.7.1 + transitivePeerDependencies: + - react-native-b4a + tinybench@2.9.0: {} tinyexec@0.3.2: {} diff --git a/src/lib/cacheDb.ts b/src/lib/cacheDb.ts new file mode 100644 index 0000000..b659821 --- /dev/null +++ b/src/lib/cacheDb.ts @@ -0,0 +1,133 @@ +import { maybeInitializePool } from '$lib/server/db'; +import type { Pool } from 'pg'; + +export interface CacheEntry { + id: number; + cache_key: string; + data: Buffer; + size_bytes: number; + expires_at: Date; + created_at: Date; +} + +export class CacheDbService { + private db: Pool; + private defaultTTL: number; + + constructor(db?: Pool, defaultTTLMinutes: number = 60) { + this.db = db || maybeInitializePool(); + this.defaultTTL = defaultTTLMinutes; + } + + async get(key: string): Promise { + const query = ` + SELECT data, expires_at + FROM cache + WHERE cache_key = $1 AND expires_at > NOW() + `; + + try { + const result = await this.db.query(query, [key]); + + if (result.rows.length === 0) { + return null; + } + + return result.rows[0].data; + } catch (error) { + console.error('Error getting cache entry:', error); + return null; + } + } + + async set(key: string, data: Buffer, ttlMinutes?: number): Promise { + const ttl = ttlMinutes || this.defaultTTL; + const query = ` + INSERT INTO cache (cache_key, data, size_bytes, expires_at) + VALUES ($1, $2, $3, NOW() + INTERVAL '${ttl} minutes') + ON CONFLICT (cache_key) + DO UPDATE SET + data = EXCLUDED.data, + size_bytes = EXCLUDED.size_bytes, + expires_at = EXCLUDED.expires_at + `; + + try { + await this.db.query(query, [key, data, data.length]); + } catch (error) { + console.error('Error setting cache entry:', error); + throw error; + } + } + + async delete(key: string): Promise { + const query = 'DELETE FROM cache WHERE cache_key = $1'; + + try { + const result = await this.db.query(query, [key]); + return (result.rowCount ?? 0) > 0; + } catch (error) { + console.error('Error deleting cache entry:', error); + return false; + } + } + + async clear(): Promise { + const query = 'DELETE FROM cache'; + + try { + await this.db.query(query); + } catch (error) { + console.error('Error clearing cache:', error); + throw error; + } + } + + async deleteExpired(): Promise { + const query = 'DELETE FROM cache WHERE expires_at <= NOW()'; + + try { + const result = await this.db.query(query); + return result.rowCount ?? 0; + } catch (error) { + console.error('Error deleting expired cache entries:', error); + return 0; + } + } + + async getStatus(): Promise<{ count: number; keys: string[]; totalSizeBytes: number }> { + const query = ` + SELECT cache_key, size_bytes + FROM cache + WHERE expires_at > NOW() + ORDER BY created_at DESC + `; + + try { + const result = await this.db.query(query); + const keys = result.rows.map((row) => row.cache_key); + const totalSizeBytes = result.rows.reduce((sum, row) => sum + row.size_bytes, 0); + + return { + count: result.rows.length, + keys, + totalSizeBytes, + }; + } catch (error) { + console.error('Error getting cache status:', error); + return { count: 0, keys: [], totalSizeBytes: 0 }; + } + } + + async has(key: string): Promise { + const query = 'SELECT 1 FROM cache WHERE cache_key = $1 AND expires_at > NOW()'; + + try { + const result = await this.db.query(query, [key]); + return result.rows.length > 0; + } catch (error) { + console.error('Error checking cache entry:', error); + return false; + } + } +} diff --git a/src/lib/fetchMarkdown.ts b/src/lib/fetchMarkdown.ts new file mode 100644 index 0000000..2771eef --- /dev/null +++ b/src/lib/fetchMarkdown.ts @@ -0,0 +1,346 @@ +import type { PresetConfig } from '$lib/presets'; +import { env } from '$env/dynamic/private'; +import tarStream from 'tar-stream'; +import { Readable } from 'stream'; +import { createGunzip } from 'zlib'; +import { minimatch } from 'minimatch'; +import { getPresetContent } from './presetCache'; +import { CacheDbService } from '$lib/server/cacheDb'; +import { log, logAlways, logErrorAlways } from '$lib/log'; +import { cleanTarballPath } from '$lib/utils/pathUtils'; + +let cacheService: CacheDbService | null = null; + +function getCacheService(): CacheDbService { + if (!cacheService) { + cacheService = new CacheDbService(); + } + return cacheService; +} + +function sortFilesWithinGroup(files: string[]): string[] { + return files.sort((a, b) => { + const aPath = a.split('\n')[0].replace('## ', ''); + const bPath = b.split('\n')[0].replace('## ', ''); + + // Check if one path is a parent of the other + if (bPath.startsWith(aPath.replace('/index.md', '/'))) return -1; + if (aPath.startsWith(bPath.replace('/index.md', '/'))) return 1; + + return aPath.localeCompare(bPath); + }); +} + +export async function fetchRepositoryTarball(owner: string, repo: string): Promise { + const cacheKey = `${owner}/${repo}`; + const cache = getCacheService(); + + const cachedBuffer = await cache.get(cacheKey); + if (cachedBuffer) { + logAlways(`Using cached tarball for ${cacheKey} from database`); + return cachedBuffer; + } + + const url = `https://api.github.com/repos/${owner}/${repo}/tarball`; + + logAlways(`Fetching tarball from: ${url}`); + + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${env.GITHUB_TOKEN}`, + Accept: 'application/vnd.github.v3.raw', + }, + }); + + if (!response.ok) { + throw new Error(`Failed to fetch tarball: ${response.statusText}`); + } + + if (!response.body) { + throw new Error('Response body is null'); + } + + const chunks: Uint8Array[] = []; + const reader = response.body.getReader(); + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + } + + const buffer = Buffer.concat(chunks); + + // Cache the buffer in database with 60 minutes TTL + await cache.set(cacheKey, buffer, 60); + + return buffer; +} + +export async function processMarkdownFromTarball( + tarballBuffer: Buffer, + presetConfig: PresetConfig, + includePathInfo: boolean, +): Promise { + const { glob, ignore = [], minimize = undefined } = presetConfig; + + // Create a Map to store files for each glob pattern while maintaining order + const globResults = new Map(); + const filePathsByPattern = new Map(); + glob.forEach((pattern) => { + globResults.set(pattern, []); + filePathsByPattern.set(pattern, []); + }); + + const extractStream = tarStream.extract(); + + let processedFiles = 0; + let matchedFiles = 0; + + extractStream.on('entry', (header, stream, next) => { + processedFiles++; + let matched = false; + + for (const pattern of glob) { + if (shouldIncludeFile(header.name, pattern, ignore)) { + matched = true; + matchedFiles++; + + if (header.type === 'file') { + let content = ''; + stream.on('data', (chunk) => (content += chunk.toString())); + stream.on('end', () => { + // Use the unified path utility to clean tarball paths + const cleanPath = cleanTarballPath(header.name); + + const processedContent = minimizeContent(content, minimize); + + if (includePathInfo) { + const files = globResults.get(pattern) || []; + files.push({ + path: cleanPath, + content: processedContent, + }); + globResults.set(pattern, files); + } else { + const contentWithHeader = `## ${cleanPath}\n\n${processedContent}`; + + const files = globResults.get(pattern) || []; + files.push(contentWithHeader); + globResults.set(pattern, files); + } + + const paths = filePathsByPattern.get(pattern) || []; + paths.push(cleanPath); + filePathsByPattern.set(pattern, paths); + + next(); + }); + return; + } + } + } + + if (!matched) { + stream.resume(); + next(); + } + }); + + const tarballStream = Readable.from(tarballBuffer); + const gunzipStream = createGunzip(); + + tarballStream.pipe(gunzipStream).pipe(extractStream); + + await new Promise((resolve) => extractStream.on('finish', resolve)); + + logAlways(`Total files processed: ${processedFiles}`); + logAlways(`Files matching glob: ${matchedFiles}`); + log('\nFinal file order:'); + + glob.forEach((pattern, index) => { + const paths = filePathsByPattern.get(pattern) || []; + const sortedPaths = includePathInfo + ? paths + : sortFilesWithinGroup(paths.map((p) => `## ${p}`)).map((p) => p.replace('## ', '')); + + if (sortedPaths.length > 0) { + log(`\nGlob pattern ${index + 1}: ${pattern}`); + sortedPaths.forEach((path, i) => { + log(` ${i + 1}. ${path}`); + }); + } + }); + + // Combine results in the order of glob patterns + const orderedResults: unknown[] = []; + for (const pattern of glob) { + const filesForPattern = globResults.get(pattern) || []; + if (includePathInfo) { + orderedResults.push(...filesForPattern); + } else { + orderedResults.push(...sortFilesWithinGroup(filesForPattern as string[])); + } + } + + return orderedResults as string[] | { path: string; content: string }[]; +} + +function shouldIncludeFile(filename: string, glob: string, ignore: string[] = []): boolean { + const shouldIgnore = ignore.some((pattern) => minimatch(filename, pattern)); + if (shouldIgnore) { + logAlways(`❌ Ignored by pattern: ${filename}`); + return false; + } + + return minimatch(filename, glob); +} + +export async function clearRepositoryCache(): Promise { + const cache = getCacheService(); + await cache.clear(); + logAlways('Repository cache cleared'); +} + +export async function getRepositoryCacheStatus(): Promise<{ + size: number; + repositories: string[]; + totalSizeBytes: number; +}> { + const cache = getCacheService(); + const status = await cache.getStatus(); + return { + size: status.count, + repositories: status.keys, + totalSizeBytes: status.totalSizeBytes, + }; +} + +export interface MinimizeOptions { + normalizeWhitespace?: boolean; + removeLegacy?: boolean; + removePlaygroundLinks?: boolean; + removePrettierIgnore?: boolean; + removeNoteBlocks?: boolean; + removeDetailsBlocks?: boolean; + removeHtmlComments?: boolean; + removeDiffMarkers?: boolean; +} + +const defaultOptions: MinimizeOptions = { + normalizeWhitespace: false, + removeLegacy: false, + removePlaygroundLinks: false, + removePrettierIgnore: true, + removeNoteBlocks: true, + removeDetailsBlocks: true, + removeHtmlComments: false, + removeDiffMarkers: true, +}; + +function removeQuoteBlocks(content: string, blockType: string): string { + return content + .split('\n') + .reduce((acc: string[], line: string, index: number, lines: string[]) => { + // If we find a block (with or without additional text), skip it and all subsequent blockquote lines + if (line.trim().startsWith(`> [!${blockType}]`)) { + // Skip all subsequent lines that are part of the blockquote + let i = index; + while (i < lines.length && (lines[i].startsWith('>') || lines[i].trim() === '')) { + i++; + } + // Update the index to skip all these lines + index = i - 1; + return acc; + } + + acc.push(line); + return acc; + }, []) + .join('\n'); +} + +function removeDiffMarkersFromContent(content: string): string { + let inCodeBlock = false; + const lines = content.split('\n'); + const processedLines = lines.map((line) => { + // Track if we're entering or leaving a code block + // eslint-disable-next-line no-useless-escape + if (line.trim().startsWith('\`\`\`')) { + inCodeBlock = !inCodeBlock; + return line; + } + + if (inCodeBlock) { + // Handle lines that end with --- or +++ with possible whitespace after + // eslint-disable-next-line no-useless-escape + line = line.replace(/(\+{3}|\-{3})[\s]*$/g, ''); + + // Handle triple markers at start while preserving indentation + // This captures the whitespace before the marker and adds it back + // eslint-disable-next-line no-useless-escape + line = line.replace(/^(\s*)(\+{3}|\-{3})\s*/g, '$1'); + + // Handle single + or - markers at start while preserving indentation + // eslint-disable-next-line no-useless-escape + line = line.replace(/^(\s*)[\+\-](\s)/g, '$1'); + + // Handle multi-line diff blocks where --- or +++ might be in the middle of line + // eslint-disable-next-line no-useless-escape + line = line.replace(/[\s]*(\+{3}|\-{3})[\s]*/g, ''); + } + + return line; + }); + + return processedLines.join('\n'); +} + +export function minimizeContent(content: string, options?: Partial): string { + const settings: MinimizeOptions = options ? { ...defaultOptions, ...options } : defaultOptions; + + let minimized = content; + + minimized = minimized.replace(/NOTE: do not edit this file, it is generated in.*$/gm, ''); + + if (settings.removeDiffMarkers) { + minimized = removeDiffMarkersFromContent(minimized); + } + + if (settings.removeLegacy) { + minimized = removeQuoteBlocks(minimized, 'LEGACY'); + } + + if (settings.removeNoteBlocks) { + minimized = removeQuoteBlocks(minimized, 'NOTE'); + } + + if (settings.removeDetailsBlocks) { + minimized = removeQuoteBlocks(minimized, 'DETAILS'); + } + + if (settings.removePlaygroundLinks) { + // Replace playground URLs with /[link] but keep the original link text + minimized = minimized.replace(/\[([^\]]+)\]\(\/playground[^)]+\)/g, '[$1](/REMOVED)'); + } + + if (settings.removePrettierIgnore) { + minimized = minimized + .split('\n') + .filter((line) => line.trim() !== '') + .join('\n'); + } + + if (settings.removeHtmlComments) { + // Replace all HTML comments (including multi-line) with empty string + minimized = minimized.replace(//g, ''); + } + + if (settings.normalizeWhitespace) { + minimized = minimized.replace(/\s+/g, ' '); + } + + minimized = minimized.trim(); + + return minimized; +} diff --git a/src/lib/log.ts b/src/lib/log.ts new file mode 100644 index 0000000..4980037 --- /dev/null +++ b/src/lib/log.ts @@ -0,0 +1,32 @@ +import { dev } from '$app/environment'; +// eslint-disable-next-line @typescript-eslint/naming-convention, func-style +export const log = (...props: unknown[]) => { + if (dev) { + console.log(...props); + } +}; + +// eslint-disable-next-line @typescript-eslint/naming-convention, func-style +export const logWarning = (...props: unknown[]) => { + if (dev) { + console.warn(...props); + } +}; +// eslint-disable-next-line @typescript-eslint/naming-convention, func-style +export const logError = (...props: unknown[]) => { + if (dev) { + console.error(...props); + } +}; +// eslint-disable-next-line @typescript-eslint/naming-convention, func-style +export const logAlways = (...props: unknown[]) => { + console.log(...props); +}; +// eslint-disable-next-line @typescript-eslint/naming-convention, func-style +export const logWarningAlways = (...props: unknown[]) => { + console.warn(...props); +}; +// eslint-disable-next-line @typescript-eslint/naming-convention, func-style +export const logErrorAlways = (...props: unknown[]) => { + console.error(...props); +}; diff --git a/src/lib/presetCache.ts b/src/lib/presetCache.ts new file mode 100644 index 0000000..704b3fc --- /dev/null +++ b/src/lib/presetCache.ts @@ -0,0 +1,213 @@ +import { ContentSyncService } from '$lib/server/contentSync'; +import { presets } from '$lib/presets'; +import { log, logAlways, logErrorAlways } from '$lib/log'; +import { cleanDocumentationPath } from '$lib/utils/pathUtils'; +import { CacheDbService } from '$lib/server/cacheDb'; + +// Maximum age of cached content in milliseconds (24 hours) +export const MAX_CACHE_AGE_MS = 24 * 60 * 60 * 1000; + +let cacheService: CacheDbService | null = null; + +function getCacheService(): CacheDbService { + if (!cacheService) { + cacheService = new CacheDbService(); + } + return cacheService; +} + +export async function getPresetContent(presetKey: string): Promise { + try { + const preset = presets[presetKey]; + if (!preset) { + log(`Preset not found: ${presetKey}`); + return null; + } + + // Check cache first + const cache = getCacheService(); + const cacheKey = `preset:${presetKey}`; + + try { + const cachedData = await cache.get(cacheKey); + if (cachedData) { + const cachedContent = cachedData.toString('utf8'); + logAlways(`Using cached content for preset ${presetKey}`); + return cachedContent; + } + } catch (cacheError) { + logErrorAlways(`Error reading cache for preset ${presetKey}:`, cacheError); + // Continue with normal flow if cache read fails + } + + // Try to get files from the content table first + let filesWithPaths = await ContentSyncService.getPresetContentFromDb(presetKey); + + // If no content in database, fetch from GitHub and sync + if (!filesWithPaths || filesWithPaths.length === 0) { + logAlways(`No content in database for preset ${presetKey}, fetching from GitHub...`); + + // Sync the repository first + await ContentSyncService.syncRepository(); + + // Try again from database + filesWithPaths = await ContentSyncService.getPresetContentFromDb(presetKey); + + if (!filesWithPaths || filesWithPaths.length === 0) { + log(`Still no content found for preset: ${presetKey} after sync`); + return null; + } + } + + // Format files with headers and preserve the order from database + // The files are already correctly ordered by glob pattern precedence + // Use the unified path utility to clean paths + const files = filesWithPaths.map((f) => { + const cleanPath = cleanDocumentationPath(f.path); + return `## ${cleanPath}\n\n${f.content}`; + }); + + // DO NOT sort - files are already in correct glob pattern order from ContentSyncService + const content = files.join('\n\n'); + + logAlways(`Generated content for ${presetKey} on-demand (${filesWithPaths.length} files)`); + + // Cache the generated content for 1 hour (60 minutes) + try { + const contentBuffer = Buffer.from(content, 'utf8'); + await cache.set(cacheKey, contentBuffer, 60); // 60 minutes TTL + logAlways(`Cached content for preset ${presetKey} (expires in 1 hour)`); + } catch (cacheError) { + logErrorAlways(`Error caching content for preset ${presetKey}:`, cacheError); + // Don't fail the request if caching fails + } + + return content; + } catch (error) { + logErrorAlways(`Error generating preset content for ${presetKey}:`, error); + return null; + } +} + +export async function getPresetSizeKb(presetKey: string): Promise { + try { + const content = await getPresetContent(presetKey); + if (!content) { + return null; + } + + const sizeKb = Math.floor(new TextEncoder().encode(content).length / 1024); + return sizeKb; + } catch (error) { + logErrorAlways(`Error calculating preset size for ${presetKey}:`, error); + return null; + } +} + +export async function isPresetStale(presetKey: string): Promise { + try { + // Check if the repository content is stale + return await ContentSyncService.isRepositoryContentStale(); + } catch (error) { + logErrorAlways(`Error checking preset staleness for ${presetKey}:`, error); + return true; // On error, assume stale + } +} + +export async function presetExists(presetKey: string): Promise { + try { + const preset = presets[presetKey]; + if (!preset) { + return false; + } + + // A preset "exists" if it's defined in presets.ts + // The content will be generated on-demand + return true; + } catch (error) { + logErrorAlways(`Error checking preset existence for ${presetKey}:`, error); + return false; + } +} + +export async function getPresetMetadata(presetKey: string): Promise<{ + size_kb: number; + document_count: number; + updated_at: Date; + is_stale: boolean; +} | null> { + try { + const preset = presets[presetKey]; + if (!preset) { + return null; + } + + // Try to get files from content table or GitHub + const content = await getPresetContent(presetKey); + if (!content) { + return null; + } + + // Get the files again to count them (this will use cached data) + const filesWithPaths = await ContentSyncService.getPresetContentFromDb(presetKey); + const documentCount = filesWithPaths?.length || 0; + + const sizeKb = Math.floor(new TextEncoder().encode(content).length / 1024); + const isStale = await isPresetStale(presetKey); + + return { + size_kb: sizeKb, + document_count: documentCount, + updated_at: new Date(), // Since it's generated on-demand, it's always "now" + is_stale: isStale, + }; + } catch (error) { + logErrorAlways(`Error getting preset metadata for ${presetKey}:`, error); + return null; + } +} + +/** + * Clear the cache for a specific preset + */ +export async function clearPresetCache(presetKey: string): Promise { + try { + const cache = getCacheService(); + const cacheKey = `preset:${presetKey}`; + const success = await cache.delete(cacheKey); + + if (success) { + logAlways(`Cleared cache for preset ${presetKey}`); + } + + return success; + } catch (error) { + logErrorAlways(`Error clearing cache for preset ${presetKey}:`, error); + return false; + } +} + +/** + * Clear cache for all presets + */ +export async function clearAllPresetCaches(): Promise { + try { + const cache = getCacheService(); + const allPresetKeys = Object.keys(presets); + let clearedCount = 0; + + for (const presetKey of allPresetKeys) { + const cacheKey = `preset:${presetKey}`; + const success = await cache.delete(cacheKey); + if (success) { + clearedCount++; + } + } + + logAlways(`Cleared cache for ${clearedCount} presets`); + return clearedCount; + } catch (error) { + logErrorAlways(`Error clearing all preset caches:`, error); + return 0; + } +} diff --git a/src/lib/presets.ts b/src/lib/presets.ts new file mode 100644 index 0000000..fbf4e4d --- /dev/null +++ b/src/lib/presets.ts @@ -0,0 +1,267 @@ +import type { MinimizeOptions } from './fetchMarkdown'; +import { SVELTE_5_PROMPT } from '$lib/utils/prompts'; + +export type PresetConfig = { + title: string; + description?: string; + glob: string[]; + ignore?: string[]; + prompt?: string; + minimize?: MinimizeOptions; + distilled?: boolean; + distilledFilenameBase?: string; +}; + +// eslint-disable-next-line @typescript-eslint/naming-convention +export const combinedPresets: Record = { + 'svelte-complete-distilled': { + title: '🔮 Svelte + SvelteKit (Recommended - LLM Distilled)', + description: 'AI-condensed version of the docs focused on code examples and key concepts', + glob: [ + // Svelte + '**/apps/svelte.dev/content/docs/svelte/**/*.md', + // SvelteKit + '**/apps/svelte.dev/content/docs/kit/**/*.md', + ], + minimize: { + normalizeWhitespace: false, + removeLegacy: true, + removePlaygroundLinks: true, + removePrettierIgnore: true, + removeNoteBlocks: false, + removeDetailsBlocks: false, + removeHtmlComments: true, + removeDiffMarkers: true, + }, + ignore: [ + // Svelte ignores (same as medium preset) + '**/apps/svelte.dev/content/docs/svelte/07-misc/04-custom-elements.md', + '**/apps/svelte.dev/content/docs/svelte/07-misc/06-v4-migration-guide.md', + '**/apps/svelte.dev/content/docs/svelte/07-misc/07-v5-migration-guide.md', + '**/apps/svelte.dev/content/docs/svelte/07-misc/99-faq.md', + '**/apps/svelte.dev/content/docs/svelte/07-misc/xx-reactivity-indepth.md', + '**/apps/svelte.dev/content/docs/svelte/98-reference/21-svelte-legacy.md', + '**/apps/svelte.dev/content/docs/svelte/99-legacy/**/*.md', + '**/apps/svelte.dev/content/docs/svelte/98-reference/**/*.md', + '**/xx-*.md', + // SvelteKit ignores (same as medium preset) + '**/apps/svelte.dev/content/docs/kit/25-build-and-deploy/*adapter-*.md', + '**/apps/svelte.dev/content/docs/kit/25-build-and-deploy/99-writing-adapters.md', + '**/apps/svelte.dev/content/docs/kit/30-advanced/70-packaging.md', + '**/apps/svelte.dev/content/docs/kit/40-best-practices/05-performance.md', + '**/apps/svelte.dev/content/docs/kit/40-best-practices/10-accessibility.md', + '**/apps/svelte.dev/content/docs/kit/60-appendix/**/*.md', + '**/apps/svelte.dev/content/docs/kit/98-reference/**/*.md', + '**/xx-*.md', + ], + prompt: SVELTE_5_PROMPT, + distilled: true, + distilledFilenameBase: 'svelte-complete-distilled', + }, + 'svelte-complete-medium': { + title: '⭐️ Svelte + SvelteKit (Medium preset)', + description: + 'Complete Svelte + SvelteKit docs excluding certain advanced sections, legacy, notes and migration docs', + glob: [ + // Svelte + '**/apps/svelte.dev/content/docs/svelte/**/*.md', + // SvelteKit + '**/apps/svelte.dev/content/docs/kit/**/*.md', + ], + ignore: [ + // Svelte ignores + '**/apps/svelte.dev/content/docs/svelte/07-misc/04-custom-elements.md', + '**/apps/svelte.dev/content/docs/svelte/07-misc/06-v4-migration-guide.md', + '**/apps/svelte.dev/content/docs/svelte/07-misc/07-v5-migration-guide.md', + '**/apps/svelte.dev/content/docs/svelte/07-misc/99-faq.md', + '**/apps/svelte.dev/content/docs/svelte/07-misc/xx-reactivity-indepth.md', + '**/apps/svelte.dev/content/docs/svelte/98-reference/21-svelte-legacy.md', + '**/apps/svelte.dev/content/docs/svelte/99-legacy/**/*.md', + '**/apps/svelte.dev/content/docs/svelte/98-reference/30-runtime-errors.md', + '**/apps/svelte.dev/content/docs/svelte/98-reference/30-runtime-warnings.md', + '**/apps/svelte.dev/content/docs/svelte/98-reference/30-compiler-errors.md', + '**/apps/svelte.dev/content/docs/svelte/98-reference/30-compiler-warnings.md', + '**/xx-*.md', + // SvelteKit ignores + '**/apps/svelte.dev/content/docs/kit/25-build-and-deploy/*adapter-*.md', + '**/apps/svelte.dev/content/docs/kit/25-build-and-deploy/99-writing-adapters.md', + '**/apps/svelte.dev/content/docs/kit/30-advanced/70-packaging.md', + '**/apps/svelte.dev/content/docs/kit/40-best-practices/05-performance.md', + '**/apps/svelte.dev/content/docs/kit/40-best-practices/10-accessibility.md', // May the a11y gods have mercy on our souls + '**/apps/svelte.dev/content/docs/kit/60-appendix/**/*.md', + '**/xx-*.md', + ], + prompt: SVELTE_5_PROMPT, + minimize: { + removeLegacy: true, + removePlaygroundLinks: true, + removeNoteBlocks: true, + removeDetailsBlocks: true, + removeHtmlComments: true, + normalizeWhitespace: true, + }, + }, + 'svelte-complete': { + title: 'Svelte + SvelteKit (Large preset)', + description: 'Complete Svelte + SvelteKit docs excluding legacy, notes and migration docs', + glob: [ + '**/apps/svelte.dev/content/docs/svelte/**/*.md', + '**/apps/svelte.dev/content/docs/kit/**/*.md', + ], + ignore: [], + prompt: SVELTE_5_PROMPT, + minimize: { + removeLegacy: true, + removePlaygroundLinks: true, + removeNoteBlocks: true, + removeDetailsBlocks: true, + removeHtmlComments: true, + normalizeWhitespace: true, + }, + }, + 'svelte-complete-tiny': { + title: 'Svelte + SvelteKit (Tiny preset)', + description: 'Tutorial content only', + glob: [ + '**/apps/svelte.dev/content/tutorial/**/*.md', + '**/apps/svelte.dev/content/docs/svelte/02-runes/**/*.md', + ], + ignore: [], + prompt: SVELTE_5_PROMPT, + minimize: { + removeLegacy: true, + removePlaygroundLinks: true, + removeNoteBlocks: true, + removeDetailsBlocks: true, + removeHtmlComments: true, + normalizeWhitespace: true, + }, + }, + 'svelte-migration': { + title: 'Svelte + SvelteKit migration guide', + description: 'Only Svelte + SvelteKit docs for migrating ', + glob: [ + // Svelte + '**/apps/svelte.dev/content/docs/svelte/07-misc/07-v5-migration-guide.md', + // SvelteKit + '**/apps/svelte.dev/content/docs/kit/60-appendix/30-migrating-to-sveltekit-2.md', + ], + ignore: [], + prompt: SVELTE_5_PROMPT, + minimize: { + removeLegacy: true, + removePlaygroundLinks: true, + removeNoteBlocks: true, + removeDetailsBlocks: true, + removeHtmlComments: true, + normalizeWhitespace: true, + }, + }, +}; + +// eslint-disable-next-line @typescript-eslint/naming-convention +export const sveltePresets: Record = { + svelte: { + title: 'Svelte (Full)', + description: 'Complete documentation including legacy and reference', + glob: ['**/apps/svelte.dev/content/docs/svelte/**/*.md'], + ignore: [], + prompt: SVELTE_5_PROMPT, + minimize: {}, + }, + 'svelte-medium': { + title: 'Svelte (Medium)', + description: 'Complete documentation including legacy and reference', + glob: ['**/apps/svelte.dev/content/docs/svelte/**/*.md'], + ignore: [ + // Svelte ignores + '**/apps/svelte.dev/content/docs/svelte/07-misc/04-custom-elements.md', + '**/apps/svelte.dev/content/docs/svelte/07-misc/06-v4-migration-guide.md', + '**/apps/svelte.dev/content/docs/svelte/07-misc/07-v5-migration-guide.md', + '**/apps/svelte.dev/content/docs/svelte/07-misc/99-faq.md', + '**/apps/svelte.dev/content/docs/svelte/07-misc/xx-reactivity-indepth.md', + '**/apps/svelte.dev/content/docs/svelte/98-reference/21-svelte-legacy.md', + '**/apps/svelte.dev/content/docs/svelte/99-legacy/**/*.md', + '**/apps/svelte.dev/content/docs/svelte/98-reference/30-runtime-errors.md', + '**/apps/svelte.dev/content/docs/svelte/98-reference/30-runtime-warnings.md', + '**/apps/svelte.dev/content/docs/svelte/98-reference/30-compiler-errors.md', + '**/apps/svelte.dev/content/docs/svelte/98-reference/30-compiler-warnings.md', + ], + prompt: SVELTE_5_PROMPT, + minimize: { + removeLegacy: true, + removePlaygroundLinks: true, + removeNoteBlocks: true, + removeDetailsBlocks: true, + removeHtmlComments: true, + normalizeWhitespace: true, + }, + }, +}; + +// eslint-disable-next-line @typescript-eslint/naming-convention +export const svelteKitPresets: Record = { + sveltekit: { + title: 'SvelteKit (Full)', + description: 'Complete documentation including legacy and reference', + prompt: SVELTE_5_PROMPT, + glob: ['**/apps/svelte.dev/content/docs/kit/**/*.md'], + minimize: {}, + }, + 'sveltekit-medium': { + title: 'SvelteKit (Medium)', + description: 'Complete documentation including legacy and reference', + prompt: SVELTE_5_PROMPT, + glob: ['**/apps/svelte.dev/content/docs/kit/**/*.md'], + minimize: { + removeLegacy: true, + removePlaygroundLinks: true, + removeNoteBlocks: true, + removeDetailsBlocks: true, + removeHtmlComments: true, + normalizeWhitespace: true, + }, + ignore: [ + // SvelteKit ignores + '**/apps/svelte.dev/content/docs/kit/25-build-and-deploy/*adapter-*.md', + '**/apps/svelte.dev/content/docs/kit/25-build-and-deploy/99-writing-adapters.md', + '**/apps/svelte.dev/content/docs/kit/30-advanced/70-packaging.md', + '**/apps/svelte.dev/content/docs/kit/40-best-practices/05-performance.md', + '**/apps/svelte.dev/content/docs/kit/40-best-practices/10-accessibility.md', // May the a11y gods have mercy on our souls + '**/apps/svelte.dev/content/docs/kit/60-appendix/**/*.md', + '**/xx-*.md', + ], + }, +}; + +// eslint-disable-next-line @typescript-eslint/naming-convention +export const otherPresets: Record = { + 'svelte-cli': { + title: 'Svelte CLI - npx sv', + glob: ['**/apps/svelte.dev/content/docs/cli/**/*.md'], + ignore: [], + minimize: {}, + }, +}; + +export const presets = { + ...combinedPresets, + ...sveltePresets, + ...svelteKitPresets, + ...otherPresets, +}; + +// eslint-disable-next-line @typescript-eslint/naming-convention +export function transformAndSortPresets(presetsObject: Record) { + return Object.entries(presetsObject) + .map(([key, value]) => ({ + key: key.toLowerCase(), + ...value, + })) + .sort(); +} + +export const DEFAULT_REPOSITORY = { + owner: 'sveltejs', + repo: 'svelte.dev', +} as const; diff --git a/src/lib/utils/pathUtils.test.ts b/src/lib/utils/pathUtils.test.ts new file mode 100644 index 0000000..0bd71cd --- /dev/null +++ b/src/lib/utils/pathUtils.test.ts @@ -0,0 +1,335 @@ +import { describe, it, expect } from 'vitest' +import { + cleanDocumentationPath, + cleanTarballPath, + extractTitleFromPath, + removeFrontmatter +} from './pathUtils' + +describe('pathUtils', () => { + describe('cleanDocumentationPath', () => { + it('should remove apps/svelte.dev/content/ prefix', () => { + const input = 'apps/svelte.dev/content/docs/svelte/01-introduction.md' + const expected = 'docs/svelte/01-introduction.md' + expect(cleanDocumentationPath(input)).toBe(expected) + }) + + it('should handle paths without the prefix', () => { + const input = 'docs/svelte/01-introduction.md' + const expected = 'docs/svelte/01-introduction.md' + expect(cleanDocumentationPath(input)).toBe(expected) + }) + + it('should handle empty string', () => { + const input = '' + const expected = '' + expect(cleanDocumentationPath(input)).toBe(expected) + }) + + it('should handle partial prefix matches', () => { + const input = 'apps/svelte.dev/content-extra/docs/svelte/01-introduction.md' + const expected = 'apps/svelte.dev/content-extra/docs/svelte/01-introduction.md' + expect(cleanDocumentationPath(input)).toBe(expected) + }) + + it('should handle paths with similar but different prefixes', () => { + const input = 'apps/svelte.dev/contents/docs/svelte/01-introduction.md' + const expected = 'apps/svelte.dev/contents/docs/svelte/01-introduction.md' + expect(cleanDocumentationPath(input)).toBe(expected) + }) + + it('should handle SvelteKit documentation paths', () => { + const input = 'apps/svelte.dev/content/docs/kit/01-routing.md' + const expected = 'docs/kit/01-routing.md' + expect(cleanDocumentationPath(input)).toBe(expected) + }) + + it('should handle tutorial paths', () => { + const input = 'apps/svelte.dev/content/tutorial/01-introduction/01-hello-world.md' + const expected = 'tutorial/01-introduction/01-hello-world.md' + expect(cleanDocumentationPath(input)).toBe(expected) + }) + }) + + describe('cleanTarballPath', () => { + it('should remove the first segment from tarball paths', () => { + const input = 'svelte.dev-main/apps/svelte.dev/content/docs/svelte/01-introduction.md' + const expected = 'apps/svelte.dev/content/docs/svelte/01-introduction.md' + expect(cleanTarballPath(input)).toBe(expected) + }) + + it('should handle paths with different repo prefixes', () => { + const input = 'svelte-12345/apps/svelte.dev/content/docs/kit/01-routing.md' + const expected = 'apps/svelte.dev/content/docs/kit/01-routing.md' + expect(cleanTarballPath(input)).toBe(expected) + }) + + it('should handle single segment paths', () => { + const input = 'single-segment' + const expected = '' + expect(cleanTarballPath(input)).toBe(expected) + }) + + it('should handle empty string', () => { + const input = '' + const expected = '' + expect(cleanTarballPath(input)).toBe(expected) + }) + + it('should handle paths with no segments', () => { + const input = 'just-filename.md' + const expected = '' + expect(cleanTarballPath(input)).toBe(expected) + }) + + it('should handle complex nested paths', () => { + const input = 'repo-name/very/deep/nested/path/to/file.md' + const expected = 'very/deep/nested/path/to/file.md' + expect(cleanTarballPath(input)).toBe(expected) + }) + }) + + describe('extractTitleFromPath', () => { + it('should extract filename and remove .md extension and numbered prefix', () => { + const input = 'docs/svelte/01-introduction.md' + const expected = 'introduction' + expect(extractTitleFromPath(input)).toBe(expected) + }) + + it('should remove numbered prefixes', () => { + const input = 'docs/svelte/01-introduction.md' + const expected = 'introduction' + expect(extractTitleFromPath(input)).toBe(expected) + }) + + it('should handle files without numbered prefixes', () => { + const input = 'docs/svelte/reactivity.md' + const expected = 'reactivity' + expect(extractTitleFromPath(input)).toBe(expected) + }) + + it('should handle files without .md extension', () => { + const input = 'docs/svelte/01-introduction' + const expected = 'introduction' + expect(extractTitleFromPath(input)).toBe(expected) + }) + + it('should handle complex numbered prefixes', () => { + const input = 'docs/svelte/99-advanced-topics.md' + const expected = 'advanced-topics' + expect(extractTitleFromPath(input)).toBe(expected) + }) + + it('should handle files with multiple numbered prefixes', () => { + const input = 'docs/svelte/01-02-nested-numbering.md' + const expected = '02-nested-numbering' + expect(extractTitleFromPath(input)).toBe(expected) + }) + + it('should handle just a filename', () => { + const input = '01-introduction.md' + const expected = 'introduction' + expect(extractTitleFromPath(input)).toBe(expected) + }) + + it('should handle empty string', () => { + const input = '' + const expected = '' + expect(extractTitleFromPath(input)).toBe(expected) + }) + + it('should handle paths with no filename', () => { + const input = 'docs/svelte/' + const expected = '' + expect(extractTitleFromPath(input)).toBe(expected) + }) + + it('should handle files with hyphens but no numbers', () => { + const input = 'docs/svelte/state-management.md' + const expected = 'state-management' + expect(extractTitleFromPath(input)).toBe(expected) + }) + + it('should handle files with numbers in the middle', () => { + const input = 'docs/svelte/svelte5-features.md' + const expected = 'svelte5-features' + expect(extractTitleFromPath(input)).toBe(expected) + }) + + it('should handle tutorial paths', () => { + const input = 'tutorial/01-introduction/01-hello-world.md' + const expected = 'hello-world' + expect(extractTitleFromPath(input)).toBe(expected) + }) + + it('should handle SvelteKit paths', () => { + const input = 'docs/kit/01-routing.md' + const expected = 'routing' + expect(extractTitleFromPath(input)).toBe(expected) + }) + }) + + describe('removeFrontmatter', () => { + it('should remove valid frontmatter from content', () => { + const input = `--- +title: Introduction +description: Getting started guide +--- + +# Introduction + +This is the main content.` + const expected = `# Introduction + +This is the main content.` + expect(removeFrontmatter(input)).toBe(expected) + }) + + it('should handle content without frontmatter', () => { + const input = `# Introduction + +This is content without frontmatter.` + const expected = `# Introduction + +This is content without frontmatter.` + expect(removeFrontmatter(input)).toBe(expected) + }) + + it('should handle empty content', () => { + const input = '' + const expected = '' + expect(removeFrontmatter(input)).toBe(expected) + }) + + it('should handle malformed frontmatter (no closing delimiter)', () => { + const input = `--- +title: Introduction +This is malformed frontmatter without closing delimiter + +# Content here` + const expected = input // Should return original content unchanged + expect(removeFrontmatter(input)).toBe(expected) + }) + + it('should handle frontmatter with complex YAML', () => { + const input = `--- +title: Complex Example +tags: + - svelte + - tutorial +metadata: + author: John Doe + date: 2024-01-15 +--- + +# Complex Example + +Content with complex frontmatter.` + const expected = `# Complex Example + +Content with complex frontmatter.` + expect(removeFrontmatter(input)).toBe(expected) + }) + + it('should handle content that starts with --- but is not frontmatter', () => { + const input = `--- +This is not YAML frontmatter, just content that starts with ---` + const expected = input // Should return original content unchanged + expect(removeFrontmatter(input)).toBe(expected) + }) + + it('should handle frontmatter with empty lines', () => { + const input = `--- +title: Introduction + +description: A guide +--- + +# Content` + const expected = `# Content` + expect(removeFrontmatter(input)).toBe(expected) + }) + + it('should trim whitespace after removing frontmatter', () => { + const input = `--- +title: Introduction +--- + + +# Content with leading whitespace` + const expected = `# Content with leading whitespace` + expect(removeFrontmatter(input)).toBe(expected) + }) + + it('should handle frontmatter at the end of content', () => { + const input = `--- +title: Only Frontmatter +---` + const expected = `` + expect(removeFrontmatter(input)).toBe(expected) + }) + }) + + describe('integration tests', () => { + it('should work together for typical documentation workflow', () => { + // Simulate a typical path from tarball to display + const tarballPath = 'svelte.dev-main/apps/svelte.dev/content/docs/svelte/01-introduction.md' + + // Clean tarball path + const cleanedFromTarball = cleanTarballPath(tarballPath) + expect(cleanedFromTarball).toBe('apps/svelte.dev/content/docs/svelte/01-introduction.md') + + // This would be stored in DB and later cleaned for display + const cleanedForDisplay = cleanDocumentationPath(cleanedFromTarball) + expect(cleanedForDisplay).toBe('docs/svelte/01-introduction.md') + + // Extract title for metadata + const title = extractTitleFromPath(cleanedFromTarball) + expect(title).toBe('introduction') + }) + + it('should handle SvelteKit paths through full workflow', () => { + const tarballPath = 'svelte.dev-main/apps/svelte.dev/content/docs/kit/01-routing.md' + + const cleanedFromTarball = cleanTarballPath(tarballPath) + expect(cleanedFromTarball).toBe('apps/svelte.dev/content/docs/kit/01-routing.md') + + const cleanedForDisplay = cleanDocumentationPath(cleanedFromTarball) + expect(cleanedForDisplay).toBe('docs/kit/01-routing.md') + + const title = extractTitleFromPath(cleanedFromTarball) + expect(title).toBe('routing') + }) + + it('should handle tutorial paths through full workflow', () => { + const tarballPath = + 'svelte.dev-main/apps/svelte.dev/content/tutorial/01-introduction/01-hello-world.md' + + const cleanedFromTarball = cleanTarballPath(tarballPath) + expect(cleanedFromTarball).toBe( + 'apps/svelte.dev/content/tutorial/01-introduction/01-hello-world.md' + ) + + const cleanedForDisplay = cleanDocumentationPath(cleanedFromTarball) + expect(cleanedForDisplay).toBe('tutorial/01-introduction/01-hello-world.md') + + const title = extractTitleFromPath(cleanedFromTarball) + expect(title).toBe('hello-world') + }) + + it('should handle content processing with frontmatter removal', () => { + const content = `--- +title: Introduction +--- + +# Introduction + +This is the content.` + + const contentWithoutFrontmatter = removeFrontmatter(content) + expect(contentWithoutFrontmatter).toBe(`# Introduction + +This is the content.`) + }) + }) +}) diff --git a/src/lib/utils/pathUtils.ts b/src/lib/utils/pathUtils.ts new file mode 100644 index 0000000..a213103 --- /dev/null +++ b/src/lib/utils/pathUtils.ts @@ -0,0 +1,110 @@ +/** + * Unified path utilities for handling documentation paths + */ + +/** + * Clean a path by removing the "apps/svelte.dev/content/" prefix + * This is used to convert database paths to display paths + * + * @param path - The path to clean + * @returns The cleaned path + */ +export function cleanDocumentationPath(path: string): string { + const prefix = 'apps/svelte.dev/content/' + if (path.startsWith(prefix)) { + return path.substring(prefix.length) + } + return path +} + +/** + * Clean a tarball path by removing the repository directory prefix (first segment) + * This is used when processing files from GitHub tarballs + * + * @param path - The path to clean + * @returns The cleaned path without the repo directory prefix + */ +export function cleanTarballPath(path: string): string { + // Remove only the repo directory prefix (first segment) + return path.split('/').slice(1).join('/') +} + +/** + * Extract the title from a file path by removing prefixes and file extensions + * + * @param filePath - The file path to extract title from + * @returns The extracted title + */ +export function extractTitleFromPath(filePath: string): string { + if (!filePath) { + return '' + } + + const pathParts = filePath.split('/') + const filename = pathParts[pathParts.length - 1] + + // Handle empty filename (e.g., paths ending with '/') + if (!filename) { + return '' + } + + // Remove .md extension and numbered prefixes + return filename.replace('.md', '').replace(/^\d+-/, '') +} + +/** + * Remove frontmatter from markdown content using a tokenizer approach + * Frontmatter is YAML metadata at the beginning of files between --- delimiters + * + * @param content - The markdown content that may contain frontmatter + * @returns The content with frontmatter removed + */ +export function removeFrontmatter(content: string): string { + if (!content || content.length === 0) { + return content + } + + // Check if content starts with frontmatter delimiter + if (!content.startsWith('---\n')) { + return content + } + + let position = 4 // Start after the opening "---\n" + let insideFrontmatter = true + let frontmatterEndOffset: number | null = null + + // Traverse the string character by character + while (position < content.length && insideFrontmatter) { + const char = content[position] + + // Look for potential end of frontmatter: \n--- + if (char === '\n' && position + 3 < content.length) { + const nextThree = content.substring(position + 1, position + 4) + if (nextThree === '---') { + // Check what comes after the closing --- + const afterClosing = position + 4 + + if (afterClosing >= content.length) { + // End of string - this is valid frontmatter + frontmatterEndOffset = content.length + insideFrontmatter = false + } else if (content[afterClosing] === '\n') { + // Followed by newline - this is valid frontmatter + frontmatterEndOffset = afterClosing + 1 + insideFrontmatter = false + } + // If followed by something else, it's not the end delimiter, continue searching + } + } + + position++ + } + + // If we never found the end of frontmatter, it's malformed + if (frontmatterEndOffset === null) { + return content + } + + // Return content after the frontmatter, trimmed + return content.substring(frontmatterEndOffset).trim() +} diff --git a/src/lib/utils/prompts.ts b/src/lib/utils/prompts.ts new file mode 100644 index 0000000..c48e629 --- /dev/null +++ b/src/lib/utils/prompts.ts @@ -0,0 +1,165 @@ +export const SVELTE_5_PROMPT = + 'Always use Svelte 5 runes and Svelte 5 syntax. Runes do not need to be imported, they are globals. $state() runes are always declared using `let`, never with `const`. When passing a function to $derived, you must always use $derived.by(() => ...). Error boundaries can only catch errors during component rendering and at the top level of an $effect inside the error boundary. Error boundaries do not catch errors in onclick or other event handlers.'; + +export const DISTILLATION_PROMPT = ` +You are an expert in web development, specifically Svelte 5 and SvelteKit. Your task is to condense and distill the Svelte documentation into a concise format while preserving the most important information. +Shorten the text information AS MUCH AS POSSIBLE while covering key concepts. + +Focus on: +1. Code examples with short explanations of how they work +2. Key concepts and APIs with their usage patterns +3. Important gotchas and best practices +4. Patterns that developers commonly use + +Remove: +1. Redundant explanations +2. Verbose content that can be simplified +3. Marketing language +4. Legacy or deprecated content +5. Anything else that is not strictly necessary + +Keep your output in markdown format. Preserve code blocks with their language annotations. +Maintain headings but feel free to combine or restructure sections to improve clarity. + +Make sure all code examples use Svelte 5 runes syntax ($state, $derived, $effect, etc.) + +Keep the following Svelte 5 syntax rules in mind: +* There is no colon (:) in event modifiers. You MUST use "onclick" instead of "on:click". +* Runes do not need to be imported, they are globals. +* $state() runes are always declared using let, never with const. +* When passing a function to $derived, you must always use $derived.by(() => ...). +* Error boundaries can only catch errors during component rendering and at the top level of an $effect inside the error boundary. +* Error boundaries do not catch errors in onclick or other event handlers. + +IMPORTANT: All code examples MUST come from the documentation verbatim, do NOT create new code examples. Do NOT modify existing code examples. +IMPORTANT: Because of changes in Svelte 5 syntax, do not include content from your existing knowledge, you may only use knowledge from the documentation to condense. + +Here is the documentation you must condense: + +`; + +export const SVELTE_DEVELOPER_PROMPT = `You are an expert in web development, specifically Svelte 5 and SvelteKit, with expert-level knowledge of Svelte 5, SvelteKit, and TypeScript. + +## Core Expertise: + +### Svelte 5 Runes & Reactivity +- **$state**: Reactive state declaration (always use let, never const) +- **$derived**: Computed values (always use $derived.by(() => ...) for functions) +- **$effect**: Side effects and cleanup (runs after DOM updates) +- **$props**: Component props with destructuring and defaults +- **$bindable**: Two-way binding for props + +### Critical Syntax Rules: +${SVELTE_5_PROMPT} + +### Additional Rules: +- Props: let { count = 0, name } = $props() +- Bindable: let { value = $bindable() } = $props() +- Children: let { children } = $props() +- Cleanup: $effect(() => { return () => cleanup() }) +- Context: setContext/getContext work with runes +- Snippets: {#snippet name(params)} for reusable templates + +### SvelteKit Essentials: +- File-based routing with route groups and parameters +- Load functions: +page.ts (universal) vs +page.server.ts (server-only) +- Form actions in +page.server.ts with progressive enhancement +- Layout nesting and data inheritance +- Error and loading states with +error.svelte and loading UI + +### TypeScript Integration: +- Always use TypeScript for type safety +- Properly type PageData, PageLoad, Actions, RequestHandler +- Generic components with proper type inference +- .svelte.ts for shared reactive state + +## MCP Tool Usage Guide: + +### Template Prompts (Efficient Documentation Injection): +Use these for instant access to curated documentation sets: +- **svelte-core**: Core Svelte 5 (introduction, runes, template syntax, styling) +- **svelte-advanced**: Advanced Svelte 5 (special elements, runtime, misc) +- **svelte-complete**: Complete Svelte 5 documentation +- **sveltekit-core**: Core SvelteKit (getting started, core concepts) +- **sveltekit-production**: Production SvelteKit (build/deploy, advanced, best practices) +- **sveltekit-complete**: Complete SvelteKit documentation + +### Resources Access: +- **📦 Preset Resources**: Use svelte-llm://svelte-core, svelte-llm://svelte-advanced, svelte-llm://svelte-complete, svelte-llm://sveltekit-core, svelte-llm://sveltekit-production, svelte-llm://sveltekit-complete for curated documentation sets +- **📄 Individual Docs**: Use svelte-llm://doc/[path] for specific documentation files +- Access via list_resources or direct URI for browsing and reference + +### When to use list_sections + get_documentation: +- **Specific Topics**: When you need particular sections not covered by presets +- **Custom Combinations**: When presets don't match the exact scope needed +- **Deep Dives**: When you need detailed information on specific APIs +- **Troubleshooting**: When investigating specific issues or edge cases + +### Strategic Approach: +1. **Start with Template Prompts**: Use template prompts (svelte-core, sveltekit-core, etc.) for immediate context injection +2. **Browse via Resources**: Use preset resources for reading/reference during development +3. **Supplement with Specific Docs**: Use list_sections + get_documentation only when presets don't cover your needs +4. **Combine Efficiently**: Use multiple template prompts if you need both Svelte and SvelteKit context + +### Documentation Fetching Priority: +1. **Template Prompts First**: Always try relevant template prompts before individual sections +2. **Preset Resources**: Use for browsing and reference +3. **Individual Sections**: Only when specific content not in presets is needed +4. **Multiple Sources**: Combine template prompts with specific sections as needed + +## Best Practices: +- Write production-ready TypeScript code +- Include proper error handling and loading states +- Consider accessibility (ARIA, keyboard navigation) +- Optimize for performance (lazy loading, minimal reactivity) +- Use semantic HTML and proper component composition +- Implement proper cleanup in effects +- Handle edge cases and provide fallbacks`; + +// eslint-disable-next-line @typescript-eslint/naming-convention, func-style +export const createSvelteDeveloperPromptWithTask = (task?: string): string => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const basePrompt = SVELTE_DEVELOPER_PROMPT; + + if (!task) { + return ( + basePrompt + + ` + +## Your Approach: +When helping with Svelte/SvelteKit development: +1. **Use Template Prompts**: Start with relevant template prompts (svelte-core, sveltekit-core, etc.) for immediate context +2. **Supplement as Needed**: Use list_sections + get_documentation only for content not covered by templates +3. **Provide Complete Solutions**: Include working TypeScript code with proper types +4. **Explain Trade-offs**: Discuss architectural decisions and alternatives +5. **Optimize**: Suggest performance improvements and best practices` + ); + } + + return ( + basePrompt + + ` + +## Current Task: +${task} + +## Task-Specific Approach: +1. **Inject Relevant Context**: Use appropriate template prompts based on "${task.substring(0, 50)}...": + - Component tasks: Use svelte-core for runes, template syntax + - Advanced features: Use svelte-advanced for special elements, runtime + - Full applications: Use svelte-complete + sveltekit-core/complete + - Production apps: Use sveltekit-production for deployment, best practices +2. **Supplement with Specific Docs**: Use list_sections + get_documentation only if templates don't cover specific needs +3. **Design Architecture**: + - Component structure and composition + - State management approach + - TypeScript types and interfaces + - Error handling strategy +4. **Implement Solution**: + - Complete, working code + - Proper types and error boundaries + - Performance optimizations + - Accessibility considerations +5. **Explain Implementation**: Provide rationale for choices and discuss alternatives` + ); +}; From 250b1be7aa8dae9b736a2c104775c975c076be97 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Mon, 22 Sep 2025 00:05:03 +0200 Subject: [PATCH 02/14] Refactor cache service to use Drizzle ORM --- src/lib/cacheDb.ts | 117 ++++++++++++++++++++---------------- src/lib/fetchMarkdown.ts | 2 +- src/lib/presetCache.ts | 2 +- src/lib/server/db/schema.ts | 16 ++++- 4 files changed, 83 insertions(+), 54 deletions(-) diff --git a/src/lib/cacheDb.ts b/src/lib/cacheDb.ts index b659821..ff1f036 100644 --- a/src/lib/cacheDb.ts +++ b/src/lib/cacheDb.ts @@ -1,5 +1,6 @@ -import { maybeInitializePool } from '$lib/server/db'; -import type { Pool } from 'pg'; +import { db } from '$lib/server/db'; +import { cache } from '$lib/server/db/schema'; +import { and, eq, sql } from 'drizzle-orm'; export interface CacheEntry { id: number; @@ -11,29 +12,28 @@ export interface CacheEntry { } export class CacheDbService { - private db: Pool; private defaultTTL: number; - constructor(db?: Pool, defaultTTLMinutes: number = 60) { - this.db = db || maybeInitializePool(); + constructor(defaultTTLMinutes: number = 60) { this.defaultTTL = defaultTTLMinutes; } async get(key: string): Promise { - const query = ` - SELECT data, expires_at - FROM cache - WHERE cache_key = $1 AND expires_at > NOW() - `; - try { - const result = await this.db.query(query, [key]); - - if (result.rows.length === 0) { + const result = await db + .select({ data: cache.data }) + .from(cache) + .where(and( + eq(cache.cache_key, key), + sql`${cache.expires_at} > ${new Date()}` + )) + .limit(1); + + if (result.length === 0) { return null; } - return result.rows[0].data; + return result[0].data; } catch (error) { console.error('Error getting cache entry:', error); return null; @@ -42,18 +42,29 @@ export class CacheDbService { async set(key: string, data: Buffer, ttlMinutes?: number): Promise { const ttl = ttlMinutes || this.defaultTTL; - const query = ` - INSERT INTO cache (cache_key, data, size_bytes, expires_at) - VALUES ($1, $2, $3, NOW() + INTERVAL '${ttl} minutes') - ON CONFLICT (cache_key) - DO UPDATE SET - data = EXCLUDED.data, - size_bytes = EXCLUDED.size_bytes, - expires_at = EXCLUDED.expires_at - `; + const expires_at = new Date(Date.now() + ttl * 60 * 1000); + const now = new Date(); try { - await this.db.query(query, [key, data, data.length]); + await db + .insert(cache) + .values({ + cache_key: key, + data, + size_bytes: data.length, + expires_at, + created_at: now, + updated_at: now, + }) + .onConflictDoUpdate({ + target: cache.cache_key, + set: { + data, + size_bytes: data.length, + expires_at, + updated_at: now, + }, + }); } catch (error) { console.error('Error setting cache entry:', error); throw error; @@ -61,11 +72,11 @@ export class CacheDbService { } async delete(key: string): Promise { - const query = 'DELETE FROM cache WHERE cache_key = $1'; - try { - const result = await this.db.query(query, [key]); - return (result.rowCount ?? 0) > 0; + const result = await db + .delete(cache) + .where(eq(cache.cache_key, key)); + return result.rowsAffected > 0; } catch (error) { console.error('Error deleting cache entry:', error); return false; @@ -73,10 +84,8 @@ export class CacheDbService { } async clear(): Promise { - const query = 'DELETE FROM cache'; - try { - await this.db.query(query); + await db.delete(cache); } catch (error) { console.error('Error clearing cache:', error); throw error; @@ -84,11 +93,11 @@ export class CacheDbService { } async deleteExpired(): Promise { - const query = 'DELETE FROM cache WHERE expires_at <= NOW()'; - try { - const result = await this.db.query(query); - return result.rowCount ?? 0; + const result = await db + .delete(cache) + .where(sql`${cache.expires_at} <= ${new Date()}`); + return result.rowsAffected; } catch (error) { console.error('Error deleting expired cache entries:', error); return 0; @@ -96,20 +105,21 @@ export class CacheDbService { } async getStatus(): Promise<{ count: number; keys: string[]; totalSizeBytes: number }> { - const query = ` - SELECT cache_key, size_bytes - FROM cache - WHERE expires_at > NOW() - ORDER BY created_at DESC - `; - try { - const result = await this.db.query(query); - const keys = result.rows.map((row) => row.cache_key); - const totalSizeBytes = result.rows.reduce((sum, row) => sum + row.size_bytes, 0); + const result = await db + .select({ + cache_key: cache.cache_key, + size_bytes: cache.size_bytes, + }) + .from(cache) + .where(sql`${cache.expires_at} > ${new Date()}`) + .orderBy(cache.created_at); + + const keys = result.map((row) => row.cache_key); + const totalSizeBytes = result.reduce((sum, row) => sum + row.size_bytes, 0); return { - count: result.rows.length, + count: result.length, keys, totalSizeBytes, }; @@ -120,11 +130,16 @@ export class CacheDbService { } async has(key: string): Promise { - const query = 'SELECT 1 FROM cache WHERE cache_key = $1 AND expires_at > NOW()'; - try { - const result = await this.db.query(query, [key]); - return result.rows.length > 0; + const result = await db + .select({ exists: sql`1` }) + .from(cache) + .where(and( + eq(cache.cache_key, key), + sql`${cache.expires_at} > ${new Date()}` + )) + .limit(1); + return result.length > 0; } catch (error) { console.error('Error checking cache entry:', error); return false; diff --git a/src/lib/fetchMarkdown.ts b/src/lib/fetchMarkdown.ts index 2771eef..6927c81 100644 --- a/src/lib/fetchMarkdown.ts +++ b/src/lib/fetchMarkdown.ts @@ -5,7 +5,7 @@ import { Readable } from 'stream'; import { createGunzip } from 'zlib'; import { minimatch } from 'minimatch'; import { getPresetContent } from './presetCache'; -import { CacheDbService } from '$lib/server/cacheDb'; +import { CacheDbService } from '$lib/cacheDb'; import { log, logAlways, logErrorAlways } from '$lib/log'; import { cleanTarballPath } from '$lib/utils/pathUtils'; diff --git a/src/lib/presetCache.ts b/src/lib/presetCache.ts index 704b3fc..a1aace9 100644 --- a/src/lib/presetCache.ts +++ b/src/lib/presetCache.ts @@ -2,7 +2,7 @@ import { ContentSyncService } from '$lib/server/contentSync'; import { presets } from '$lib/presets'; import { log, logAlways, logErrorAlways } from '$lib/log'; import { cleanDocumentationPath } from '$lib/utils/pathUtils'; -import { CacheDbService } from '$lib/server/cacheDb'; +import { CacheDbService } from '$lib/cacheDb'; // Maximum age of cached content in milliseconds (24 hours) export const MAX_CACHE_AGE_MS = 24 * 60 * 60 * 1000; diff --git a/src/lib/server/db/schema.ts b/src/lib/server/db/schema.ts index 776205e..241d978 100644 --- a/src/lib/server/db/schema.ts +++ b/src/lib/server/db/schema.ts @@ -1,4 +1,4 @@ -import { integer, sqliteTable, text } from 'drizzle-orm/sqlite-core'; +import { blob, integer, sqliteTable, text } from 'drizzle-orm/sqlite-core'; import { float_32_array } from './utils'; /** @@ -89,3 +89,17 @@ export const content_distilled = sqliteTable('content_distilled', { .notNull() .$defaultFn(() => new Date()), }); + +export const cache = sqliteTable('cache', { + id: integer('id').primaryKey(), + cache_key: text('cache_key').notNull().unique(), + data: blob('data', { mode: 'buffer' }).notNull(), + size_bytes: integer('size_bytes').notNull(), + expires_at: integer('expires_at', { mode: 'timestamp' }).notNull(), + created_at: integer('created_at', { mode: 'timestamp' }) + .notNull() + .$defaultFn(() => new Date()), + updated_at: integer('updated_at', { mode: 'timestamp' }) + .notNull() + .$defaultFn(() => new Date()), +}); From c15503d847fed89e176fdf60de9e1a0ae4950fd1 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Mon, 22 Sep 2025 00:55:14 +0200 Subject: [PATCH 03/14] wip --- src/lib/server/contentDb.ts | 383 ++++++++++++++++++++++++++++++++++ src/lib/server/contentSync.ts | 271 ++++++++++++++++++++++++ src/lib/server/db.ts | 71 +++++++ src/lib/types/db.ts | 112 ++++++++++ 4 files changed, 837 insertions(+) create mode 100644 src/lib/server/contentDb.ts create mode 100644 src/lib/server/contentSync.ts create mode 100644 src/lib/server/db.ts create mode 100644 src/lib/types/db.ts diff --git a/src/lib/server/contentDb.ts b/src/lib/server/contentDb.ts new file mode 100644 index 0000000..02046d1 --- /dev/null +++ b/src/lib/server/contentDb.ts @@ -0,0 +1,383 @@ +import { query } from '$lib/server/db'; +import type { + DbContent, + DbContentDistilled, + CreateContentInput, + ContentFilter, + ContentStats, +} from '$lib/types/db'; +import { logAlways, logErrorAlways } from '$lib/log'; + +// Type mapping for table names to their corresponding types +type TableTypeMap = { + content: DbContent; + content_distilled: DbContentDistilled; +}; + +// Union type for valid table names +type TableName = keyof TableTypeMap; + +export class ContentDbService { + static extractFilename(path: string): string { + return path.split('/').pop() || path; + } + + static async upsertContent(input: CreateContentInput): Promise { + try { + const result = await query( + `INSERT INTO content ( + path, filename, content, size_bytes, metadata + ) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (path) DO UPDATE SET + content = EXCLUDED.content, + size_bytes = EXCLUDED.size_bytes, + metadata = EXCLUDED.metadata, + updated_at = CURRENT_TIMESTAMP + RETURNING *`, + [ + input.path, + input.filename, + input.content, + input.size_bytes, + input.metadata ? JSON.stringify(input.metadata) : '{}', + ], + ); + + logAlways(`Upserted content for ${input.path}`); + return result.rows[0] as DbContent; + } catch (error) { + logErrorAlways(`Failed to upsert content for ${input.path}:`, error); + throw new Error( + `Failed to upsert content: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + static async getContentByPath(path: string): Promise { + try { + const result = await query('SELECT * FROM content WHERE path = $1', [path]); + return result.rows.length > 0 ? (result.rows[0] as DbContent) : null; + } catch (error) { + logErrorAlways(`Failed to get content ${path}:`, error); + throw new Error( + `Failed to get content: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + static async getAllContent(): Promise { + try { + const result = await query('SELECT * FROM content ORDER BY path'); + return result.rows as DbContent[]; + } catch (error) { + logErrorAlways('Failed to get all content:', error); + throw new Error( + `Failed to get content: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + /** + * Generic search method that works with both content and content_distilled tables + */ + static async searchContent( + searchQuery: string, + tableName: T, + pathPattern: string = 'apps/svelte.dev/content/docs/%', + ): Promise { + try { + const lowerQuery = searchQuery.toLowerCase(); + + // Build table-specific WHERE clauses + let baseWhereClause = ''; + let params: (string | number)[] = []; + let paramIndex = 1; + + if (tableName === 'content') { + // For content table, include path filter + baseWhereClause = `WHERE path LIKE $${paramIndex}`; + params = [pathPattern]; + paramIndex = 2; + } else { + // For content_distilled table, no additional filters needed + baseWhereClause = ''; + paramIndex = 1; + } + + // First, try exact title match using JSON operators + const exactTitleQueryStr = ` + SELECT * FROM ${tableName} + ${baseWhereClause}${baseWhereClause ? ' AND' : 'WHERE'} LOWER(metadata->>'title') = $${paramIndex} + LIMIT 1 + `; + + const exactTitleParams = [...params, lowerQuery]; + const exactTitleResult = await query(exactTitleQueryStr, exactTitleParams); + + if (exactTitleResult.rows.length > 0) { + return exactTitleResult.rows[0] as TableTypeMap[T]; + } + + // Then try partial title match + const partialTitleQueryStr = ` + SELECT * FROM ${tableName} + ${baseWhereClause}${baseWhereClause ? ' AND' : 'WHERE'} LOWER(metadata->>'title') LIKE $${paramIndex} + LIMIT 1 + `; + + const partialTitleParams = [...params, `%${lowerQuery}%`]; + const partialTitleResult = await query(partialTitleQueryStr, partialTitleParams); + + if (partialTitleResult.rows.length > 0) { + return partialTitleResult.rows[0] as TableTypeMap[T]; + } + + // Finally try path match for backward compatibility + const pathMatchQueryStr = ` + SELECT * FROM ${tableName} + ${baseWhereClause}${baseWhereClause ? ' AND' : 'WHERE'} LOWER(path) LIKE $${paramIndex} + LIMIT 1 + `; + + const pathMatchParams = [...params, `%${lowerQuery}%`]; + const pathMatchResult = await query(pathMatchQueryStr, pathMatchParams); + + return pathMatchResult.rows.length > 0 ? (pathMatchResult.rows[0] as TableTypeMap[T]) : null; + } catch (error) { + logErrorAlways(`Failed to search ${tableName} for "${searchQuery}":`, error); + throw new Error( + `Failed to search ${tableName}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + static async searchAllContent( + searchQuery: string, + pathPattern: string = 'apps/svelte.dev/content/docs/%', + limit: number = 50, + ): Promise { + try { + const lowerQuery = searchQuery.toLowerCase(); + + // Combine all search types into one query with UNION + const combinedQueryStr = ` + -- Exact title matches first + (SELECT * FROM content + WHERE path LIKE $1 + AND LOWER(metadata->>'title') = $2 + ORDER BY path + LIMIT $3) + + UNION + + -- Then partial title matches + (SELECT * FROM content + WHERE path LIKE $1 + AND LOWER(metadata->>'title') LIKE $4 + AND LOWER(metadata->>'title') != $2 + ORDER BY path + LIMIT $3) + + UNION + + -- Finally path matches + (SELECT * FROM content + WHERE path LIKE $1 + AND LOWER(path) LIKE $4 + AND (metadata->>'title' IS NULL OR LOWER(metadata->>'title') NOT LIKE $4) + ORDER BY path + LIMIT $3) + + ORDER BY path + LIMIT $3 + `; + + const params = [pathPattern, lowerQuery, limit, `%${lowerQuery}%`]; + + const result = await query(combinedQueryStr, params); + + return result.rows as DbContent[]; + } catch (error) { + logErrorAlways(`Failed to search all content for "${searchQuery}":`, error); + throw new Error( + `Failed to search content: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + static async getDocumentationSections( + pathPattern: string = 'apps/svelte.dev/content/docs/%', + minContentLength: number = 100, + ): Promise; content: string }>> { + try { + const sectionsQueryStr = ` + SELECT path, metadata, content + FROM content + WHERE path LIKE $1 + AND LENGTH(content) >= $2 + ORDER BY path + `; + + const params = [pathPattern, minContentLength]; + + const result = await query(sectionsQueryStr, params); + + return result.rows.map((row) => ({ + path: row.path, + metadata: row.metadata, + content: row.content, + })); + } catch (error) { + logErrorAlways('Failed to get documentation sections:', error); + throw new Error( + `Failed to get sections: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + static async getFilteredContent( + pathPattern: string = 'apps/svelte.dev/content/docs/%', + minContentLength: number = 200, + ): Promise { + try { + const filterQueryStr = ` + SELECT * + FROM content + WHERE path LIKE $1 + AND LENGTH(content) >= $2 + ORDER BY path + `; + + const params = [pathPattern, minContentLength]; + + const result = await query(filterQueryStr, params); + return result.rows as DbContent[]; + } catch (error) { + logErrorAlways('Failed to get filtered content:', error); + throw new Error( + `Failed to get filtered content: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + static async getContentStats(): Promise { + try { + const totalResult = await query( + `SELECT + COUNT(*) as total_files, + COALESCE(SUM(size_bytes), 0) as total_size_bytes, + MAX(updated_at) as last_updated + FROM content`, + ); + + return { + total_files: parseInt(totalResult.rows[0].total_files), + total_size_bytes: parseInt(totalResult.rows[0].total_size_bytes), + last_updated: totalResult.rows[0].last_updated, + }; + } catch (error) { + logErrorAlways('Failed to get content stats:', error); + throw new Error( + `Failed to get stats: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + static async deleteContent(path: string): Promise { + try { + const result = await query('DELETE FROM content WHERE path = $1', [path]); + return (result.rowCount ?? 0) > 0; + } catch (error) { + logErrorAlways(`Failed to delete content ${path}:`, error); + throw new Error( + `Failed to delete content: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + static async deleteAllContent(): Promise { + try { + const result = await query('DELETE FROM content'); + return result.rowCount ?? 0; + } catch (error) { + logErrorAlways('Failed to delete all content:', error); + throw new Error( + `Failed to delete content: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + static async hasContentChanged(path: string, newContent: string): Promise { + try { + const existing = await ContentDbService.getContentByPath(path); + if (!existing) return true; + + return existing.content !== newContent; + } catch (error) { + logErrorAlways(`Failed to check content change for ${path}:`, error); + return true; // Assume changed on error + } + } + + static async batchUpsertContent(contents: CreateContentInput[]): Promise { + try { + const results: DbContent[] = []; + + // Process in chunks to avoid overwhelming the database + const chunkSize = 200; + for (let i = 0; i < contents.length; i += chunkSize) { + const chunk = contents.slice(i, i + chunkSize); + + const chunkResults = await Promise.all( + chunk.map((content) => ContentDbService.upsertContent(content)), + ); + + results.push(...chunkResults); + } + + logAlways(`Batch upserted ${results.length} content items`); + return results; + } catch (error) { + logErrorAlways('Failed to batch upsert content:', error); + throw new Error( + `Failed to batch upsert: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + static extractFrontmatter(content: string): Record { + const metadata: Record = {}; + + if (!content.startsWith('---\n')) { + return metadata; + } + + const endIndex = content.indexOf('\n---\n', 4); + if (endIndex === -1) { + return metadata; + } + + const frontmatter = content.substring(4, endIndex); + const lines = frontmatter.split('\n'); + + for (const line of lines) { + const colonIndex = line.indexOf(':'); + if (colonIndex > 0) { + const key = line.substring(0, colonIndex).trim(); + const value = line.substring(colonIndex + 1).trim(); + + // Remove quotes if present + const cleanValue = value.replace(/^["'](.*)["']$/, '$1'); + + // Try to parse as JSON for nested structures + try { + metadata[key] = JSON.parse(cleanValue); + } catch { + metadata[key] = cleanValue; + } + } + } + + return metadata; + } +} diff --git a/src/lib/server/contentSync.ts b/src/lib/server/contentSync.ts new file mode 100644 index 0000000..f796120 --- /dev/null +++ b/src/lib/server/contentSync.ts @@ -0,0 +1,271 @@ +import { + fetchRepositoryTarball, + processMarkdownFromTarball, + minimizeContent, +} from '$lib/fetchMarkdown'; +import { ContentDbService } from '$lib/server/contentDb'; +import type { CreateContentInput } from '$lib/types/db'; +import { presets, DEFAULT_REPOSITORY } from '$lib/presets'; +import { logAlways, logErrorAlways, log } from '$lib/log'; + +function sortFilesWithinGroup( + files: Array<{ path: string; content: string }>, +): Array<{ path: string; content: string }> { + return files.sort((a, b) => { + const aPath = a.path; + const bPath = b.path; + + // Check if one path is a parent of the other + if (bPath.startsWith(aPath.replace('/index.md', '/'))) return -1; + if (aPath.startsWith(bPath.replace('/index.md', '/'))) return 1; + + return aPath.localeCompare(bPath); + }); +} + +export class ContentSyncService { + static readonly MAX_CONTENT_AGE_MS = 24 * 60 * 60 * 1000; + + static async syncRepository( + options: { + returnStats?: boolean; + } = {}, + ): Promise<{ + success: boolean; + stats: { + total_files: number; + total_size_bytes: number; + last_updated: Date; + }; + sync_details: { + upserted_files: number; + deleted_files: number; + unchanged_files: number; + }; + timestamp: string; + }> { + const { returnStats = true } = options; + const { owner, repo: repoName } = DEFAULT_REPOSITORY; + + logAlways(`Starting sync for repository: ${owner}/${repoName}`); + + let upsertedFiles = 0; + let deletedFiles = 0; + let unchangedFiles = 0; + + try { + logAlways(`Step 1: Syncing repository ${owner}/${repoName}`); + + const tarballBuffer = await fetchRepositoryTarball(owner, repoName); + + const filesWithPaths = (await processMarkdownFromTarball( + tarballBuffer, + { + glob: ['**/*.md', '**/*.mdx'], + ignore: [], + title: `Sync ${owner}/${repoName}`, + distilled: false, + }, + true, + )) as Array<{ + path: string; + content: string; + }>; + + logAlways(`Found ${filesWithPaths.length} markdown files in ${owner}/${repoName}`); + + const existingFiles = await ContentDbService.getAllContent(); + const existingPaths = new Set(existingFiles.map((file) => file.path)); + + const foundPaths = new Set(filesWithPaths.map((file) => file.path)); + + const contentInputs: CreateContentInput[] = []; + + for (const file of filesWithPaths) { + const filename = ContentDbService.extractFilename(file.path); + const sizeBytes = new TextEncoder().encode(file.content).length; + + const metadata = ContentDbService.extractFrontmatter(file.content); + + const hasChanged = await ContentDbService.hasContentChanged(file.path, file.content); + + if (hasChanged) { + contentInputs.push({ + path: file.path, + filename, + content: file.content, + size_bytes: sizeBytes, + metadata, + }); + } else { + unchangedFiles++; + } + } + + if (contentInputs.length > 0) { + logAlways(`Upserting ${contentInputs.length} changed files`); + await ContentDbService.batchUpsertContent(contentInputs); + upsertedFiles = contentInputs.length; + } else { + logAlways(`No file content changes detected`); + } + + // Handle deletions - find files in DB that are no longer in the repository + const deletedPaths = Array.from(existingPaths).filter((path) => !foundPaths.has(path)); + + if (deletedPaths.length > 0) { + logAlways(`Deleting ${deletedPaths.length} files that no longer exist`); + + for (const deletedPath of deletedPaths) { + logAlways(` Deleting: ${deletedPath}`); + await ContentDbService.deleteContent(deletedPath); + } + deletedFiles = deletedPaths.length; + } else { + logAlways(`No deleted files detected`); + } + + let stats; + if (returnStats) { + logAlways(`Step 2: Collecting final statistics`); + stats = await ContentSyncService.getContentStats(); + } else { + logAlways(`Step 2: Skipping stats collection (returnStats = false)`); + // Return minimal stats structure + stats = { + total_files: 0, + total_size_bytes: 0, + last_updated: new Date(), + }; + } + + logAlways( + `Sync completed successfully: ${upsertedFiles} upserted, ${deletedFiles} deleted, ${unchangedFiles} unchanged`, + ); + + return { + success: true, + stats, + sync_details: { + upserted_files: upsertedFiles, + deleted_files: deletedFiles, + unchanged_files: unchangedFiles, + }, + timestamp: new Date().toISOString(), + }; + } catch (error) { + logErrorAlways(`Failed to sync repository ${owner}/${repoName}:`, error); + throw error; + } + } + + static async isRepositoryContentStale(): Promise { + try { + const stats = await ContentDbService.getContentStats(); + + if (stats.total_files === 0) { + return true; // No content, consider stale + } + + const lastUpdated = new Date(stats.last_updated); + const contentAge = Date.now() - lastUpdated.getTime(); + + const isStale = contentAge > ContentSyncService.MAX_CONTENT_AGE_MS; + + if (isStale) { + logAlways( + `Repository content is stale (age: ${Math.floor(contentAge / 1000 / 60)} minutes)`, + ); + } + + return isStale; + } catch (error) { + logErrorAlways(`Error checking repository staleness:`, error); + return true; // On error, assume stale + } + } + + static async getPresetContentFromDb( + presetKey: string, + ): Promise | null> { + const preset = presets[presetKey]; + if (!preset) { + return null; + } + + try { + const allContent = await ContentDbService.getAllContent(); + + if (allContent.length === 0) { + return null; + } + + log(`Checking ${allContent.length} files against glob patterns for preset ${presetKey}`); + log(`Glob patterns: ${JSON.stringify(preset.glob)}`); + log(`Ignore patterns: ${JSON.stringify(preset.ignore || [])}`); + + const { minimatch } = await import('minimatch'); + + const orderedResults: Array<{ path: string; content: string }> = []; + + // Process one glob pattern at a time + for (const pattern of preset.glob) { + log(`\nProcessing glob pattern: ${pattern}`); + + const matchingFiles: Array<{ path: string; content: string }> = []; + + for (const dbContent of allContent) { + const shouldIgnore = preset.ignore?.some((ignorePattern) => { + const matches = minimatch(dbContent.path, ignorePattern); + if (matches) { + log(` File ${dbContent.path} ignored by pattern: ${ignorePattern}`); + } + return matches; + }); + if (shouldIgnore) continue; + + if (minimatch(dbContent.path, pattern)) { + log(` File ${dbContent.path} matched`); + + let processedContent = dbContent.content; + if (preset.minimize && Object.keys(preset.minimize).length > 0) { + processedContent = minimizeContent(dbContent.content, preset.minimize); + } + + matchingFiles.push({ + path: dbContent.path, + content: processedContent, + }); + } + } + + const sortedFiles = sortFilesWithinGroup(matchingFiles); + + log(` Found ${sortedFiles.length} files for pattern: ${pattern}`); + sortedFiles.forEach((file, i) => { + log(` ${i + 1}. ${file.path}`); + }); + + orderedResults.push(...sortedFiles); + } + + logAlways( + `Found ${orderedResults.length} files matching preset ${presetKey} from database in natural glob order`, + ); + + log('\nFinal file order:'); + orderedResults.forEach((file, i) => { + log(` ${i + 1}. ${file.path}`); + }); + + return orderedResults; + } catch (error) { + logErrorAlways(`Failed to get preset content from database for ${presetKey}:`, error); + return null; + } + } + + static async getContentStats() { + return ContentDbService.getContentStats(); + } +} diff --git a/src/lib/server/db.ts b/src/lib/server/db.ts new file mode 100644 index 0000000..f6febbf --- /dev/null +++ b/src/lib/server/db.ts @@ -0,0 +1,71 @@ +import PG from 'pg'; +import type { Pool } from 'pg'; +import type { QueryResult } from 'pg'; +import type { QueryConfig } from '$lib/types/db'; + +import { env } from '$env/dynamic/private'; +import { logAlways, log, logErrorAlways } from '$lib/log'; + +let pool: Pool | null = null; + +export function maybeInitializePool(): Pool { + if (!pool) { + logAlways('🐘 Initializing Postgres connection!'); + pool = new PG.Pool({ + connectionString: env.DB_URL || 'postgres://admin:admin@localhost:5432/db', + max: parseInt(env.DB_CLIENTS || '10'), + }); + } + return pool; +} + +export async function query( + incomingQuery: string, + params: unknown[] = [], + config: QueryConfig = {}, +): Promise { + const pool = maybeInitializePool(); + + if (!pool) { + throw new Error('Database connection pool is not initialized'); + } + + const timingStart = new Date(); + + if (config.debug === true || env?.DB_DEBUG === 'true') { + log('----'); + log(`🔰 Query: ${incomingQuery}`); + log('📊 Data: ', params); + } + + try { + const results = await pool.query(incomingQuery, params); + + if (config.debug === true || env?.DB_DEBUG === 'true') { + log('⏰ Postgres query execution time: %dms', new Date().getTime() - timingStart.getTime()); + log('----'); + } + + return results; + } catch (error) { + logErrorAlways('Database query error:', { + query: incomingQuery, + params, + error: error instanceof Error ? error.message : String(error), + }); + + // Re-throw the error to let it bubble up + throw error; + } +} + +export async function disconnect(): Promise { + if (pool !== null) { + logAlways('😵 Disconnecting from Postgres!'); + const thisPool = pool; + pool = null; + return await thisPool.end(); + } + + return; +} diff --git a/src/lib/types/db.ts b/src/lib/types/db.ts new file mode 100644 index 0000000..4efa4f3 --- /dev/null +++ b/src/lib/types/db.ts @@ -0,0 +1,112 @@ +export interface QueryConfig { + debug?: boolean; +} + +// Enum for distillable preset names +export enum DistillablePreset { + SVELTE_DISTILLED = 'svelte-distilled', + SVELTEKIT_DISTILLED = 'sveltekit-distilled', + SVELTE_COMPLETE_DISTILLED = 'svelte-complete-distilled', +} + +// Database table types + +export interface DbDistillation { + id: number; + preset_name: DistillablePreset; + version: string; // 'latest' or '2024-01-15' + content: string; + size_kb: number; + document_count: number; + distillation_job_id: number | null; + created_at: Date; +} + +export interface DbDistillationJob { + id: number; + preset_name: string; + batch_id: string | null; + status: 'pending' | 'processing' | 'completed' | 'failed'; + model_used: string; + total_files: number; + processed_files: number; + successful_files: number; + minimize_applied: boolean; + total_input_tokens: number; + total_output_tokens: number; + started_at: Date | null; + completed_at: Date | null; + error_message: string | null; + metadata: Record; // JSONB + created_at: Date; + updated_at: Date; +} + +export interface DbContent { + id: number; + path: string; + filename: string; + content: string; + size_bytes: number; + metadata: Record; + created_at: Date; + updated_at: Date; +} + +export interface DbContentDistilled { + id: number; + path: string; + filename: string; + content: string; + size_bytes: number; + metadata: Record; + created_at: Date; + updated_at: Date; +} + +// Input types for creating/updating records + +export interface CreateDistillationInput { + preset_name: DistillablePreset; + version: string; + content: string; + size_kb: number; + document_count: number; + distillation_job_id?: number; +} + +export interface CreateDistillationJobInput { + preset_name: string; + batch_id?: string; + status: 'pending' | 'processing' | 'completed' | 'failed'; + model_used: string; + total_files: number; + minimize_applied?: boolean; + metadata?: Record; +} + +export interface CreateContentInput { + path: string; + filename: string; + content: string; + size_bytes: number; + metadata?: Record; +} + +export interface CreateContentDistilledInput { + path: string; + filename: string; + content: string; + size_bytes: number; + metadata?: Record; +} + +export interface ContentFilter { + path_pattern?: string; // For glob pattern matching +} + +export interface ContentStats { + total_files: number; + total_size_bytes: number; + last_updated: Date; +} From 65ecfa58f8576c3e306edac34611fd66cb3e4dbd Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Mon, 22 Sep 2025 00:59:43 +0200 Subject: [PATCH 04/14] Create +server.ts --- src/routes/test/+server.ts | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 src/routes/test/+server.ts diff --git a/src/routes/test/+server.ts b/src/routes/test/+server.ts new file mode 100644 index 0000000..440c5c3 --- /dev/null +++ b/src/routes/test/+server.ts @@ -0,0 +1,8 @@ +import { json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { fetchRepositoryTarball } from '$lib/fetchMarkdown'; + +export const GET: RequestHandler = async () => { + const tarball_buffer = await fetchRepositoryTarball('sveltejs', 'svelte.dev'); + return json({ data: tarballBuffer }); +}; From f355e6af78589b16b4dd3f78f9d6e283b316ed79 Mon Sep 17 00:00:00 2001 From: paoloricciuti Date: Tue, 23 Sep 2025 23:31:21 +0200 Subject: [PATCH 05/14] fix: move everything where it should be --- apps/mcp-remote/package.json | 5 +- .../mcp-remote/src/lib}/cacheDb.ts | 18 +- .../mcp-remote/src/lib}/fetchMarkdown.ts | 0 .../src => apps/mcp-remote/src/lib}/log.ts | 0 .../mcp-remote/src/lib}/presetCache.ts | 0 .../mcp-remote/src/lib}/presets.ts | 0 .../mcp-remote/src/lib}/server/contentDb.ts | 0 .../mcp-remote/src/lib}/server/contentSync.ts | 0 .../mcp-remote/src/lib}/server/db.ts | 0 .../mcp-remote/src/lib}/types/db.ts | 0 .../src/lib/utils/pathUtils.test.ts | 335 ++++++++++++++++++ .../mcp-remote/src/lib}/utils/pathUtils.ts | 0 .../mcp-remote/src/lib}/utils/prompts.ts | 0 .../mcp-remote/src}/routes/test/+server.ts | 2 +- package.json | 7 - packages/mcp-schema/src/schema.js | 5 - .../mcp-server/src/utils/pathUtils.test.ts | 335 ------------------ pnpm-lock.yaml | 106 ++++-- 18 files changed, 413 insertions(+), 400 deletions(-) rename {packages/mcp-server/src => apps/mcp-remote/src/lib}/cacheDb.ts (88%) rename {packages/mcp-server/src => apps/mcp-remote/src/lib}/fetchMarkdown.ts (100%) rename {packages/mcp-server/src => apps/mcp-remote/src/lib}/log.ts (100%) rename {packages/mcp-server/src => apps/mcp-remote/src/lib}/presetCache.ts (100%) rename {packages/mcp-server/src => apps/mcp-remote/src/lib}/presets.ts (100%) rename {packages/mcp-server/src => apps/mcp-remote/src/lib}/server/contentDb.ts (100%) rename {packages/mcp-server/src => apps/mcp-remote/src/lib}/server/contentSync.ts (100%) rename {packages/mcp-server/src => apps/mcp-remote/src/lib}/server/db.ts (100%) rename {packages/mcp-server/src => apps/mcp-remote/src/lib}/types/db.ts (100%) create mode 100644 apps/mcp-remote/src/lib/utils/pathUtils.test.ts rename {packages/mcp-server/src => apps/mcp-remote/src/lib}/utils/pathUtils.ts (100%) rename {packages/mcp-server/src => apps/mcp-remote/src/lib}/utils/prompts.ts (100%) rename {src => apps/mcp-remote/src}/routes/test/+server.ts (87%) delete mode 100644 packages/mcp-server/src/utils/pathUtils.test.ts diff --git a/apps/mcp-remote/package.json b/apps/mcp-remote/package.json index 4cb1f11..77ff726 100644 --- a/apps/mcp-remote/package.json +++ b/apps/mcp-remote/package.json @@ -64,6 +64,9 @@ "dependencies": { "@sveltejs/mcp-schema": "workspace:^", "@sveltejs/mcp-server": "workspace:^", - "@tmcp/transport-http": "^0.6.2" + "@tmcp/transport-http": "^0.6.2", + "@types/tar-stream": "^3.1.4", + "minimatch": "^10.0.3", + "tar-stream": "^3.1.7" } } diff --git a/packages/mcp-server/src/cacheDb.ts b/apps/mcp-remote/src/lib/cacheDb.ts similarity index 88% rename from packages/mcp-server/src/cacheDb.ts rename to apps/mcp-remote/src/lib/cacheDb.ts index ff1f036..570da72 100644 --- a/packages/mcp-server/src/cacheDb.ts +++ b/apps/mcp-remote/src/lib/cacheDb.ts @@ -23,10 +23,7 @@ export class CacheDbService { const result = await db .select({ data: cache.data }) .from(cache) - .where(and( - eq(cache.cache_key, key), - sql`${cache.expires_at} > ${new Date()}` - )) + .where(and(eq(cache.cache_key, key), sql`${cache.expires_at} > ${new Date()}`)) .limit(1); if (result.length === 0) { @@ -73,9 +70,7 @@ export class CacheDbService { async delete(key: string): Promise { try { - const result = await db - .delete(cache) - .where(eq(cache.cache_key, key)); + const result = await db.delete(cache).where(eq(cache.cache_key, key)); return result.rowsAffected > 0; } catch (error) { console.error('Error deleting cache entry:', error); @@ -94,9 +89,7 @@ export class CacheDbService { async deleteExpired(): Promise { try { - const result = await db - .delete(cache) - .where(sql`${cache.expires_at} <= ${new Date()}`); + const result = await db.delete(cache).where(sql`${cache.expires_at} <= ${new Date()}`); return result.rowsAffected; } catch (error) { console.error('Error deleting expired cache entries:', error); @@ -134,10 +127,7 @@ export class CacheDbService { const result = await db .select({ exists: sql`1` }) .from(cache) - .where(and( - eq(cache.cache_key, key), - sql`${cache.expires_at} > ${new Date()}` - )) + .where(and(eq(cache.cache_key, key), sql`${cache.expires_at} > ${new Date()}`)) .limit(1); return result.length > 0; } catch (error) { diff --git a/packages/mcp-server/src/fetchMarkdown.ts b/apps/mcp-remote/src/lib/fetchMarkdown.ts similarity index 100% rename from packages/mcp-server/src/fetchMarkdown.ts rename to apps/mcp-remote/src/lib/fetchMarkdown.ts diff --git a/packages/mcp-server/src/log.ts b/apps/mcp-remote/src/lib/log.ts similarity index 100% rename from packages/mcp-server/src/log.ts rename to apps/mcp-remote/src/lib/log.ts diff --git a/packages/mcp-server/src/presetCache.ts b/apps/mcp-remote/src/lib/presetCache.ts similarity index 100% rename from packages/mcp-server/src/presetCache.ts rename to apps/mcp-remote/src/lib/presetCache.ts diff --git a/packages/mcp-server/src/presets.ts b/apps/mcp-remote/src/lib/presets.ts similarity index 100% rename from packages/mcp-server/src/presets.ts rename to apps/mcp-remote/src/lib/presets.ts diff --git a/packages/mcp-server/src/server/contentDb.ts b/apps/mcp-remote/src/lib/server/contentDb.ts similarity index 100% rename from packages/mcp-server/src/server/contentDb.ts rename to apps/mcp-remote/src/lib/server/contentDb.ts diff --git a/packages/mcp-server/src/server/contentSync.ts b/apps/mcp-remote/src/lib/server/contentSync.ts similarity index 100% rename from packages/mcp-server/src/server/contentSync.ts rename to apps/mcp-remote/src/lib/server/contentSync.ts diff --git a/packages/mcp-server/src/server/db.ts b/apps/mcp-remote/src/lib/server/db.ts similarity index 100% rename from packages/mcp-server/src/server/db.ts rename to apps/mcp-remote/src/lib/server/db.ts diff --git a/packages/mcp-server/src/types/db.ts b/apps/mcp-remote/src/lib/types/db.ts similarity index 100% rename from packages/mcp-server/src/types/db.ts rename to apps/mcp-remote/src/lib/types/db.ts diff --git a/apps/mcp-remote/src/lib/utils/pathUtils.test.ts b/apps/mcp-remote/src/lib/utils/pathUtils.test.ts new file mode 100644 index 0000000..b187296 --- /dev/null +++ b/apps/mcp-remote/src/lib/utils/pathUtils.test.ts @@ -0,0 +1,335 @@ +import { describe, it, expect } from 'vitest'; +import { + cleanDocumentationPath, + cleanTarballPath, + extractTitleFromPath, + removeFrontmatter, +} from './pathUtils.js'; + +describe('pathUtils', () => { + describe('cleanDocumentationPath', () => { + it('should remove apps/svelte.dev/content/ prefix', () => { + const input = 'apps/svelte.dev/content/docs/svelte/01-introduction.md'; + const expected = 'docs/svelte/01-introduction.md'; + expect(cleanDocumentationPath(input)).toBe(expected); + }); + + it('should handle paths without the prefix', () => { + const input = 'docs/svelte/01-introduction.md'; + const expected = 'docs/svelte/01-introduction.md'; + expect(cleanDocumentationPath(input)).toBe(expected); + }); + + it('should handle empty string', () => { + const input = ''; + const expected = ''; + expect(cleanDocumentationPath(input)).toBe(expected); + }); + + it('should handle partial prefix matches', () => { + const input = 'apps/svelte.dev/content-extra/docs/svelte/01-introduction.md'; + const expected = 'apps/svelte.dev/content-extra/docs/svelte/01-introduction.md'; + expect(cleanDocumentationPath(input)).toBe(expected); + }); + + it('should handle paths with similar but different prefixes', () => { + const input = 'apps/svelte.dev/contents/docs/svelte/01-introduction.md'; + const expected = 'apps/svelte.dev/contents/docs/svelte/01-introduction.md'; + expect(cleanDocumentationPath(input)).toBe(expected); + }); + + it('should handle SvelteKit documentation paths', () => { + const input = 'apps/svelte.dev/content/docs/kit/01-routing.md'; + const expected = 'docs/kit/01-routing.md'; + expect(cleanDocumentationPath(input)).toBe(expected); + }); + + it('should handle tutorial paths', () => { + const input = 'apps/svelte.dev/content/tutorial/01-introduction/01-hello-world.md'; + const expected = 'tutorial/01-introduction/01-hello-world.md'; + expect(cleanDocumentationPath(input)).toBe(expected); + }); + }); + + describe('cleanTarballPath', () => { + it('should remove the first segment from tarball paths', () => { + const input = 'svelte.dev-main/apps/svelte.dev/content/docs/svelte/01-introduction.md'; + const expected = 'apps/svelte.dev/content/docs/svelte/01-introduction.md'; + expect(cleanTarballPath(input)).toBe(expected); + }); + + it('should handle paths with different repo prefixes', () => { + const input = 'svelte-12345/apps/svelte.dev/content/docs/kit/01-routing.md'; + const expected = 'apps/svelte.dev/content/docs/kit/01-routing.md'; + expect(cleanTarballPath(input)).toBe(expected); + }); + + it('should handle single segment paths', () => { + const input = 'single-segment'; + const expected = ''; + expect(cleanTarballPath(input)).toBe(expected); + }); + + it('should handle empty string', () => { + const input = ''; + const expected = ''; + expect(cleanTarballPath(input)).toBe(expected); + }); + + it('should handle paths with no segments', () => { + const input = 'just-filename.md'; + const expected = ''; + expect(cleanTarballPath(input)).toBe(expected); + }); + + it('should handle complex nested paths', () => { + const input = 'repo-name/very/deep/nested/path/to/file.md'; + const expected = 'very/deep/nested/path/to/file.md'; + expect(cleanTarballPath(input)).toBe(expected); + }); + }); + + describe('extractTitleFromPath', () => { + it('should extract filename and remove .md extension and numbered prefix', () => { + const input = 'docs/svelte/01-introduction.md'; + const expected = 'introduction'; + expect(extractTitleFromPath(input)).toBe(expected); + }); + + it('should remove numbered prefixes', () => { + const input = 'docs/svelte/01-introduction.md'; + const expected = 'introduction'; + expect(extractTitleFromPath(input)).toBe(expected); + }); + + it('should handle files without numbered prefixes', () => { + const input = 'docs/svelte/reactivity.md'; + const expected = 'reactivity'; + expect(extractTitleFromPath(input)).toBe(expected); + }); + + it('should handle files without .md extension', () => { + const input = 'docs/svelte/01-introduction'; + const expected = 'introduction'; + expect(extractTitleFromPath(input)).toBe(expected); + }); + + it('should handle complex numbered prefixes', () => { + const input = 'docs/svelte/99-advanced-topics.md'; + const expected = 'advanced-topics'; + expect(extractTitleFromPath(input)).toBe(expected); + }); + + it('should handle files with multiple numbered prefixes', () => { + const input = 'docs/svelte/01-02-nested-numbering.md'; + const expected = '02-nested-numbering'; + expect(extractTitleFromPath(input)).toBe(expected); + }); + + it('should handle just a filename', () => { + const input = '01-introduction.md'; + const expected = 'introduction'; + expect(extractTitleFromPath(input)).toBe(expected); + }); + + it('should handle empty string', () => { + const input = ''; + const expected = ''; + expect(extractTitleFromPath(input)).toBe(expected); + }); + + it('should handle paths with no filename', () => { + const input = 'docs/svelte/'; + const expected = ''; + expect(extractTitleFromPath(input)).toBe(expected); + }); + + it('should handle files with hyphens but no numbers', () => { + const input = 'docs/svelte/state-management.md'; + const expected = 'state-management'; + expect(extractTitleFromPath(input)).toBe(expected); + }); + + it('should handle files with numbers in the middle', () => { + const input = 'docs/svelte/svelte5-features.md'; + const expected = 'svelte5-features'; + expect(extractTitleFromPath(input)).toBe(expected); + }); + + it('should handle tutorial paths', () => { + const input = 'tutorial/01-introduction/01-hello-world.md'; + const expected = 'hello-world'; + expect(extractTitleFromPath(input)).toBe(expected); + }); + + it('should handle SvelteKit paths', () => { + const input = 'docs/kit/01-routing.md'; + const expected = 'routing'; + expect(extractTitleFromPath(input)).toBe(expected); + }); + }); + + describe('removeFrontmatter', () => { + it('should remove valid frontmatter from content', () => { + const input = `--- +title: Introduction +description: Getting started guide +--- + +# Introduction + +This is the main content.`; + const expected = `# Introduction + +This is the main content.`; + expect(removeFrontmatter(input)).toBe(expected); + }); + + it('should handle content without frontmatter', () => { + const input = `# Introduction + +This is content without frontmatter.`; + const expected = `# Introduction + +This is content without frontmatter.`; + expect(removeFrontmatter(input)).toBe(expected); + }); + + it('should handle empty content', () => { + const input = ''; + const expected = ''; + expect(removeFrontmatter(input)).toBe(expected); + }); + + it('should handle malformed frontmatter (no closing delimiter)', () => { + const input = `--- +title: Introduction +This is malformed frontmatter without closing delimiter + +# Content here`; + const expected = input; // Should return original content unchanged + expect(removeFrontmatter(input)).toBe(expected); + }); + + it('should handle frontmatter with complex YAML', () => { + const input = `--- +title: Complex Example +tags: + - svelte + - tutorial +metadata: + author: John Doe + date: 2024-01-15 +--- + +# Complex Example + +Content with complex frontmatter.`; + const expected = `# Complex Example + +Content with complex frontmatter.`; + expect(removeFrontmatter(input)).toBe(expected); + }); + + it('should handle content that starts with --- but is not frontmatter', () => { + const input = `--- +This is not YAML frontmatter, just content that starts with ---`; + const expected = input; // Should return original content unchanged + expect(removeFrontmatter(input)).toBe(expected); + }); + + it('should handle frontmatter with empty lines', () => { + const input = `--- +title: Introduction + +description: A guide +--- + +# Content`; + const expected = `# Content`; + expect(removeFrontmatter(input)).toBe(expected); + }); + + it('should trim whitespace after removing frontmatter', () => { + const input = `--- +title: Introduction +--- + + +# Content with leading whitespace`; + const expected = `# Content with leading whitespace`; + expect(removeFrontmatter(input)).toBe(expected); + }); + + it('should handle frontmatter at the end of content', () => { + const input = `--- +title: Only Frontmatter +---`; + const expected = ``; + expect(removeFrontmatter(input)).toBe(expected); + }); + }); + + describe('integration tests', () => { + it('should work together for typical documentation workflow', () => { + // Simulate a typical path from tarball to display + const tarball_path = 'svelte.dev-main/apps/svelte.dev/content/docs/svelte/01-introduction.md'; + + // Clean tarball path + const cleaned_from_tarball = cleanTarballPath(tarball_path); + expect(cleaned_from_tarball).toBe('apps/svelte.dev/content/docs/svelte/01-introduction.md'); + + // This would be stored in DB and later cleaned for display + const cleaned_for_display = cleanDocumentationPath(cleaned_from_tarball); + expect(cleaned_for_display).toBe('docs/svelte/01-introduction.md'); + + // Extract title for metadata + const title = extractTitleFromPath(cleaned_from_tarball); + expect(title).toBe('introduction'); + }); + + it('should handle SvelteKit paths through full workflow', () => { + const tarball_path = 'svelte.dev-main/apps/svelte.dev/content/docs/kit/01-routing.md'; + + const cleaned_from_tarball = cleanTarballPath(tarball_path); + expect(cleaned_from_tarball).toBe('apps/svelte.dev/content/docs/kit/01-routing.md'); + + const cleaned_for_display = cleanDocumentationPath(cleaned_from_tarball); + expect(cleaned_for_display).toBe('docs/kit/01-routing.md'); + + const title = extractTitleFromPath(cleaned_from_tarball); + expect(title).toBe('routing'); + }); + + it('should handle tutorial paths through full workflow', () => { + const tarball_path = + 'svelte.dev-main/apps/svelte.dev/content/tutorial/01-introduction/01-hello-world.md'; + + const cleaned_from_tarball = cleanTarballPath(tarball_path); + expect(cleaned_from_tarball).toBe( + 'apps/svelte.dev/content/tutorial/01-introduction/01-hello-world.md', + ); + + const cleaned_for_display = cleanDocumentationPath(cleaned_from_tarball); + expect(cleaned_for_display).toBe('tutorial/01-introduction/01-hello-world.md'); + + const title = extractTitleFromPath(cleaned_from_tarball); + expect(title).toBe('hello-world'); + }); + + it('should handle content processing with frontmatter removal', () => { + const content = `--- +title: Introduction +--- + +# Introduction + +This is the content.`; + + const content_without_frontmatter = removeFrontmatter(content); + expect(content_without_frontmatter).toBe(`# Introduction + +This is the content.`); + }); + }); +}); diff --git a/packages/mcp-server/src/utils/pathUtils.ts b/apps/mcp-remote/src/lib/utils/pathUtils.ts similarity index 100% rename from packages/mcp-server/src/utils/pathUtils.ts rename to apps/mcp-remote/src/lib/utils/pathUtils.ts diff --git a/packages/mcp-server/src/utils/prompts.ts b/apps/mcp-remote/src/lib/utils/prompts.ts similarity index 100% rename from packages/mcp-server/src/utils/prompts.ts rename to apps/mcp-remote/src/lib/utils/prompts.ts diff --git a/src/routes/test/+server.ts b/apps/mcp-remote/src/routes/test/+server.ts similarity index 87% rename from src/routes/test/+server.ts rename to apps/mcp-remote/src/routes/test/+server.ts index 440c5c3..8bd5cec 100644 --- a/src/routes/test/+server.ts +++ b/apps/mcp-remote/src/routes/test/+server.ts @@ -4,5 +4,5 @@ import { fetchRepositoryTarball } from '$lib/fetchMarkdown'; export const GET: RequestHandler = async () => { const tarball_buffer = await fetchRepositoryTarball('sveltejs', 'svelte.dev'); - return json({ data: tarballBuffer }); + return json({ data: tarball_buffer }); }; diff --git a/package.json b/package.json index cfdd4cb..3d38bb0 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,6 @@ "@eslint/compat": "^1.3.2", "@eslint/js": "^9.36.0", "@modelcontextprotocol/inspector": "^0.16.7", -<<<<<<< HEAD "@sveltejs/adapter-vercel": "^5.6.3", "@sveltejs/kit": "^2.22.0", "@sveltejs/vite-plugin-svelte": "^6.0.0", @@ -37,9 +36,6 @@ "dotenv": "^17.2.2", "drizzle-kit": "^0.30.2", "drizzle-orm": "^0.40.0", -======= - "eslint": "^9.36.0", ->>>>>>> main "eslint-config-prettier": "^10.0.1", "eslint-plugin-import": "^2.32.0", "eslint-plugin-svelte": "^3.12.3", @@ -47,12 +43,9 @@ "minimatch": "^10.0.3", "prettier": "^3.4.2", "prettier-plugin-svelte": "^3.3.3", -<<<<<<< HEAD "svelte": "^5.0.0", "svelte-check": "^4.0.0", "tar-stream": "^3.1.7", -======= ->>>>>>> main "typescript": "^5.0.0", "typescript-eslint": "^8.44.1", "vitest": "^3.2.3" diff --git a/packages/mcp-schema/src/schema.js b/packages/mcp-schema/src/schema.js index 905d21a..e98feee 100644 --- a/packages/mcp-schema/src/schema.js +++ b/packages/mcp-schema/src/schema.js @@ -1,10 +1,5 @@ -<<<<<<< HEAD:src/lib/server/db/schema.ts import { blob, integer, sqliteTable, text } from 'drizzle-orm/sqlite-core'; -import { float_32_array } from './utils'; -======= -import { integer, sqliteTable, text } from 'drizzle-orm/sqlite-core'; import { float_32_array } from './utils.js'; ->>>>>>> main:packages/mcp-schema/src/schema.js /** * NOTE: if you modify a schema adding a vector column you need to manually add this diff --git a/packages/mcp-server/src/utils/pathUtils.test.ts b/packages/mcp-server/src/utils/pathUtils.test.ts deleted file mode 100644 index 0bd71cd..0000000 --- a/packages/mcp-server/src/utils/pathUtils.test.ts +++ /dev/null @@ -1,335 +0,0 @@ -import { describe, it, expect } from 'vitest' -import { - cleanDocumentationPath, - cleanTarballPath, - extractTitleFromPath, - removeFrontmatter -} from './pathUtils' - -describe('pathUtils', () => { - describe('cleanDocumentationPath', () => { - it('should remove apps/svelte.dev/content/ prefix', () => { - const input = 'apps/svelte.dev/content/docs/svelte/01-introduction.md' - const expected = 'docs/svelte/01-introduction.md' - expect(cleanDocumentationPath(input)).toBe(expected) - }) - - it('should handle paths without the prefix', () => { - const input = 'docs/svelte/01-introduction.md' - const expected = 'docs/svelte/01-introduction.md' - expect(cleanDocumentationPath(input)).toBe(expected) - }) - - it('should handle empty string', () => { - const input = '' - const expected = '' - expect(cleanDocumentationPath(input)).toBe(expected) - }) - - it('should handle partial prefix matches', () => { - const input = 'apps/svelte.dev/content-extra/docs/svelte/01-introduction.md' - const expected = 'apps/svelte.dev/content-extra/docs/svelte/01-introduction.md' - expect(cleanDocumentationPath(input)).toBe(expected) - }) - - it('should handle paths with similar but different prefixes', () => { - const input = 'apps/svelte.dev/contents/docs/svelte/01-introduction.md' - const expected = 'apps/svelte.dev/contents/docs/svelte/01-introduction.md' - expect(cleanDocumentationPath(input)).toBe(expected) - }) - - it('should handle SvelteKit documentation paths', () => { - const input = 'apps/svelte.dev/content/docs/kit/01-routing.md' - const expected = 'docs/kit/01-routing.md' - expect(cleanDocumentationPath(input)).toBe(expected) - }) - - it('should handle tutorial paths', () => { - const input = 'apps/svelte.dev/content/tutorial/01-introduction/01-hello-world.md' - const expected = 'tutorial/01-introduction/01-hello-world.md' - expect(cleanDocumentationPath(input)).toBe(expected) - }) - }) - - describe('cleanTarballPath', () => { - it('should remove the first segment from tarball paths', () => { - const input = 'svelte.dev-main/apps/svelte.dev/content/docs/svelte/01-introduction.md' - const expected = 'apps/svelte.dev/content/docs/svelte/01-introduction.md' - expect(cleanTarballPath(input)).toBe(expected) - }) - - it('should handle paths with different repo prefixes', () => { - const input = 'svelte-12345/apps/svelte.dev/content/docs/kit/01-routing.md' - const expected = 'apps/svelte.dev/content/docs/kit/01-routing.md' - expect(cleanTarballPath(input)).toBe(expected) - }) - - it('should handle single segment paths', () => { - const input = 'single-segment' - const expected = '' - expect(cleanTarballPath(input)).toBe(expected) - }) - - it('should handle empty string', () => { - const input = '' - const expected = '' - expect(cleanTarballPath(input)).toBe(expected) - }) - - it('should handle paths with no segments', () => { - const input = 'just-filename.md' - const expected = '' - expect(cleanTarballPath(input)).toBe(expected) - }) - - it('should handle complex nested paths', () => { - const input = 'repo-name/very/deep/nested/path/to/file.md' - const expected = 'very/deep/nested/path/to/file.md' - expect(cleanTarballPath(input)).toBe(expected) - }) - }) - - describe('extractTitleFromPath', () => { - it('should extract filename and remove .md extension and numbered prefix', () => { - const input = 'docs/svelte/01-introduction.md' - const expected = 'introduction' - expect(extractTitleFromPath(input)).toBe(expected) - }) - - it('should remove numbered prefixes', () => { - const input = 'docs/svelte/01-introduction.md' - const expected = 'introduction' - expect(extractTitleFromPath(input)).toBe(expected) - }) - - it('should handle files without numbered prefixes', () => { - const input = 'docs/svelte/reactivity.md' - const expected = 'reactivity' - expect(extractTitleFromPath(input)).toBe(expected) - }) - - it('should handle files without .md extension', () => { - const input = 'docs/svelte/01-introduction' - const expected = 'introduction' - expect(extractTitleFromPath(input)).toBe(expected) - }) - - it('should handle complex numbered prefixes', () => { - const input = 'docs/svelte/99-advanced-topics.md' - const expected = 'advanced-topics' - expect(extractTitleFromPath(input)).toBe(expected) - }) - - it('should handle files with multiple numbered prefixes', () => { - const input = 'docs/svelte/01-02-nested-numbering.md' - const expected = '02-nested-numbering' - expect(extractTitleFromPath(input)).toBe(expected) - }) - - it('should handle just a filename', () => { - const input = '01-introduction.md' - const expected = 'introduction' - expect(extractTitleFromPath(input)).toBe(expected) - }) - - it('should handle empty string', () => { - const input = '' - const expected = '' - expect(extractTitleFromPath(input)).toBe(expected) - }) - - it('should handle paths with no filename', () => { - const input = 'docs/svelte/' - const expected = '' - expect(extractTitleFromPath(input)).toBe(expected) - }) - - it('should handle files with hyphens but no numbers', () => { - const input = 'docs/svelte/state-management.md' - const expected = 'state-management' - expect(extractTitleFromPath(input)).toBe(expected) - }) - - it('should handle files with numbers in the middle', () => { - const input = 'docs/svelte/svelte5-features.md' - const expected = 'svelte5-features' - expect(extractTitleFromPath(input)).toBe(expected) - }) - - it('should handle tutorial paths', () => { - const input = 'tutorial/01-introduction/01-hello-world.md' - const expected = 'hello-world' - expect(extractTitleFromPath(input)).toBe(expected) - }) - - it('should handle SvelteKit paths', () => { - const input = 'docs/kit/01-routing.md' - const expected = 'routing' - expect(extractTitleFromPath(input)).toBe(expected) - }) - }) - - describe('removeFrontmatter', () => { - it('should remove valid frontmatter from content', () => { - const input = `--- -title: Introduction -description: Getting started guide ---- - -# Introduction - -This is the main content.` - const expected = `# Introduction - -This is the main content.` - expect(removeFrontmatter(input)).toBe(expected) - }) - - it('should handle content without frontmatter', () => { - const input = `# Introduction - -This is content without frontmatter.` - const expected = `# Introduction - -This is content without frontmatter.` - expect(removeFrontmatter(input)).toBe(expected) - }) - - it('should handle empty content', () => { - const input = '' - const expected = '' - expect(removeFrontmatter(input)).toBe(expected) - }) - - it('should handle malformed frontmatter (no closing delimiter)', () => { - const input = `--- -title: Introduction -This is malformed frontmatter without closing delimiter - -# Content here` - const expected = input // Should return original content unchanged - expect(removeFrontmatter(input)).toBe(expected) - }) - - it('should handle frontmatter with complex YAML', () => { - const input = `--- -title: Complex Example -tags: - - svelte - - tutorial -metadata: - author: John Doe - date: 2024-01-15 ---- - -# Complex Example - -Content with complex frontmatter.` - const expected = `# Complex Example - -Content with complex frontmatter.` - expect(removeFrontmatter(input)).toBe(expected) - }) - - it('should handle content that starts with --- but is not frontmatter', () => { - const input = `--- -This is not YAML frontmatter, just content that starts with ---` - const expected = input // Should return original content unchanged - expect(removeFrontmatter(input)).toBe(expected) - }) - - it('should handle frontmatter with empty lines', () => { - const input = `--- -title: Introduction - -description: A guide ---- - -# Content` - const expected = `# Content` - expect(removeFrontmatter(input)).toBe(expected) - }) - - it('should trim whitespace after removing frontmatter', () => { - const input = `--- -title: Introduction ---- - - -# Content with leading whitespace` - const expected = `# Content with leading whitespace` - expect(removeFrontmatter(input)).toBe(expected) - }) - - it('should handle frontmatter at the end of content', () => { - const input = `--- -title: Only Frontmatter ----` - const expected = `` - expect(removeFrontmatter(input)).toBe(expected) - }) - }) - - describe('integration tests', () => { - it('should work together for typical documentation workflow', () => { - // Simulate a typical path from tarball to display - const tarballPath = 'svelte.dev-main/apps/svelte.dev/content/docs/svelte/01-introduction.md' - - // Clean tarball path - const cleanedFromTarball = cleanTarballPath(tarballPath) - expect(cleanedFromTarball).toBe('apps/svelte.dev/content/docs/svelte/01-introduction.md') - - // This would be stored in DB and later cleaned for display - const cleanedForDisplay = cleanDocumentationPath(cleanedFromTarball) - expect(cleanedForDisplay).toBe('docs/svelte/01-introduction.md') - - // Extract title for metadata - const title = extractTitleFromPath(cleanedFromTarball) - expect(title).toBe('introduction') - }) - - it('should handle SvelteKit paths through full workflow', () => { - const tarballPath = 'svelte.dev-main/apps/svelte.dev/content/docs/kit/01-routing.md' - - const cleanedFromTarball = cleanTarballPath(tarballPath) - expect(cleanedFromTarball).toBe('apps/svelte.dev/content/docs/kit/01-routing.md') - - const cleanedForDisplay = cleanDocumentationPath(cleanedFromTarball) - expect(cleanedForDisplay).toBe('docs/kit/01-routing.md') - - const title = extractTitleFromPath(cleanedFromTarball) - expect(title).toBe('routing') - }) - - it('should handle tutorial paths through full workflow', () => { - const tarballPath = - 'svelte.dev-main/apps/svelte.dev/content/tutorial/01-introduction/01-hello-world.md' - - const cleanedFromTarball = cleanTarballPath(tarballPath) - expect(cleanedFromTarball).toBe( - 'apps/svelte.dev/content/tutorial/01-introduction/01-hello-world.md' - ) - - const cleanedForDisplay = cleanDocumentationPath(cleanedFromTarball) - expect(cleanedForDisplay).toBe('tutorial/01-introduction/01-hello-world.md') - - const title = extractTitleFromPath(cleanedFromTarball) - expect(title).toBe('hello-world') - }) - - it('should handle content processing with frontmatter removal', () => { - const content = `--- -title: Introduction ---- - -# Introduction - -This is the content.` - - const contentWithoutFrontmatter = removeFrontmatter(content) - expect(contentWithoutFrontmatter).toBe(`# Introduction - -This is the content.`) - }) - }) -}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0ac45f3..b1aabe5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,9 +17,39 @@ importers: '@modelcontextprotocol/inspector': specifier: ^0.16.7 version: 0.16.8(@types/node@24.5.2)(typescript@5.9.2) - eslint: - specifier: ^9.36.0 - version: 9.36.0 + '@sveltejs/adapter-vercel': + specifier: ^5.6.3 + version: 5.10.2(@sveltejs/kit@2.42.2(@sveltejs/vite-plugin-svelte@6.2.0(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)))(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)))(rollup@4.51.0) + '@sveltejs/kit': + specifier: ^2.22.0 + version: 2.42.2(@sveltejs/vite-plugin-svelte@6.2.0(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)))(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)) + '@sveltejs/vite-plugin-svelte': + specifier: ^6.0.0 + version: 6.2.0(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)) + '@types/eslint-scope': + specifier: ^8.3.2 + version: 8.3.2 + '@types/estree': + specifier: ^1.0.8 + version: 1.0.8 + '@types/node': + specifier: ^24.3.1 + version: 24.5.2 + '@types/tar-stream': + specifier: ^3.1.4 + version: 3.1.4 + '@typescript-eslint/types': + specifier: ^8.43.0 + version: 8.44.1 + dotenv: + specifier: ^17.2.2 + version: 17.2.2 + drizzle-kit: + specifier: ^0.30.2 + version: 0.30.6 + drizzle-orm: + specifier: ^0.40.0 + version: 0.40.1(@libsql/client@0.14.0)(gel@2.1.1) eslint-config-prettier: specifier: ^10.0.1 version: 10.1.8(eslint@9.36.0) @@ -32,12 +62,24 @@ importers: globals: specifier: ^16.0.0 version: 16.4.0 + minimatch: + specifier: ^10.0.3 + version: 10.0.3 prettier: specifier: ^3.4.2 version: 3.6.2 prettier-plugin-svelte: specifier: ^3.3.3 version: 3.4.0(prettier@3.6.2)(svelte@5.39.2) + svelte: + specifier: ^5.0.0 + version: 5.39.2 + svelte-check: + specifier: ^4.0.0 + version: 4.3.1(picomatch@4.0.3)(svelte@5.39.2)(typescript@5.9.2) + tar-stream: + specifier: ^3.1.7 + version: 3.1.7 typescript: specifier: ^5.0.0 version: 5.9.2 @@ -59,6 +101,15 @@ importers: '@tmcp/transport-http': specifier: ^0.6.2 version: 0.6.2(tmcp@1.13.0(typescript@5.9.2)) + '@types/tar-stream': + specifier: ^3.1.4 + version: 3.1.4 + minimatch: + specifier: ^10.0.3 + version: 10.0.3 + tar-stream: + specifier: ^3.1.7 + version: 3.1.7 devDependencies: '@eslint/compat': specifier: ^1.3.2 @@ -84,21 +135,9 @@ importers: '@types/node': specifier: ^24.3.1 version: 24.5.2 -<<<<<<< HEAD - '@types/tar-stream': - specifier: ^3.1.4 - version: 3.1.4 - '@typescript-eslint/types': - specifier: ^8.43.0 - version: 8.44.0 - dotenv: - specifier: ^17.2.2 - version: 17.2.2 -======= '@typescript-eslint/parser': specifier: ^8.44.0 version: 8.44.0(eslint@9.36.0)(typescript@5.9.2) ->>>>>>> main drizzle-kit: specifier: ^0.30.2 version: 0.30.6 @@ -114,9 +153,6 @@ importers: globals: specifier: ^16.0.0 version: 16.4.0 - minimatch: - specifier: ^10.0.3 - version: 10.0.3 prettier: specifier: ^3.4.2 version: 3.6.2 @@ -129,15 +165,9 @@ importers: svelte-check: specifier: ^4.0.0 version: 4.3.1(picomatch@4.0.3)(svelte@5.39.2)(typescript@5.9.2) -<<<<<<< HEAD - tar-stream: - specifier: ^3.1.7 - version: 3.1.7 -======= svelte-eslint-parser: specifier: ^1.3.2 version: 1.3.2(svelte@5.39.2) ->>>>>>> main typescript: specifier: ^5.0.0 version: 5.9.2 @@ -1997,6 +2027,10 @@ packages: resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==} engines: {node: '>=0.10.0'} + dotenv@17.2.2: + resolution: {integrity: sha512-Sf2LSQP+bOlhKWWyhFsn0UsfdK/kCWRv1iuA2gXAwt3dyNabr6QSj00I2V10pidqz69soatm9ZwZvpQMTIOd5Q==} + engines: {node: '>=12'} + drizzle-kit@0.30.6: resolution: {integrity: sha512-U4wWit0fyZuGuP7iNmRleQyK2V8wCuv57vf5l3MnG4z4fzNTjY/U13M8owyQ5RavqvqxBifWORaR3wIUzlN64g==} hasBin: true @@ -3298,14 +3332,12 @@ packages: std-env@3.9.0: resolution: {integrity: sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw==} -<<<<<<< HEAD - streamx@2.22.1: - resolution: {integrity: sha512-znKXEBxfatz2GBNK02kRnCXjV+AA4kjZIUxeWSr3UGirZMJfTE9uiwKHobnbgxWyL/JWro8tTq+vOqAK1/qbSA==} -======= stop-iteration-iterator@1.1.0: resolution: {integrity: sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ==} engines: {node: '>= 0.4'} ->>>>>>> main + + streamx@2.22.1: + resolution: {integrity: sha512-znKXEBxfatz2GBNK02kRnCXjV+AA4kjZIUxeWSr3UGirZMJfTE9uiwKHobnbgxWyL/JWro8tTq+vOqAK1/qbSA==} string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} @@ -4853,7 +4885,7 @@ snapshots: '@typescript-eslint/project-service@8.44.0(typescript@5.9.2)': dependencies: '@typescript-eslint/tsconfig-utils': 8.44.0(typescript@5.9.2) - '@typescript-eslint/types': 8.44.0 + '@typescript-eslint/types': 8.44.1 debug: 4.4.3 typescript: 5.9.2 transitivePeerDependencies: @@ -5390,6 +5422,8 @@ snapshots: dependencies: esutils: 2.0.3 + dotenv@17.2.2: {} + drizzle-kit@0.30.6: dependencies: '@drizzle-team/brocli': 0.10.2 @@ -6787,7 +6821,11 @@ snapshots: std-env@3.9.0: {} -<<<<<<< HEAD + stop-iteration-iterator@1.1.0: + dependencies: + es-errors: 1.3.0 + internal-slot: 1.1.0 + streamx@2.22.1: dependencies: fast-fifo: 1.3.2 @@ -6796,12 +6834,6 @@ snapshots: bare-events: 2.7.0 transitivePeerDependencies: - react-native-b4a -======= - stop-iteration-iterator@1.1.0: - dependencies: - es-errors: 1.3.0 - internal-slot: 1.1.0 ->>>>>>> main string-width@4.2.3: dependencies: From 31d64712f007313ae018dc0a3b95bec595264299 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Wed, 24 Sep 2025 21:40:18 +0200 Subject: [PATCH 06/14] Update pnpm-lock.yaml --- pnpm-lock.yaml | 162 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 149 insertions(+), 13 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 61fdcbc..d9b2993 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,9 +17,39 @@ importers: '@modelcontextprotocol/inspector': specifier: ^0.16.7 version: 0.16.8(@types/node@24.5.2)(typescript@5.9.2) - eslint: - specifier: ^9.36.0 - version: 9.36.0(jiti@2.6.0) + '@sveltejs/adapter-vercel': + specifier: ^5.6.3 + version: 5.10.2(@sveltejs/kit@2.42.2(@sveltejs/vite-plugin-svelte@6.2.0(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)))(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)))(rollup@4.51.0) + '@sveltejs/kit': + specifier: ^2.22.0 + version: 2.42.2(@sveltejs/vite-plugin-svelte@6.2.0(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)))(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)) + '@sveltejs/vite-plugin-svelte': + specifier: ^6.0.0 + version: 6.2.0(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)) + '@types/eslint-scope': + specifier: ^8.3.2 + version: 8.3.2 + '@types/estree': + specifier: ^1.0.8 + version: 1.0.8 + '@types/node': + specifier: ^24.3.1 + version: 24.5.2 + '@types/tar-stream': + specifier: ^3.1.4 + version: 3.1.4 + '@typescript-eslint/types': + specifier: ^8.43.0 + version: 8.44.1 + dotenv: + specifier: ^17.2.2 + version: 17.2.2 + drizzle-kit: + specifier: ^0.30.2 + version: 0.30.6 + drizzle-orm: + specifier: ^0.40.0 + version: 0.40.1(@libsql/client@0.14.0)(gel@2.1.1) eslint-config-prettier: specifier: ^10.0.1 version: 10.1.8(eslint@9.36.0(jiti@2.6.0)) @@ -32,12 +62,24 @@ importers: globals: specifier: ^16.0.0 version: 16.4.0 + minimatch: + specifier: ^10.0.3 + version: 10.0.3 prettier: specifier: ^3.4.2 version: 3.6.2 prettier-plugin-svelte: specifier: ^3.3.3 version: 3.4.0(prettier@3.6.2)(svelte@5.39.2) + svelte: + specifier: ^5.0.0 + version: 5.39.2 + svelte-check: + specifier: ^4.0.0 + version: 4.3.1(picomatch@4.0.3)(svelte@5.39.2)(typescript@5.9.2) + tar-stream: + specifier: ^3.1.7 + version: 3.1.7 typescript: specifier: ^5.0.0 version: 5.9.2 @@ -59,6 +101,15 @@ importers: '@tmcp/transport-http': specifier: ^0.6.2 version: 0.6.2(tmcp@1.13.0(typescript@5.9.2)) + '@types/tar-stream': + specifier: ^3.1.4 + version: 3.1.4 + minimatch: + specifier: ^10.0.3 + version: 10.0.3 + tar-stream: + specifier: ^3.1.7 + version: 3.1.7 devDependencies: '@eslint/compat': specifier: ^1.3.2 @@ -770,6 +821,14 @@ packages: resolution: {integrity: sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==} engines: {node: '>=18.18'} + '@isaacs/balanced-match@4.0.1': + resolution: {integrity: sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==} + engines: {node: 20 || >=22} + + '@isaacs/brace-expansion@5.0.0': + resolution: {integrity: sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==} + engines: {node: 20 || >=22} + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -1587,6 +1646,9 @@ packages: '@types/node@24.5.2': resolution: {integrity: sha512-FYxk1I7wPv3K2XBaoyH2cTnocQEu8AOZ60hPbsyukMPLv5/5qr7V1i8PLHdl6Zf87I+xZXFvPCXYjiTFq+YSDQ==} + '@types/tar-stream@3.1.4': + resolution: {integrity: sha512-921gW0+g29mCJX0fRvqeHzBlE/XclDaAG0Ousy1LCghsOhvaKacDeRGEVzQP9IPfKn8Vysy7FEXAIxycpc/CMg==} + '@types/ws@8.18.1': resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} @@ -1862,9 +1924,20 @@ packages: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} engines: {node: '>= 0.4'} + b4a@1.7.2: + resolution: {integrity: sha512-DyUOdz+E8R6+sruDpQNOaV0y/dBbV6X/8ZkxrDcR0Ifc3BgKlpgG0VAtfOozA0eMtJO5GGe9FsZhueLs00pTww==} + peerDependencies: + react-native-b4a: '*' + peerDependenciesMeta: + react-native-b4a: + optional: true + balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + bare-events@2.7.0: + resolution: {integrity: sha512-b3N5eTW1g7vXkw+0CXh/HazGTcO5KYuu/RCNaJbDMPI6LHDi+7qe8EmxKUVe1sUbY2KZOVZFyj62x0OEz9qyAA==} + bindings@1.5.0: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} @@ -2117,6 +2190,10 @@ packages: resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==} engines: {node: '>=0.10.0'} + dotenv@17.2.2: + resolution: {integrity: sha512-Sf2LSQP+bOlhKWWyhFsn0UsfdK/kCWRv1iuA2gXAwt3dyNabr6QSj00I2V10pidqz69soatm9ZwZvpQMTIOd5Q==} + engines: {node: '>=12'} + drizzle-kit@0.30.6: resolution: {integrity: sha512-U4wWit0fyZuGuP7iNmRleQyK2V8wCuv57vf5l3MnG4z4fzNTjY/U13M8owyQ5RavqvqxBifWORaR3wIUzlN64g==} hasBin: true @@ -2417,6 +2494,9 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + events-universal@1.0.1: + resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} + eventsource-parser@3.0.6: resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==} engines: {node: '>=18.0.0'} @@ -2442,6 +2522,9 @@ packages: fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + fast-glob@3.3.3: resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} engines: {node: '>=8.6.0'} @@ -2929,6 +3012,10 @@ packages: resolution: {integrity: sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA==} engines: {node: '>= 0.6'} + minimatch@10.0.3: + resolution: {integrity: sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw==} + engines: {node: 20 || >=22} + minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} @@ -3476,6 +3563,9 @@ packages: resolution: {integrity: sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ==} engines: {node: '>= 0.4'} + streamx@2.23.0: + resolution: {integrity: sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg==} + string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} @@ -3551,10 +3641,16 @@ packages: tailwind-merge@2.6.0: resolution: {integrity: sha512-P+Vu1qXfzediirmHOC3xKGAYeZtPcV9g76X+xg2FD4tYgR71ewMA35Y3sCz3zhiN/dwefRpJX0yBcgwi1fXNQA==} + tar-stream@3.1.7: + resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} + tar@7.4.3: resolution: {integrity: sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==} engines: {node: '>=18'} + text-decoder@1.2.3: + resolution: {integrity: sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==} + tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -4301,6 +4397,12 @@ snapshots: '@humanwhocodes/retry@0.4.3': {} + '@isaacs/balanced-match@4.0.1': {} + + '@isaacs/brace-expansion@5.0.0': + dependencies: + '@isaacs/balanced-match': 4.0.1 + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -5079,6 +5181,10 @@ snapshots: dependencies: undici-types: 7.12.0 + '@types/tar-stream@3.1.4': + dependencies: + '@types/node': 24.5.2 + '@types/ws@8.18.1': dependencies: '@types/node': 24.5.2 @@ -5144,7 +5250,7 @@ snapshots: '@typescript-eslint/project-service@8.44.0(typescript@5.9.2)': dependencies: '@typescript-eslint/tsconfig-utils': 8.44.0(typescript@5.9.2) - '@typescript-eslint/types': 8.44.0 + '@typescript-eslint/types': 8.44.1 debug: 4.4.3 typescript: 5.9.2 transitivePeerDependencies: @@ -5300,14 +5406,6 @@ snapshots: chai: 5.3.3 tinyrainbow: 2.0.0 - '@vitest/mocker@3.2.4(vite@7.1.6(@types/node@22.18.6)(jiti@2.6.0)(yaml@2.8.1))': - dependencies: - '@vitest/spy': 3.2.4 - estree-walker: 3.0.3 - magic-string: 0.30.19 - optionalDependencies: - vite: 7.1.6(@types/node@22.18.6)(jiti@2.6.0)(yaml@2.8.1) - '@vitest/mocker@3.2.4(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1))': dependencies: '@vitest/spy': 3.2.4 @@ -5461,8 +5559,12 @@ snapshots: axobject-query@4.1.0: {} + b4a@1.7.2: {} + balanced-match@1.0.2: {} + bare-events@2.7.0: {} + bindings@1.5.0: dependencies: file-uri-to-path: 1.0.0 @@ -5697,6 +5799,8 @@ snapshots: dependencies: esutils: 2.0.3 + dotenv@17.2.2: {} + drizzle-kit@0.30.6: dependencies: '@drizzle-team/brocli': 0.10.2 @@ -6063,6 +6167,10 @@ snapshots: etag@1.8.1: {} + events-universal@1.0.1: + dependencies: + bare-events: 2.7.0 + eventsource-parser@3.0.6: {} eventsource@3.0.7: @@ -6109,6 +6217,8 @@ snapshots: fast-deep-equal@3.1.3: {} + fast-fifo@1.3.2: {} + fast-glob@3.3.3: dependencies: '@nodelib/fs.stat': 2.0.5 @@ -6585,6 +6695,10 @@ snapshots: dependencies: mime-db: 1.54.0 + minimatch@10.0.3: + dependencies: + '@isaacs/brace-expansion': 5.0.0 + minimatch@3.1.2: dependencies: brace-expansion: 1.1.12 @@ -7152,6 +7266,14 @@ snapshots: es-errors: 1.3.0 internal-slot: 1.1.0 + streamx@2.23.0: + dependencies: + events-universal: 1.0.1 + fast-fifo: 1.3.2 + text-decoder: 1.2.3 + transitivePeerDependencies: + - react-native-b4a + string-width@4.2.3: dependencies: emoji-regex: 8.0.0 @@ -7255,6 +7377,14 @@ snapshots: tailwind-merge@2.6.0: {} + tar-stream@3.1.7: + dependencies: + b4a: 1.7.2 + fast-fifo: 1.3.2 + streamx: 2.23.0 + transitivePeerDependencies: + - react-native-b4a + tar@7.4.3: dependencies: '@isaacs/fs-minipass': 4.0.1 @@ -7264,6 +7394,12 @@ snapshots: mkdirp: 3.0.1 yallist: 5.0.0 + text-decoder@1.2.3: + dependencies: + b4a: 1.7.2 + transitivePeerDependencies: + - react-native-b4a + tinybench@2.9.0: {} tinyexec@0.3.2: {} @@ -7558,7 +7694,7 @@ snapshots: dependencies: '@types/chai': 5.2.2 '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@7.1.6(@types/node@22.18.6)(jiti@2.6.0)(yaml@2.8.1)) + '@vitest/mocker': 3.2.4(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)) '@vitest/pretty-format': 3.2.4 '@vitest/runner': 3.2.4 '@vitest/snapshot': 3.2.4 From 67f4571b2eb2e1827899ea3cddf0be72c3b3e5c2 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Wed, 24 Sep 2025 21:41:58 +0200 Subject: [PATCH 07/14] Update schema.js --- packages/mcp-schema/src/schema.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/mcp-schema/src/schema.js b/packages/mcp-schema/src/schema.js index e98feee..a2a0321 100644 --- a/packages/mcp-schema/src/schema.js +++ b/packages/mcp-schema/src/schema.js @@ -51,7 +51,7 @@ export const distillation_jobs = sqliteTable('distillation_jobs', { export const content = sqliteTable('content', { id: integer('id').primaryKey(), - path: text('path').notNull(), + path: text('path').notNull().unique(), filename: text('filename').notNull(), content: text('content').notNull(), size_bytes: integer('size_bytes').notNull(), @@ -67,7 +67,7 @@ export const content = sqliteTable('content', { export const content_distilled = sqliteTable('content_distilled', { id: integer('id').primaryKey(), - path: text('path').notNull(), + path: text('path').notNull().unique(), filename: text('filename').notNull(), content: text('content').notNull(), size_bytes: integer('size_bytes').notNull(), From 3b58d51336b963aa78cea5549217e9cfb9a1bc3a Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Wed, 24 Sep 2025 21:42:14 +0200 Subject: [PATCH 08/14] Update pathUtils.ts --- apps/mcp-remote/src/lib/utils/pathUtils.ts | 48 +++++++++++----------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/apps/mcp-remote/src/lib/utils/pathUtils.ts b/apps/mcp-remote/src/lib/utils/pathUtils.ts index a213103..cf2f135 100644 --- a/apps/mcp-remote/src/lib/utils/pathUtils.ts +++ b/apps/mcp-remote/src/lib/utils/pathUtils.ts @@ -10,11 +10,11 @@ * @returns The cleaned path */ export function cleanDocumentationPath(path: string): string { - const prefix = 'apps/svelte.dev/content/' + const prefix = 'apps/svelte.dev/content/'; if (path.startsWith(prefix)) { - return path.substring(prefix.length) + return path.substring(prefix.length); } - return path + return path; } /** @@ -26,7 +26,7 @@ export function cleanDocumentationPath(path: string): string { */ export function cleanTarballPath(path: string): string { // Remove only the repo directory prefix (first segment) - return path.split('/').slice(1).join('/') + return path.split('/').slice(1).join('/'); } /** @@ -37,19 +37,19 @@ export function cleanTarballPath(path: string): string { */ export function extractTitleFromPath(filePath: string): string { if (!filePath) { - return '' + return ''; } - const pathParts = filePath.split('/') - const filename = pathParts[pathParts.length - 1] + const pathParts = filePath.split('/'); + const filename = pathParts[pathParts.length - 1]; // Handle empty filename (e.g., paths ending with '/') if (!filename) { - return '' + return ''; } // Remove .md extension and numbered prefixes - return filename.replace('.md', '').replace(/^\d+-/, '') + return filename.replace('.md', '').replace(/^\d+-/, ''); } /** @@ -61,50 +61,50 @@ export function extractTitleFromPath(filePath: string): string { */ export function removeFrontmatter(content: string): string { if (!content || content.length === 0) { - return content + return content; } // Check if content starts with frontmatter delimiter if (!content.startsWith('---\n')) { - return content + return content; } - let position = 4 // Start after the opening "---\n" - let insideFrontmatter = true - let frontmatterEndOffset: number | null = null + let position = 4; // Start after the opening "---\n" + let insideFrontmatter = true; + let frontmatterEndOffset: number | null = null; // Traverse the string character by character while (position < content.length && insideFrontmatter) { - const char = content[position] + const char = content[position]; // Look for potential end of frontmatter: \n--- if (char === '\n' && position + 3 < content.length) { - const nextThree = content.substring(position + 1, position + 4) + const nextThree = content.substring(position + 1, position + 4); if (nextThree === '---') { // Check what comes after the closing --- - const afterClosing = position + 4 + const afterClosing = position + 4; if (afterClosing >= content.length) { // End of string - this is valid frontmatter - frontmatterEndOffset = content.length - insideFrontmatter = false + frontmatterEndOffset = content.length; + insideFrontmatter = false; } else if (content[afterClosing] === '\n') { // Followed by newline - this is valid frontmatter - frontmatterEndOffset = afterClosing + 1 - insideFrontmatter = false + frontmatterEndOffset = afterClosing + 1; + insideFrontmatter = false; } // If followed by something else, it's not the end delimiter, continue searching } } - position++ + position++; } // If we never found the end of frontmatter, it's malformed if (frontmatterEndOffset === null) { - return content + return content; } // Return content after the frontmatter, trimmed - return content.substring(frontmatterEndOffset).trim() + return content.substring(frontmatterEndOffset).trim(); } From 0d773cf133b5e1759f7d20fa3ffd93fda210f160 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Wed, 24 Sep 2025 21:42:16 +0200 Subject: [PATCH 09/14] Delete db.ts --- apps/mcp-remote/src/lib/server/db.ts | 71 ---------------------------- 1 file changed, 71 deletions(-) delete mode 100644 apps/mcp-remote/src/lib/server/db.ts diff --git a/apps/mcp-remote/src/lib/server/db.ts b/apps/mcp-remote/src/lib/server/db.ts deleted file mode 100644 index f6febbf..0000000 --- a/apps/mcp-remote/src/lib/server/db.ts +++ /dev/null @@ -1,71 +0,0 @@ -import PG from 'pg'; -import type { Pool } from 'pg'; -import type { QueryResult } from 'pg'; -import type { QueryConfig } from '$lib/types/db'; - -import { env } from '$env/dynamic/private'; -import { logAlways, log, logErrorAlways } from '$lib/log'; - -let pool: Pool | null = null; - -export function maybeInitializePool(): Pool { - if (!pool) { - logAlways('🐘 Initializing Postgres connection!'); - pool = new PG.Pool({ - connectionString: env.DB_URL || 'postgres://admin:admin@localhost:5432/db', - max: parseInt(env.DB_CLIENTS || '10'), - }); - } - return pool; -} - -export async function query( - incomingQuery: string, - params: unknown[] = [], - config: QueryConfig = {}, -): Promise { - const pool = maybeInitializePool(); - - if (!pool) { - throw new Error('Database connection pool is not initialized'); - } - - const timingStart = new Date(); - - if (config.debug === true || env?.DB_DEBUG === 'true') { - log('----'); - log(`🔰 Query: ${incomingQuery}`); - log('📊 Data: ', params); - } - - try { - const results = await pool.query(incomingQuery, params); - - if (config.debug === true || env?.DB_DEBUG === 'true') { - log('⏰ Postgres query execution time: %dms', new Date().getTime() - timingStart.getTime()); - log('----'); - } - - return results; - } catch (error) { - logErrorAlways('Database query error:', { - query: incomingQuery, - params, - error: error instanceof Error ? error.message : String(error), - }); - - // Re-throw the error to let it bubble up - throw error; - } -} - -export async function disconnect(): Promise { - if (pool !== null) { - logAlways('😵 Disconnecting from Postgres!'); - const thisPool = pool; - pool = null; - return await thisPool.end(); - } - - return; -} From f9ca27b4bb77bb90a98dadec0cebe991b39e1d98 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Wed, 24 Sep 2025 21:42:25 +0200 Subject: [PATCH 10/14] format --- apps/mcp-remote/drizzle.config.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/mcp-remote/drizzle.config.ts b/apps/mcp-remote/drizzle.config.ts index 0a1fc40..966c94f 100644 --- a/apps/mcp-remote/drizzle.config.ts +++ b/apps/mcp-remote/drizzle.config.ts @@ -1,12 +1,14 @@ import { defineConfig } from 'drizzle-kit'; if (!process.env.DATABASE_URL) throw new Error('DATABASE_URL is not set'); -if (!process.env.DATABASE_TOKEN) throw new Error('DATABASE_TOKEN is not set'); export default defineConfig({ schema: './src/lib/server/db/schema.ts', dialect: 'turso', - dbCredentials: { url: process.env.DATABASE_URL, authToken: process.env.DATABASE_TOKEN }, + dbCredentials: { + url: process.env.DATABASE_URL, + authToken: process.env.DATABASE_TOKEN || '', + }, verbose: true, strict: true, }); From 7d2ad3fe129b1e4da63ad5207c56f7fc7d6f8352 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Wed, 24 Sep 2025 21:55:20 +0200 Subject: [PATCH 11/14] Update CLAUDE.md --- CLAUDE.md | 109 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 75 insertions(+), 34 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c240871..151362c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,72 +2,113 @@ This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. -## Development Commands +## Monorepo Structure + +This is a pnpm monorepo containing Svelte MCP (Model Context Protocol) server implementations across multiple packages and applications: + +- **apps/mcp-remote**: SvelteKit web application with MCP server functionality +- **packages/mcp-server**: Core MCP server implementation with code analysis tools +- **packages/mcp-stdio**: Standalone MCP server CLI with STDIO transport +- **packages/mcp-schema**: Shared schema definitions and database utilities -This is a Svelte MCP (Model Context Protocol) server implementation that includes both SvelteKit web interface and MCP server functionality. +## Development Commands ### Setup ```bash pnpm i -cp .env.example .env +cp apps/mcp-remote/.env.example apps/mcp-remote/.env # Set the VOYAGE_API_KEY for embeddings support in .env +``` + +### Starting the mcp-remote app in dev mode + +```bash +# Start the SvelteKit development server for mcp-remote +cd apps/mcp-remote pnpm dev ``` -### Common Commands +Or from the root: + +```bash +# Run dev command for all workspaces +pnpm dev +``` + +### Common Commands (from root) + +- `pnpm build` - Build all packages and applications +- `pnpm check` - Run type checking across all packages +- `pnpm lint` - Run prettier check and eslint across all packages +- `pnpm format` - Format code with prettier across all packages +- `pnpm test` - Run unit tests across all packages +- `pnpm test:watch` - Run tests in watch mode + +### mcp-remote App Commands + +Navigate to `apps/mcp-remote/` to run these commands: - `pnpm dev` - Start SvelteKit development server -- `pnpm build` - Build the application for production +- `pnpm build` - Build the SvelteKit application for production +- `pnpm build:mcp` - Build the MCP server TypeScript files - `pnpm start` - Run the MCP server (Node.js entry point) - `pnpm check` - Run Svelte type checking - `pnpm check:watch` - Run type checking in watch mode -- `pnpm lint` - Run prettier check and eslint -- `pnpm format` - Format code with prettier -- `pnpm test` - Run unit tests with vitest -- `pnpm test:watch` - Run tests in watch mode - -### Database Commands (Drizzle ORM) - - `pnpm db:push` - Push schema changes to database - `pnpm db:generate` - Generate migration files - `pnpm db:migrate` - Run migrations - `pnpm db:studio` - Open Drizzle Studio +- `pnpm inspect` - Start MCP inspector at http://localhost:6274/ + +### MCP Inspector Usage + +After running `pnpm inspect`, visit http://localhost:6274/: +- Transport type: `Streamable HTTP` +- URL: http://localhost:5173/mcp (when dev server is running) ## Architecture -### MCP Server Implementation +### Monorepo Package Structure + +- **@sveltejs/mcp-remote**: Full SvelteKit application with web interface and MCP server +- **@sveltejs/mcp-server**: Core MCP server logic and code analysis engine (private workspace package) +- **@sveltejs/mcp**: Standalone CLI MCP server with STDIO transport (publishable) +- **@sveltejs/mcp-schema**: Shared database schema and utilities (private workspace package) -The core MCP server is implemented in `src/lib/mcp/index.ts` using the `tmcp` library with: +### mcp-remote App (apps/mcp-remote) -- **Transport Layers**: Both HTTP (`HttpTransport`) and STDIO (`StdioTransport`) support -- **Schema Validation**: Uses Valibot with `ValibotJsonSchemaAdapter` -- **Main Tool**: `svelte-autofixer` - analyzes Svelte code and provides suggestions/fixes +The main SvelteKit application that provides both web interface and MCP server functionality: -### Code Analysis Engine +- **Entry Point**: `src/index.js` for Node.js MCP server +- **SvelteKit Integration**: `src/hooks.server.ts` integrates MCP HTTP transport with SvelteKit requests +- **MCP Server**: `src/lib/mcp/index.ts` - HTTP and STDIO transport support +- **Database**: SQLite with Drizzle ORM, vector storage for embeddings +- **Content Sync**: `src/lib/server/contentSync.ts` and `src/lib/server/contentDb.ts` for content management -Located in `src/lib/server/analyze/`: +### mcp-server Package (packages/mcp-server) -- **Parser** (`parse.ts`): Uses `svelte-eslint-parser` and TypeScript parser to analyze Svelte components -- **Scope Analysis**: Tracks variables, references, and scopes across the AST -- **Rune Detection**: Identifies Svelte 5 runes (`$state`, `$effect`, `$derived`, etc.) +Core MCP server implementation shared across applications: -### Autofixer System +- **Main Export**: `src/index.ts` +- **MCP Implementation**: `src/mcp/index.ts` using `tmcp` library with Valibot schema validation +- **Code Analysis**: Svelte component parsing with `svelte-eslint-parser` and TypeScript parser +- **Autofixers**: Visitor pattern implementations for code analysis and suggestions +- **Tools**: `svelte-autofixer` - analyzes Svelte code and provides suggestions/fixes -- **Autofixers** (`src/lib/mcp/autofixers.ts`): Visitor pattern implementations for code analysis -- **Walker Utility** (`src/lib/index.ts`): Enhanced AST walking with visitor mixing capabilities -- **Current Autofixer**: `assign_in_effect` - detects assignments to `$state` variables inside `$effect` blocks +### mcp-stdio Package (packages/mcp-stdio) -### Database Layer +Standalone publishable MCP server with STDIO transport: -- **ORM**: Drizzle with SQLite backend -- **Schema** (`src/lib/server/db/schema.ts`): Vector table for embeddings support -- **Utils** (`src/lib/server/db/utils.ts`): Custom float32 array type for vectors +- **CLI Binary**: `svelte-mcp` command +- **Entry Point**: `src/index.ts` +- **Transport**: Uses `@tmcp/transport-stdio` for command-line integration -### SvelteKit Integration +### Database Layer (mcp-remote) -- **Hooks** (`src/hooks.server.ts`): Integrates MCP HTTP transport with SvelteKit requests -- **Routes**: Basic web interface for the MCP server +- **ORM**: Drizzle with SQLite backend (`test.db`) +- **Schema**: Located in `src/lib/server/db/schema.ts` with vector table for embeddings +- **Configuration**: `drizzle.config.ts` in mcp-remote app ## Key Dependencies @@ -81,7 +122,7 @@ Located in `src/lib/server/analyze/`: ## Environment Configuration -Required environment variables: +For the mcp-remote app (`apps/mcp-remote/.env`): - `DATABASE_URL`: SQLite database path (default: `file:test.db`) - `VOYAGE_API_KEY`: API key for embeddings support (optional) From de106c2f248c6e2c02b6a123207df1688ec14314 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Wed, 24 Sep 2025 21:58:29 +0200 Subject: [PATCH 12/14] wip --- CLAUDE.md | 7 +++---- package.json | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 151362c..498fae2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,15 +24,14 @@ cp apps/mcp-remote/.env.example apps/mcp-remote/.env ### Starting the mcp-remote app in dev mode ```bash -# Start the SvelteKit development server for mcp-remote -cd apps/mcp-remote +# Start the SvelteKit development server for mcp-remote (from root) pnpm dev ``` -Or from the root: +Or navigate to the app directory: ```bash -# Run dev command for all workspaces +cd apps/mcp-remote pnpm dev ``` diff --git a/package.json b/package.json index 3d38bb0..3d53f78 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "description": "The official Svelte MCP server implementation", "type": "module", "scripts": { + "dev": "pnpm --filter mcp-remote dev", "build": "pnpm -r run build", "check": "pnpm -r run check", "format": "prettier --write .", From 5923a98d3c12649d2065458bf0b1b0c12c77c4f9 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Wed, 24 Sep 2025 22:00:07 +0200 Subject: [PATCH 13/14] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e167ec5..ede829f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Repo for the official Svelte MCP server. ``` pnpm i -cp .env.example .env +cp apps/mcp-remote/.env.example apps/mcp-remote/.env pnpm dev ``` From 7440f7a8a730ab84b0d4acb4240281b623ccc44a Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Wed, 24 Sep 2025 22:11:11 +0200 Subject: [PATCH 14/14] add adapter-node --- apps/mcp-remote/package.json | 1 + pnpm-lock.yaml | 93 ++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/apps/mcp-remote/package.json b/apps/mcp-remote/package.json index 77ff726..b14a945 100644 --- a/apps/mcp-remote/package.json +++ b/apps/mcp-remote/package.json @@ -41,6 +41,7 @@ "@eslint/js": "^9.36.0", "@libsql/client": "^0.14.0", "@modelcontextprotocol/inspector": "^0.16.7", + "@sveltejs/adapter-node": "^5.3.2", "@sveltejs/adapter-vercel": "^5.6.3", "@sveltejs/kit": "^2.22.0", "@sveltejs/vite-plugin-svelte": "^6.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d9b2993..2fc9b70 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -123,6 +123,9 @@ importers: '@modelcontextprotocol/inspector': specifier: ^0.16.7 version: 0.16.8(@types/node@24.5.2)(typescript@5.9.2) + '@sveltejs/adapter-node': + specifier: ^5.3.2 + version: 5.3.2(@sveltejs/kit@2.42.2(@sveltejs/vite-plugin-svelte@6.2.0(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)))(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1))) '@sveltejs/adapter-vercel': specifier: ^5.6.3 version: 5.10.2(@sveltejs/kit@2.42.2(@sveltejs/vite-plugin-svelte@6.2.0(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)))(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)))(rollup@4.51.0) @@ -1421,6 +1424,33 @@ packages: '@rolldown/pluginutils@1.0.0-beta.9': resolution: {integrity: sha512-e9MeMtVWo186sgvFFJOPGy7/d2j2mZhLJIdVW0C/xDluuOvymEATqz6zKsP0ZmXGzQtqlyjz5sC1sYQUoJG98w==} + '@rollup/plugin-commonjs@28.0.6': + resolution: {integrity: sha512-XSQB1K7FUU5QP+3lOQmVCE3I0FcbbNvmNT4VJSj93iUjayaARrTQeoRdiYQoftAJBLrR9t2agwAd3ekaTgHNlw==} + engines: {node: '>=16.0.0 || 14 >= 14.17'} + peerDependencies: + rollup: ^2.68.0||^3.0.0||^4.0.0 + peerDependenciesMeta: + rollup: + optional: true + + '@rollup/plugin-json@6.1.0': + resolution: {integrity: sha512-EGI2te5ENk1coGeADSIwZ7G2Q8CJS2sF120T7jLw4xFw9n7wIOXHo+kIYRAoVpJAN+kmqZSoO3Fp4JtoNF4ReA==} + engines: {node: '>=14.0.0'} + peerDependencies: + rollup: ^1.20.0||^2.0.0||^3.0.0||^4.0.0 + peerDependenciesMeta: + rollup: + optional: true + + '@rollup/plugin-node-resolve@16.0.1': + resolution: {integrity: sha512-tk5YCxJWIG81umIvNkSod2qK5KyQW19qcBF/B78n1bjtOON6gzKoVeSzAE8yHCZEDmqkHKkxplExA8KzdJLJpA==} + engines: {node: '>=14.0.0'} + peerDependencies: + rollup: ^2.78.0||^3.0.0||^4.0.0 + peerDependenciesMeta: + rollup: + optional: true + '@rollup/pluginutils@5.3.0': resolution: {integrity: sha512-5EdhGZtnu3V88ces7s53hhfK5KSASnJZv8Lulpc04cWO3REESroJXg73DFsOmgbU2BhwV0E20bu2IDZb3VKW4Q==} engines: {node: '>=14.0.0'} @@ -1546,6 +1576,11 @@ packages: peerDependencies: acorn: ^8.9.0 + '@sveltejs/adapter-node@5.3.2': + resolution: {integrity: sha512-nBJSipMb1KLjnAM7uzb+YpnA1VWKb+WdR+0mXEnXI6K1A3XYWbjkcjnW20ubg07sicK8XaGY/FAX3PItw39qBQ==} + peerDependencies: + '@sveltejs/kit': ^2.4.0 + '@sveltejs/adapter-vercel@5.10.2': resolution: {integrity: sha512-uWm0jtXbwvXxmELiIXSQ7tcPjlG8roadujxImIxqbKKZ64itZDwTbUsVXYEfUX59LvLjolW9jaODhL6sBTh5NQ==} peerDependencies: @@ -1646,6 +1681,9 @@ packages: '@types/node@24.5.2': resolution: {integrity: sha512-FYxk1I7wPv3K2XBaoyH2cTnocQEu8AOZ60hPbsyukMPLv5/5qr7V1i8PLHdl6Zf87I+xZXFvPCXYjiTFq+YSDQ==} + '@types/resolve@1.20.2': + resolution: {integrity: sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==} + '@types/tar-stream@3.1.4': resolution: {integrity: sha512-921gW0+g29mCJX0fRvqeHzBlE/XclDaAG0Ousy1LCghsOhvaKacDeRGEVzQP9IPfKn8Vysy7FEXAIxycpc/CMg==} @@ -2041,6 +2079,9 @@ packages: resolution: {integrity: sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==} engines: {node: '>=18'} + commondir@1.0.1: + resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} + concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} @@ -2813,6 +2854,9 @@ packages: resolution: {integrity: sha512-1Qed0/Hr2m+YqxnM09CjA2d/i6YZNfF6R2oRAOj36eUdS6qIV/huPJNSEpKbupewFs+ZsJlxsjjPbc0/afW6Lw==} engines: {node: '>= 0.4'} + is-module@1.0.0: + resolution: {integrity: sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g==} + is-negative-zero@2.0.3: resolution: {integrity: sha512-5KoIu2Ngpyek75jXodFvnafB6DJgr3u8uuK0LEZJjrU19DrMD3EVERaR8sjz8CCGgpZvxPl9SuE1GMVPFHx1mw==} engines: {node: '>= 0.4'} @@ -2828,6 +2872,9 @@ packages: is-promise@4.0.0: resolution: {integrity: sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==} + is-reference@1.2.1: + resolution: {integrity: sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ==} + is-reference@3.0.3: resolution: {integrity: sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==} @@ -4985,6 +5032,34 @@ snapshots: '@rolldown/pluginutils@1.0.0-beta.9': {} + '@rollup/plugin-commonjs@28.0.6(rollup@4.51.0)': + dependencies: + '@rollup/pluginutils': 5.3.0(rollup@4.51.0) + commondir: 1.0.1 + estree-walker: 2.0.2 + fdir: 6.5.0(picomatch@4.0.3) + is-reference: 1.2.1 + magic-string: 0.30.19 + picomatch: 4.0.3 + optionalDependencies: + rollup: 4.51.0 + + '@rollup/plugin-json@6.1.0(rollup@4.51.0)': + dependencies: + '@rollup/pluginutils': 5.3.0(rollup@4.51.0) + optionalDependencies: + rollup: 4.51.0 + + '@rollup/plugin-node-resolve@16.0.1(rollup@4.51.0)': + dependencies: + '@rollup/pluginutils': 5.3.0(rollup@4.51.0) + '@types/resolve': 1.20.2 + deepmerge: 4.3.1 + is-module: 1.0.0 + resolve: 1.22.10 + optionalDependencies: + rollup: 4.51.0 + '@rollup/pluginutils@5.3.0(rollup@4.51.0)': dependencies: '@types/estree': 1.0.8 @@ -5064,6 +5139,14 @@ snapshots: dependencies: acorn: 8.15.0 + '@sveltejs/adapter-node@5.3.2(@sveltejs/kit@2.42.2(@sveltejs/vite-plugin-svelte@6.2.0(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)))(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)))': + dependencies: + '@rollup/plugin-commonjs': 28.0.6(rollup@4.51.0) + '@rollup/plugin-json': 6.1.0(rollup@4.51.0) + '@rollup/plugin-node-resolve': 16.0.1(rollup@4.51.0) + '@sveltejs/kit': 2.42.2(@sveltejs/vite-plugin-svelte@6.2.0(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)))(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)) + rollup: 4.51.0 + '@sveltejs/adapter-vercel@5.10.2(@sveltejs/kit@2.42.2(@sveltejs/vite-plugin-svelte@6.2.0(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)))(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)))(rollup@4.51.0)': dependencies: '@sveltejs/kit': 2.42.2(@sveltejs/vite-plugin-svelte@6.2.0(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)))(svelte@5.39.2)(vite@7.1.6(@types/node@24.5.2)(jiti@2.6.0)(yaml@2.8.1)) @@ -5181,6 +5264,8 @@ snapshots: dependencies: undici-types: 7.12.0 + '@types/resolve@1.20.2': {} + '@types/tar-stream@3.1.4': dependencies: '@types/node': 24.5.2 @@ -5682,6 +5767,8 @@ snapshots: commander@13.1.0: {} + commondir@1.0.1: {} + concat-map@0.0.1: {} concurrently@9.2.1: @@ -6520,6 +6607,8 @@ snapshots: is-map@2.0.3: {} + is-module@1.0.0: {} + is-negative-zero@2.0.3: {} is-number-object@1.1.1: @@ -6531,6 +6620,10 @@ snapshots: is-promise@4.0.0: {} + is-reference@1.2.1: + dependencies: + '@types/estree': 1.0.8 + is-reference@3.0.3: dependencies: '@types/estree': 1.0.8