From 1e18609a781d36a9d658179db9297d2ffbae37b5 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 00:19:48 +0100 Subject: [PATCH 01/49] wip --- .../svelte.dev/src/routes/llms.txt/+server.ts | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 apps/svelte.dev/src/routes/llms.txt/+server.ts diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts new file mode 100644 index 0000000000..121aed068e --- /dev/null +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -0,0 +1,66 @@ +import { read } from '$app/server'; +import type { RequestHandler } from './$types'; + +// Import all markdown files +const docs = import.meta.glob('../../../content/docs/**/*.md', { + eager: true, + query: '?raw' +}); + +// Sort function to ensure correct order (svelte -> kit -> cli) +function getSectionPriority(path: string): number { + if (path.includes('/docs/svelte/')) return 0; + if (path.includes('/docs/kit/')) return 1; + if (path.includes('/docs/cli/')) return 2; + return 3; +} + +export const GET: RequestHandler = async () => { + let content = ''; + + // Get all file paths and sort them + const paths = Object.keys(docs).sort((a, b) => { + const priorityA = getSectionPriority(a); + const priorityB = getSectionPriority(b); + if (priorityA !== priorityB) return priorityA - priorityB; + return a.localeCompare(b); + }); + + let currentSection = ''; + + // Process each file + for (const path of paths) { + // Determine section + let section = ''; + if (path.includes('/docs/svelte/')) section = 'SVELTE DOCS'; + else if (path.includes('/docs/kit/')) section = 'SVELTEKIT DOCS'; + else if (path.includes('/docs/cli/')) section = 'CLI DOCS'; + else continue; // Skip other content + + // Add section header if we're entering a new section + if (section !== currentSection) { + if (currentSection) content += '\n\n'; + content += '=====================================\n'; + content += `============ ${section} ===========\n`; + content += '=====================================\n\n'; + currentSection = section; + } + + // Add file path and content + content += `File: ${path.replace('../../../content/', '')}\n\n`; + content += docs[path]; + content += '\n\n-------------------\n\n'; + } + + const headers: HeadersInit = { + 'Content-Type': 'text/plain; charset=utf-8', + 'Cache-Control': 'public, max-age=3600' + }; + + return new Response(content, { + status: 200, + headers + }); +}; + +export const prerender = true; From d665b8697583abf0e2b554b94cf44e9a9944c81b Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 00:20:51 +0100 Subject: [PATCH 02/49] Update +server.ts --- apps/svelte.dev/src/routes/llms.txt/+server.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 121aed068e..5d7738cfd6 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,8 +1,7 @@ -import { read } from '$app/server'; import type { RequestHandler } from './$types'; // Import all markdown files -const docs = import.meta.glob('../../../content/docs/**/*.md', { +const docs = import.meta.glob<{ default: string }>('../../../content/docs/**/*.md', { eager: true, query: '?raw' }); @@ -48,7 +47,7 @@ export const GET: RequestHandler = async () => { // Add file path and content content += `File: ${path.replace('../../../content/', '')}\n\n`; - content += docs[path]; + content += docs[path].default; // Access the default export to get the actual content content += '\n\n-------------------\n\n'; } From d6041cefda5b916108fa3b622e35c099dd712f5f Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 00:27:17 +0100 Subject: [PATCH 03/49] Update +server.ts --- .../svelte.dev/src/routes/llms.txt/+server.ts | 50 ++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 5d7738cfd6..89289a9bc3 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,5 +1,7 @@ import type { RequestHandler } from './$types'; +const PREFIX = 'This is the complete developer documentation for Svelte and SvelteKit.'; + // Import all markdown files const docs = import.meta.glob<{ default: string }>('../../../content/docs/**/*.md', { eager: true, @@ -14,16 +16,32 @@ function getSectionPriority(path: string): number { return 3; } +function comparePaths(a: string, b: string): number { + // First compare by section + const priorityA = getSectionPriority(a); + const priorityB = getSectionPriority(b); + if (priorityA !== priorityB) return priorityA - priorityB; + + // Get directory paths + const dirA = a.split('/').slice(0, -1).join('/'); + const dirB = b.split('/').slice(0, -1).join('/'); + + // If in the same directory, prioritize index.md + if (dirA === dirB) { + if (a.endsWith('index.md')) return -1; + if (b.endsWith('index.md')) return 1; + return a.localeCompare(b); + } + + // Otherwise sort by directory path + return dirA.localeCompare(dirB); +} + export const GET: RequestHandler = async () => { - let content = ''; + let content = `${PREFIX}\n\n`; // Get all file paths and sort them - const paths = Object.keys(docs).sort((a, b) => { - const priorityA = getSectionPriority(a); - const priorityB = getSectionPriority(b); - if (priorityA !== priorityB) return priorityA - priorityB; - return a.localeCompare(b); - }); + const paths = Object.keys(docs).sort(comparePaths); let currentSection = ''; @@ -31,24 +49,22 @@ export const GET: RequestHandler = async () => { for (const path of paths) { // Determine section let section = ''; - if (path.includes('/docs/svelte/')) section = 'SVELTE DOCS'; - else if (path.includes('/docs/kit/')) section = 'SVELTEKIT DOCS'; - else if (path.includes('/docs/cli/')) section = 'CLI DOCS'; + if (path.includes('/docs/svelte/')) section = 'Svelte documentation'; + else if (path.includes('/docs/kit/')) section = 'SvelteKit documentation'; + else if (path.includes('/docs/cli/')) section = 'Svelte CLI documentation'; else continue; // Skip other content // Add section header if we're entering a new section if (section !== currentSection) { - if (currentSection) content += '\n\n'; - content += '=====================================\n'; - content += `============ ${section} ===========\n`; - content += '=====================================\n\n'; + if (currentSection) content += '\n'; + content += `# Start of ${section}\n\n`; currentSection = section; } // Add file path and content - content += `File: ${path.replace('../../../content/', '')}\n\n`; - content += docs[path].default; // Access the default export to get the actual content - content += '\n\n-------------------\n\n'; + content += `## ${path.replace('../../../content/', '')}\n\n`; + content += docs[path].default; + content += '\n'; } const headers: HeadersInit = { From b74efcac2381865875832fd14651562686df5db7 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 21:57:46 +0100 Subject: [PATCH 04/49] wip --- .../routes/docs/[...path]/llms.txt/+server.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts diff --git a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts new file mode 100644 index 0000000000..444fc81c34 --- /dev/null +++ b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts @@ -0,0 +1,46 @@ +import type { RequestHandler } from './$types'; +// TODO: figure this out +// import { index } from '$lib/server/content'; +import { error, json } from '@sveltejs/kit'; + +const packages = ['svelte', 'kit', 'cli']; + +export const prerender = true; + +function filterDocs(allDocs: Record, type: string) { + const typePathMap = { + svelte: '/docs/svelte/', + kit: '/docs/kit/', + cli: '/docs/cli/' + } as const; + + return Object.entries(allDocs).reduce( + (filtered, [path, content]) => { + if (path.includes(typePathMap[type as keyof typeof typePathMap])) { + filtered[path] = content; + } + return filtered; + }, + {} as Record + ); +} + +export const GET: RequestHandler = async ({ params }) => { + const currentPackage = params.path; + + if (!packages.includes(currentPackage)) { + error(404, { + message: 'Not Found' + }); + } + + const headers: HeadersInit = { + 'Content-Type': 'text/plain; charset=utf-8', + 'Cache-Control': 'public, max-age=3600' + }; + + return new Response('hello', { + status: 200, + headers + }); +}; From 437464f23c0019eb52f30b439df53c0db3713223 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 22:04:33 +0100 Subject: [PATCH 05/49] wip --- .../routes/docs/[...path]/llms.txt/+server.ts | 93 ++++++++++++++----- 1 file changed, 71 insertions(+), 22 deletions(-) diff --git a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts index 444fc81c34..8a2d7cfd00 100644 --- a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts @@ -1,46 +1,95 @@ import type { RequestHandler } from './$types'; -// TODO: figure this out -// import { index } from '$lib/server/content'; -import { error, json } from '@sveltejs/kit'; +import { error } from '@sveltejs/kit'; -const packages = ['svelte', 'kit', 'cli']; +const PREFIX = 'This is the filtered developer documentation for Svelte and SvelteKit.'; -export const prerender = true; +const packages = ['svelte', 'kit', 'cli'] as const; +type Package = (typeof packages)[number]; + +const docs = import.meta.glob('../../../../../content/docs/**/*.md', { + eager: true, + query: '?raw' +}); -function filterDocs(allDocs: Record, type: string) { +function filterDocs(allDocs: Record, type: Package) { const typePathMap = { - svelte: '/docs/svelte/', - kit: '/docs/kit/', - cli: '/docs/cli/' + svelte: 'svelte', + kit: 'kit', + cli: 'cli' } as const; return Object.entries(allDocs).reduce( (filtered, [path, content]) => { - if (path.includes(typePathMap[type as keyof typeof typePathMap])) { + const normalizedPath = path.toLowerCase(); + if (normalizedPath.includes(`/docs/${typePathMap[type]}/`)) { filtered[path] = content; } return filtered; }, - {} as Record + {} as Record ); } +function sortPaths(paths: string[]): string[] { + return paths.sort((a, b) => { + // Get directory paths + const dirA = a.split('/').slice(0, -1).join('/'); + const dirB = b.split('/').slice(0, -1).join('/'); + + // If in the same directory, prioritize index.md + if (dirA === dirB) { + if (a.endsWith('index.md')) return -1; + if (b.endsWith('index.md')) return 1; + return a.localeCompare(b); + } + + // Otherwise sort by directory path + return dirA.localeCompare(dirB); + }); +} + +function generateContent(filteredDocs: Record, type: Package): string { + let content = `${PREFIX}\n\n# ${type} Documentation\n\n`; + + // Get all file paths and sort them + const paths = sortPaths(Object.keys(filteredDocs)); + + // Log for debugging + console.log('Filtered paths:', paths); + + // Process each file + for (const path of paths) { + content += `## ${path.replace('../../../../../content/', '')}\n\n`; + content += filteredDocs[path].default; + content += '\n\n'; + } + + return content; +} + export const GET: RequestHandler = async ({ params }) => { - const currentPackage = params.path; + // Extract the first part of the path (svelte, kit, or cli) + const [packageType] = params.path.split('/'); - if (!packages.includes(currentPackage)) { - error(404, { - message: 'Not Found' - }); + if (!packages.includes(packageType as Package)) { + error(404, 'Not Found'); } - const headers: HeadersInit = { - 'Content-Type': 'text/plain; charset=utf-8', - 'Cache-Control': 'public, max-age=3600' - }; + const filteredDocs = filterDocs(docs, packageType as Package); - return new Response('hello', { + if (Object.keys(filteredDocs).length === 0) { + error(404, 'No documentation found for this package'); + } + + const content = generateContent(filteredDocs, packageType as Package); + + return new Response(content, { status: 200, - headers + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + 'Cache-Control': 'public, max-age=3600' + } }); }; + +export const prerender = true; From 15ed068412308c61ddb3fc7bc8a1c96ddd004c4e Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 22:07:27 +0100 Subject: [PATCH 06/49] Update +server.ts --- .../svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts index 8a2d7cfd00..247a3cfbfb 100644 --- a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts @@ -8,7 +8,8 @@ type Package = (typeof packages)[number]; const docs = import.meta.glob('../../../../../content/docs/**/*.md', { eager: true, - query: '?raw' + query: '?raw', + import: 'default' }); function filterDocs(allDocs: Record, type: Package) { @@ -60,7 +61,7 @@ function generateContent(filteredDocs: Record, type: Package): s // Process each file for (const path of paths) { content += `## ${path.replace('../../../../../content/', '')}\n\n`; - content += filteredDocs[path].default; + content += filteredDocs[path]; content += '\n\n'; } From 98a9f907aac8b037d62817e8ca33b2f987a2da00 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 22:36:20 +0100 Subject: [PATCH 07/49] Update +server.ts --- .../routes/docs/[...path]/llms.txt/+server.ts | 64 +++++++++---------- 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts index 247a3cfbfb..bab5d3846a 100644 --- a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts @@ -1,76 +1,74 @@ import type { RequestHandler } from './$types'; +import type { EntryGenerator } from './$types'; import { error } from '@sveltejs/kit'; -const PREFIX = 'This is the filtered developer documentation for Svelte and SvelteKit.'; - const packages = ['svelte', 'kit', 'cli'] as const; type Package = (typeof packages)[number]; +export const prerender = true; + +export const entries: EntryGenerator = () => { + return packages.map((type) => ({ path: type })); +}; + const docs = import.meta.glob('../../../../../content/docs/**/*.md', { eager: true, query: '?raw', import: 'default' }); -function filterDocs(allDocs: Record, type: Package) { - const typePathMap = { - svelte: 'svelte', - kit: 'kit', - cli: 'cli' - } as const; - - return Object.entries(allDocs).reduce( - (filtered, [path, content]) => { - const normalizedPath = path.toLowerCase(); - if (normalizedPath.includes(`/docs/${typePathMap[type]}/`)) { - filtered[path] = content; - } - return filtered; - }, - {} as Record - ); +function getPrefix(type: Package): string { + const names = { + svelte: 'Svelte', + kit: 'SvelteKit', + cli: 'Svelte CLI' + }; + return `This is the developer documentation for ${names[type]}.`; +} + +function filterDocs(allDocs: Record, type: Package): Record { + const filtered: Record = {}; + + for (const [path, content] of Object.entries(allDocs)) { + if (path.toLowerCase().includes(`/docs/${type}/`)) { + filtered[path] = content; + } + } + + return filtered; } function sortPaths(paths: string[]): string[] { return paths.sort((a, b) => { - // Get directory paths const dirA = a.split('/').slice(0, -1).join('/'); const dirB = b.split('/').slice(0, -1).join('/'); - // If in the same directory, prioritize index.md if (dirA === dirB) { if (a.endsWith('index.md')) return -1; if (b.endsWith('index.md')) return 1; return a.localeCompare(b); } - // Otherwise sort by directory path return dirA.localeCompare(dirB); }); } function generateContent(filteredDocs: Record, type: Package): string { - let content = `${PREFIX}\n\n# ${type} Documentation\n\n`; + let content = `${getPrefix(type)}\n\n`; - // Get all file paths and sort them const paths = sortPaths(Object.keys(filteredDocs)); - // Log for debugging - console.log('Filtered paths:', paths); - - // Process each file for (const path of paths) { - content += `## ${path.replace('../../../../../content/', '')}\n\n`; + content += `# ${path.replace('../../../../../content/', '')}\n\n`; content += filteredDocs[path]; - content += '\n\n'; + content += '\n'; } return content; } export const GET: RequestHandler = async ({ params }) => { - // Extract the first part of the path (svelte, kit, or cli) - const [packageType] = params.path.split('/'); + const packageType = params.path; if (!packages.includes(packageType as Package)) { error(404, 'Not Found'); @@ -92,5 +90,3 @@ export const GET: RequestHandler = async ({ params }) => { } }); }; - -export const prerender = true; From 16046c0264a8916848a8bd9a6760bac693bba825 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 22:41:02 +0100 Subject: [PATCH 08/49] wip --- apps/svelte.dev/src/lib/server/content.ts | 6 ++++++ apps/svelte.dev/src/lib/server/llm-content.ts | 0 .../src/routes/docs/[...path]/llms.txt/+server.ts | 11 +++-------- 3 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 apps/svelte.dev/src/lib/server/llm-content.ts diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index 45f0d4b0b2..d41745dd7e 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -14,6 +14,12 @@ const assets = import.meta.glob('../../../content/**/+assets/**', { import: 'default' }); +export const documentsContent = import.meta.glob('../../../content/**/*.md', { + eager: true, + query: '?raw', + import: 'default' +}); + // https://github.com/vitejs/vite/issues/17453 export const index = await create_index(documents, assets, '../../../content', read); diff --git a/apps/svelte.dev/src/lib/server/llm-content.ts b/apps/svelte.dev/src/lib/server/llm-content.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts index bab5d3846a..2c94a49b0c 100644 --- a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts @@ -1,6 +1,7 @@ import type { RequestHandler } from './$types'; import type { EntryGenerator } from './$types'; import { error } from '@sveltejs/kit'; +import { documentsContent } from '$lib/server/content'; const packages = ['svelte', 'kit', 'cli'] as const; type Package = (typeof packages)[number]; @@ -11,12 +12,6 @@ export const entries: EntryGenerator = () => { return packages.map((type) => ({ path: type })); }; -const docs = import.meta.glob('../../../../../content/docs/**/*.md', { - eager: true, - query: '?raw', - import: 'default' -}); - function getPrefix(type: Package): string { const names = { svelte: 'Svelte', @@ -59,7 +54,7 @@ function generateContent(filteredDocs: Record, type: Package): s const paths = sortPaths(Object.keys(filteredDocs)); for (const path of paths) { - content += `# ${path.replace('../../../../../content/', '')}\n\n`; + content += `# ${path.replace('../../../content/', '')}\n\n`; content += filteredDocs[path]; content += '\n'; } @@ -74,7 +69,7 @@ export const GET: RequestHandler = async ({ params }) => { error(404, 'Not Found'); } - const filteredDocs = filterDocs(docs, packageType as Package); + const filteredDocs = filterDocs(documentsContent, packageType as Package); if (Object.keys(filteredDocs).length === 0) { error(404, 'No documentation found for this package'); From c2534f9acefcd14b4e3033db5e24168a32b09423 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 22:46:48 +0100 Subject: [PATCH 09/49] cleanup --- apps/svelte.dev/src/lib/server/content.ts | 56 ++++++++++++++++ .../routes/docs/[...path]/llms.txt/+server.ts | 65 +++---------------- 2 files changed, 65 insertions(+), 56 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index d41745dd7e..b2dbdc9c2a 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -129,3 +129,59 @@ function create_docs() { export const docs = create_docs(); export const examples = index.examples.children; + +function sortPaths(paths: string[]): string[] { + return paths.sort((a, b) => { + const dirA = a.split('/').slice(0, -1).join('/'); + const dirB = b.split('/').slice(0, -1).join('/'); + + if (dirA === dirB) { + if (a.endsWith('index.md')) return -1; + if (b.endsWith('index.md')) return 1; + return a.localeCompare(b); + } + + return dirA.localeCompare(dirB); + }); +} + +export const packages = ['svelte', 'kit', 'cli'] as const; +export type Package = (typeof packages)[number]; + +function getDocumentationTitle(type: Package): string { + const names = { + svelte: 'Svelte', + kit: 'SvelteKit', + cli: 'Svelte CLI' + }; + return `This is the developer documentation for ${names[type]}.`; +} + +export function filterDocsByPackage( + allDocs: Record, + type: Package +): Record { + const filtered: Record = {}; + + for (const [path, content] of Object.entries(allDocs)) { + if (path.toLowerCase().includes(`/docs/${type}/`)) { + filtered[path] = content; + } + } + + return filtered; +} + +export function generateLlmContent(filteredDocs: Record, type: Package): string { + let content = `${getDocumentationTitle(type)}\n\n`; + + const paths = sortPaths(Object.keys(filteredDocs)); + + for (const path of paths) { + content += `# ${path.replace('../../../content/', '')}\n\n`; + content += filteredDocs[path]; + content += '\n'; + } + + return content; +} diff --git a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts index 2c94a49b0c..137c0f3135 100644 --- a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts @@ -1,10 +1,13 @@ import type { RequestHandler } from './$types'; import type { EntryGenerator } from './$types'; import { error } from '@sveltejs/kit'; -import { documentsContent } from '$lib/server/content'; - -const packages = ['svelte', 'kit', 'cli'] as const; -type Package = (typeof packages)[number]; +import { + documentsContent, + filterDocsByPackage, + generateLlmContent, + packages, + type Package +} from '$lib/server/content'; export const prerender = true; @@ -12,56 +15,6 @@ export const entries: EntryGenerator = () => { return packages.map((type) => ({ path: type })); }; -function getPrefix(type: Package): string { - const names = { - svelte: 'Svelte', - kit: 'SvelteKit', - cli: 'Svelte CLI' - }; - return `This is the developer documentation for ${names[type]}.`; -} - -function filterDocs(allDocs: Record, type: Package): Record { - const filtered: Record = {}; - - for (const [path, content] of Object.entries(allDocs)) { - if (path.toLowerCase().includes(`/docs/${type}/`)) { - filtered[path] = content; - } - } - - return filtered; -} - -function sortPaths(paths: string[]): string[] { - return paths.sort((a, b) => { - const dirA = a.split('/').slice(0, -1).join('/'); - const dirB = b.split('/').slice(0, -1).join('/'); - - if (dirA === dirB) { - if (a.endsWith('index.md')) return -1; - if (b.endsWith('index.md')) return 1; - return a.localeCompare(b); - } - - return dirA.localeCompare(dirB); - }); -} - -function generateContent(filteredDocs: Record, type: Package): string { - let content = `${getPrefix(type)}\n\n`; - - const paths = sortPaths(Object.keys(filteredDocs)); - - for (const path of paths) { - content += `# ${path.replace('../../../content/', '')}\n\n`; - content += filteredDocs[path]; - content += '\n'; - } - - return content; -} - export const GET: RequestHandler = async ({ params }) => { const packageType = params.path; @@ -69,13 +22,13 @@ export const GET: RequestHandler = async ({ params }) => { error(404, 'Not Found'); } - const filteredDocs = filterDocs(documentsContent, packageType as Package); + const filteredDocs = filterDocsByPackage(documentsContent, packageType as Package); if (Object.keys(filteredDocs).length === 0) { error(404, 'No documentation found for this package'); } - const content = generateContent(filteredDocs, packageType as Package); + const content = generateLlmContent(filteredDocs, packageType as Package); return new Response(content, { status: 200, From 0d295843868bd76bbb058b01bf085f747176e786 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 22:48:58 +0100 Subject: [PATCH 10/49] wip --- apps/svelte.dev/src/lib/server/llm-content.ts | 0 apps/svelte.dev/src/routes/llms.txt/+server.ts | 13 ++++--------- 2 files changed, 4 insertions(+), 9 deletions(-) delete mode 100644 apps/svelte.dev/src/lib/server/llm-content.ts diff --git a/apps/svelte.dev/src/lib/server/llm-content.ts b/apps/svelte.dev/src/lib/server/llm-content.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 89289a9bc3..a759d6e9ea 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,12 +1,7 @@ import type { RequestHandler } from './$types'; +import { documentsContent } from '$lib/server/content'; -const PREFIX = 'This is the complete developer documentation for Svelte and SvelteKit.'; - -// Import all markdown files -const docs = import.meta.glob<{ default: string }>('../../../content/docs/**/*.md', { - eager: true, - query: '?raw' -}); +const PREFIX = 'This is the abridged developer documentation for Svelte and SvelteKit.'; // Sort function to ensure correct order (svelte -> kit -> cli) function getSectionPriority(path: string): number { @@ -41,7 +36,7 @@ export const GET: RequestHandler = async () => { let content = `${PREFIX}\n\n`; // Get all file paths and sort them - const paths = Object.keys(docs).sort(comparePaths); + const paths = Object.keys(documentsContent).sort(comparePaths); let currentSection = ''; @@ -63,7 +58,7 @@ export const GET: RequestHandler = async () => { // Add file path and content content += `## ${path.replace('../../../content/', '')}\n\n`; - content += docs[path].default; + content += documentsContent[path]; content += '\n'; } From 6d05a5a8543270796d37e6b5adc1a7ace0ee7b91 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 22:52:24 +0100 Subject: [PATCH 11/49] refactor --- apps/svelte.dev/src/lib/server/content.ts | 17 +++++++++- .../svelte.dev/src/routes/llms.txt/+server.ts | 33 ++----------------- 2 files changed, 18 insertions(+), 32 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index b2dbdc9c2a..797f038f8d 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -130,17 +130,32 @@ export const docs = create_docs(); export const examples = index.examples.children; -function sortPaths(paths: string[]): string[] { +function getSectionPriority(path: string): number { + if (path.includes('/docs/svelte/')) return 0; + if (path.includes('/docs/kit/')) return 1; + if (path.includes('/docs/cli/')) return 2; + return 3; +} + +export function sortPaths(paths: string[]): string[] { return paths.sort((a, b) => { + // First compare by section priority + const priorityA = getSectionPriority(a); + const priorityB = getSectionPriority(b); + if (priorityA !== priorityB) return priorityA - priorityB; + + // Get directory paths const dirA = a.split('/').slice(0, -1).join('/'); const dirB = b.split('/').slice(0, -1).join('/'); + // If in the same directory, prioritize index.md if (dirA === dirB) { if (a.endsWith('index.md')) return -1; if (b.endsWith('index.md')) return 1; return a.localeCompare(b); } + // Otherwise sort by directory path return dirA.localeCompare(dirB); }); } diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index a759d6e9ea..630061d38c 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,42 +1,13 @@ import type { RequestHandler } from './$types'; -import { documentsContent } from '$lib/server/content'; +import { documentsContent, sortPaths } from '$lib/server/content'; const PREFIX = 'This is the abridged developer documentation for Svelte and SvelteKit.'; -// Sort function to ensure correct order (svelte -> kit -> cli) -function getSectionPriority(path: string): number { - if (path.includes('/docs/svelte/')) return 0; - if (path.includes('/docs/kit/')) return 1; - if (path.includes('/docs/cli/')) return 2; - return 3; -} - -function comparePaths(a: string, b: string): number { - // First compare by section - const priorityA = getSectionPriority(a); - const priorityB = getSectionPriority(b); - if (priorityA !== priorityB) return priorityA - priorityB; - - // Get directory paths - const dirA = a.split('/').slice(0, -1).join('/'); - const dirB = b.split('/').slice(0, -1).join('/'); - - // If in the same directory, prioritize index.md - if (dirA === dirB) { - if (a.endsWith('index.md')) return -1; - if (b.endsWith('index.md')) return 1; - return a.localeCompare(b); - } - - // Otherwise sort by directory path - return dirA.localeCompare(dirB); -} - export const GET: RequestHandler = async () => { let content = `${PREFIX}\n\n`; // Get all file paths and sort them - const paths = Object.keys(documentsContent).sort(comparePaths); + const paths = sortPaths(Object.keys(documentsContent)); let currentSection = ''; From 464630fcb49a09b85a72b8b2a8924c77fa7c8727 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 22:58:03 +0100 Subject: [PATCH 12/49] cleanupo --- apps/svelte.dev/src/lib/server/content.ts | 19 ++++++++++------ .../svelte.dev/src/routes/llms.txt/+server.ts | 22 +++++++++---------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index 797f038f8d..765ccd3479 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -163,13 +163,18 @@ export function sortPaths(paths: string[]): string[] { export const packages = ['svelte', 'kit', 'cli'] as const; export type Package = (typeof packages)[number]; -function getDocumentationTitle(type: Package): string { - const names = { - svelte: 'Svelte', - kit: 'SvelteKit', - cli: 'Svelte CLI' - }; - return `This is the developer documentation for ${names[type]}.`; +const documentationNames = { + svelte: 'Svelte', + kit: 'SvelteKit', + cli: 'Svelte CLI' +}; + +export function getDocumentationTitle(type: Package): string { + return `This is the developer documentation for ${documentationNames[type]}.`; +} + +export function getDocumentationStartTitle(type: Package): string { + return `# Start of ${documentationNames[type]} documentation`; } export function filterDocsByPackage( diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 630061d38c..6603a7cb9a 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,33 +1,33 @@ import type { RequestHandler } from './$types'; -import { documentsContent, sortPaths } from '$lib/server/content'; +import { + documentsContent, + getDocumentationStartTitle, + getDocumentationTitle, + sortPaths +} from '$lib/server/content'; const PREFIX = 'This is the abridged developer documentation for Svelte and SvelteKit.'; export const GET: RequestHandler = async () => { let content = `${PREFIX}\n\n`; - // Get all file paths and sort them const paths = sortPaths(Object.keys(documentsContent)); let currentSection = ''; - // Process each file for (const path of paths) { - // Determine section let section = ''; - if (path.includes('/docs/svelte/')) section = 'Svelte documentation'; - else if (path.includes('/docs/kit/')) section = 'SvelteKit documentation'; - else if (path.includes('/docs/cli/')) section = 'Svelte CLI documentation'; - else continue; // Skip other content + if (path.includes('/docs/svelte/')) section = `${getDocumentationStartTitle('svelte')}`; + else if (path.includes('/docs/kit/')) section = `${getDocumentationStartTitle('kit')}`; + else if (path.includes('/docs/cli/')) section = `${getDocumentationStartTitle('cli')}`; + else continue; - // Add section header if we're entering a new section if (section !== currentSection) { if (currentSection) content += '\n'; - content += `# Start of ${section}\n\n`; + content += `${section}\n\n`; currentSection = section; } - // Add file path and content content += `## ${path.replace('../../../content/', '')}\n\n`; content += documentsContent[path]; content += '\n'; From 42388876aefc4bec6c22c386c81bb53fe2bd7f88 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 22:59:37 +0100 Subject: [PATCH 13/49] Update +server.ts --- apps/svelte.dev/src/routes/llms.txt/+server.ts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 6603a7cb9a..249b65f484 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,10 +1,5 @@ import type { RequestHandler } from './$types'; -import { - documentsContent, - getDocumentationStartTitle, - getDocumentationTitle, - sortPaths -} from '$lib/server/content'; +import { documentsContent, getDocumentationStartTitle, sortPaths } from '$lib/server/content'; const PREFIX = 'This is the abridged developer documentation for Svelte and SvelteKit.'; @@ -17,9 +12,9 @@ export const GET: RequestHandler = async () => { for (const path of paths) { let section = ''; - if (path.includes('/docs/svelte/')) section = `${getDocumentationStartTitle('svelte')}`; - else if (path.includes('/docs/kit/')) section = `${getDocumentationStartTitle('kit')}`; - else if (path.includes('/docs/cli/')) section = `${getDocumentationStartTitle('cli')}`; + if (path.includes('/docs/svelte/')) section = getDocumentationStartTitle('svelte'); + else if (path.includes('/docs/kit/')) section = getDocumentationStartTitle('kit'); + else if (path.includes('/docs/cli/')) section = getDocumentationStartTitle('cli'); else continue; if (section !== currentSection) { From d4943341e1138813c8cc9ffd93232b137073e6c9 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 23:01:00 +0100 Subject: [PATCH 14/49] Update content.ts --- apps/svelte.dev/src/lib/server/content.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index 765ccd3479..828b8d1cf1 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -163,18 +163,18 @@ export function sortPaths(paths: string[]): string[] { export const packages = ['svelte', 'kit', 'cli'] as const; export type Package = (typeof packages)[number]; -const documentationNames = { +const DOCUMENTATION_NAMES: Record = { svelte: 'Svelte', kit: 'SvelteKit', cli: 'Svelte CLI' }; export function getDocumentationTitle(type: Package): string { - return `This is the developer documentation for ${documentationNames[type]}.`; + return `This is the developer documentation for ${DOCUMENTATION_NAMES[type]}.`; } export function getDocumentationStartTitle(type: Package): string { - return `# Start of ${documentationNames[type]} documentation`; + return `# Start of ${DOCUMENTATION_NAMES[type]} documentation`; } export function filterDocsByPackage( From 4315f7ab2ee6c0becbe0df14a5724de397843e7a Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 23:02:49 +0100 Subject: [PATCH 15/49] wip --- apps/svelte.dev/src/lib/server/content.ts | 24 +++++++++++++ .../svelte.dev/src/routes/llms.txt/+server.ts | 36 ++++--------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index 828b8d1cf1..07e03fc7cb 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -205,3 +205,27 @@ export function generateLlmContent(filteredDocs: Record, type: P return content; } + +export function generateCombinedContent(documentsContent: Record): string { + let content = ''; + let currentSection = ''; + const paths = sortPaths(Object.keys(documentsContent)); + + for (const path of paths) { + const docType = packages.find((pkg) => path.includes(`/docs/${pkg}/`)); + if (!docType) continue; + + const section = getDocumentationStartTitle(docType); + if (section !== currentSection) { + if (currentSection) content += '\n'; + content += `${section}\n\n`; + currentSection = section; + } + + content += `## ${path.replace('../../../content/', '')}\n\n`; + content += documentsContent[path]; + content += '\n'; + } + + return content; +} diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 249b65f484..8e6cbef44d 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,41 +1,17 @@ import type { RequestHandler } from './$types'; -import { documentsContent, getDocumentationStartTitle, sortPaths } from '$lib/server/content'; +import { documentsContent, generateCombinedContent } from '$lib/server/content'; const PREFIX = 'This is the abridged developer documentation for Svelte and SvelteKit.'; export const GET: RequestHandler = async () => { - let content = `${PREFIX}\n\n`; - - const paths = sortPaths(Object.keys(documentsContent)); - - let currentSection = ''; - - for (const path of paths) { - let section = ''; - if (path.includes('/docs/svelte/')) section = getDocumentationStartTitle('svelte'); - else if (path.includes('/docs/kit/')) section = getDocumentationStartTitle('kit'); - else if (path.includes('/docs/cli/')) section = getDocumentationStartTitle('cli'); - else continue; - - if (section !== currentSection) { - if (currentSection) content += '\n'; - content += `${section}\n\n`; - currentSection = section; - } - - content += `## ${path.replace('../../../content/', '')}\n\n`; - content += documentsContent[path]; - content += '\n'; - } - - const headers: HeadersInit = { - 'Content-Type': 'text/plain; charset=utf-8', - 'Cache-Control': 'public, max-age=3600' - }; + const content = `${PREFIX}\n\n${generateCombinedContent(documentsContent)}`; return new Response(content, { status: 200, - headers + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + 'Cache-Control': 'public, max-age=3600' + } }); }; From 7bb3ebd0b846e4d034cbf30b606c84715c19bfce Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 23:05:38 +0100 Subject: [PATCH 16/49] Create +server.ts --- .../src/routes/llms-full.txt/+server.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 apps/svelte.dev/src/routes/llms-full.txt/+server.ts diff --git a/apps/svelte.dev/src/routes/llms-full.txt/+server.ts b/apps/svelte.dev/src/routes/llms-full.txt/+server.ts new file mode 100644 index 0000000000..fca5eb6168 --- /dev/null +++ b/apps/svelte.dev/src/routes/llms-full.txt/+server.ts @@ -0,0 +1,18 @@ +import type { RequestHandler } from './$types'; +import { documentsContent, generateCombinedContent } from '$lib/server/content'; + +const PREFIX = 'This is the full developer documentation for Svelte and SvelteKit.'; + +export const GET: RequestHandler = async () => { + const content = `${PREFIX}\n\n${generateCombinedContent(documentsContent)}`; + + return new Response(content, { + status: 200, + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + 'Cache-Control': 'public, max-age=3600' + } + }); +}; + +export const prerender = true; From 8612605bf65a4ff0c3e69709c7a9016d99d0fde3 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 23:39:07 +0100 Subject: [PATCH 17/49] minimize llms.txt --- apps/svelte.dev/src/lib/server/content.ts | 101 +++++++++++++++++- .../svelte.dev/src/routes/llms.txt/+server.ts | 9 +- 2 files changed, 105 insertions(+), 5 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index 07e03fc7cb..a9c155fe5a 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -192,21 +192,111 @@ export function filterDocsByPackage( return filtered; } -export function generateLlmContent(filteredDocs: Record, type: Package): string { +interface MinimizeOptions { + removeLegacy: boolean; + removeNoteBlocks: boolean; + removeDetailsBlocks: boolean; + removePlaygroundLinks: boolean; + removePrettierIgnore: boolean; + normalizeWhitespace: boolean; +} + +const defaultOptions: MinimizeOptions = { + removeLegacy: false, + removeNoteBlocks: false, + removeDetailsBlocks: false, + removePlaygroundLinks: false, + removePrettierIgnore: false, + normalizeWhitespace: false +}; + +function removeQuoteBlocks(content: string, blockType: string): string { + return content + .split('\n') + .reduce((acc: string[], line: string, index: number, lines: string[]) => { + // If we find a block (with or without additional text), skip it and all subsequent blockquote lines + if (line.trim().startsWith(`> [!${blockType}]`)) { + // Skip all subsequent lines that are part of the blockquote + let i = index; + while (i < lines.length && (lines[i].startsWith('>') || lines[i].trim() === '')) { + i++; + } + // Update the index to skip all these lines + index = i - 1; + return acc; + } + + // Only add the line if it's not being skipped + acc.push(line); + return acc; + }, []) + .join('\n'); +} + +function minimizeContent(content: string, options?: Partial): string { + // Merge with defaults, but only for properties that are defined + const settings: MinimizeOptions = options ? { ...defaultOptions, ...options } : defaultOptions; + + let minimized = content; + + if (settings.removeLegacy) { + minimized = removeQuoteBlocks(minimized, 'LEGACY'); + } + + if (settings.removeNoteBlocks) { + minimized = removeQuoteBlocks(minimized, 'NOTE'); + } + + if (settings.removeDetailsBlocks) { + minimized = removeQuoteBlocks(minimized, 'DETAILS'); + } + + if (settings.removePlaygroundLinks) { + // Replace playground URLs with /[link] but keep the original link text + minimized = minimized.replace(/\[([^\]]+)\]\(\/playground[^)]+\)/g, '[$1](/REMOVED)'); + } + + if (settings.removePrettierIgnore) { + minimized = minimized + .split('\n') + .filter((line) => line.trim() !== '') + .join('\n'); + } + + if (settings.normalizeWhitespace) { + minimized = minimized.replace(/\s+/g, ' '); + } + + minimized = minimized.trim(); + + return minimized; +} + +export function generateLlmContent( + filteredDocs: Record, + type: Package, + minimizeOptions?: Partial +): string { let content = `${getDocumentationTitle(type)}\n\n`; const paths = sortPaths(Object.keys(filteredDocs)); for (const path of paths) { content += `# ${path.replace('../../../content/', '')}\n\n`; - content += filteredDocs[path]; + const docContent = minimizeOptions + ? minimizeContent(filteredDocs[path], minimizeOptions) + : filteredDocs[path]; + content += docContent; content += '\n'; } return content; } -export function generateCombinedContent(documentsContent: Record): string { +export function generateCombinedContent( + documentsContent: Record, + minimizeOptions?: Partial +): string { let content = ''; let currentSection = ''; const paths = sortPaths(Object.keys(documentsContent)); @@ -223,7 +313,10 @@ export function generateCombinedContent(documentsContent: Record } content += `## ${path.replace('../../../content/', '')}\n\n`; - content += documentsContent[path]; + const docContent = minimizeOptions + ? minimizeContent(documentsContent[path], minimizeOptions) + : documentsContent[path]; + content += docContent; content += '\n'; } diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 8e6cbef44d..08c3db8661 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -4,7 +4,14 @@ import { documentsContent, generateCombinedContent } from '$lib/server/content'; const PREFIX = 'This is the abridged developer documentation for Svelte and SvelteKit.'; export const GET: RequestHandler = async () => { - const content = `${PREFIX}\n\n${generateCombinedContent(documentsContent)}`; + const content = `${PREFIX}\n\n${generateCombinedContent(documentsContent, { + removeLegacy: true, + removeNoteBlocks: true, + removeDetailsBlocks: true, + removePlaygroundLinks: true, + removePrettierIgnore: true, + normalizeWhitespace: true + })}`; return new Response(content, { status: 200, From 8b2544ce9131c7067dbaeb2cb6a3144eb6e50d9d Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Tue, 10 Dec 2024 23:52:50 +0100 Subject: [PATCH 18/49] Filter llms.txt --- apps/svelte.dev/package.json | 1 + apps/svelte.dev/src/lib/server/content.ts | 16 ++++++++ .../svelte.dev/src/routes/llms.txt/+server.ts | 41 +++++++++++++++---- pnpm-lock.yaml | 11 +++++ 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/apps/svelte.dev/package.json b/apps/svelte.dev/package.json index 54a05f92d9..d08543e290 100644 --- a/apps/svelte.dev/package.json +++ b/apps/svelte.dev/package.json @@ -68,6 +68,7 @@ "lightningcss": "^1.25.1", "magic-string": "^0.30.11", "marked": "^14.1.2", + "minimatch": "^10.0.1", "prettier": "^3.3.2", "prettier-plugin-svelte": "^3.2.4", "satori": "^0.10.13", diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index a9c155fe5a..d99520c481 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -1,6 +1,8 @@ +import { dev } from '$app/environment'; import { read } from '$app/server'; import type { Document } from '@sveltejs/site-kit'; import { create_index } from '@sveltejs/site-kit/server/content'; +import { minimatch } from 'minimatch'; const documents = import.meta.glob('../../../content/**/*.md', { eager: true, @@ -272,6 +274,16 @@ function minimizeContent(content: string, options?: Partial): s return minimized; } +function shouldIncludeFile(filename: string, ignore: string[] = []): boolean { + const shouldIgnore = ignore.some((pattern) => minimatch(filename, pattern)); + if (shouldIgnore) { + if (dev) console.log(`❌ Ignored by pattern: ${filename}`); + return false; + } + + return true; +} + export function generateLlmContent( filteredDocs: Record, type: Package, @@ -295,6 +307,7 @@ export function generateLlmContent( export function generateCombinedContent( documentsContent: Record, + ignore: string[] = [], minimizeOptions?: Partial ): string { let content = ''; @@ -302,6 +315,9 @@ export function generateCombinedContent( const paths = sortPaths(Object.keys(documentsContent)); for (const path of paths) { + // Skip files that match ignore patterns + if (!shouldIncludeFile(path, ignore)) continue; + const docType = packages.find((pkg) => path.includes(`/docs/${pkg}/`)); if (!docType) continue; diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 08c3db8661..05b344a870 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -4,14 +4,39 @@ import { documentsContent, generateCombinedContent } from '$lib/server/content'; const PREFIX = 'This is the abridged developer documentation for Svelte and SvelteKit.'; export const GET: RequestHandler = async () => { - const content = `${PREFIX}\n\n${generateCombinedContent(documentsContent, { - removeLegacy: true, - removeNoteBlocks: true, - removeDetailsBlocks: true, - removePlaygroundLinks: true, - removePrettierIgnore: true, - normalizeWhitespace: true - })}`; + const content = `${PREFIX}\n\n${generateCombinedContent( + documentsContent, + [ + // Svelte ignores + '../../../content/docs/svelte/07-misc/04-custom-elements.md', + '../../../content/docs/svelte/07-misc/06-v4-migration-guide.md', + '../../../content/docs/svelte/07-misc/07-v5-migration-guide.md', + '../../../content/docs/svelte/07-misc/99-faq.md', + '../../../content/docs/svelte/07-misc/xx-reactivity-indepth.md', + '../../../content/docs/svelte/98-reference/21-svelte-legacy.md', + '../../../content/docs/svelte/99-legacy/**/*.md', + '../../../content/docs/svelte/98-reference/30-runtime-errors.md', + '../../../content/docs/svelte/98-reference/30-runtime-warnings.md', + '../../../content/docs/svelte/98-reference/30-compiler-errors.md', + '../../../content/docs/svelte/98-reference/30-compiler-warnings.md', + '**/xx-*.md', + + // SvelteKit ignores + '../../../content/docs/kit/25-build-and-deploy/*adapter-*.md', + '../../../content/docs/kit/25-build-and-deploy/99-writing-adapters.md', + '../../../content/docs/kit/30-advanced/70-packaging.md', + '../../../content/docs/kit/40-best-practices/05-performance.md', + '../../../content/docs/kit/60-appendix/**/*.md' + ], + { + removeLegacy: true, + removeNoteBlocks: true, + removeDetailsBlocks: true, + removePlaygroundLinks: true, + removePrettierIgnore: true, + normalizeWhitespace: true + } + )}`; return new Response(content, { status: 200, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 63b89c9313..6745e92bd7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -162,6 +162,9 @@ importers: marked: specifier: ^14.1.2 version: 14.1.2 + minimatch: + specifier: ^10.0.1 + version: 10.0.1 prettier: specifier: ^3.3.2 version: 3.3.2 @@ -2452,6 +2455,10 @@ packages: resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} engines: {node: '>=4'} + minimatch@10.0.1: + resolution: {integrity: sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==} + engines: {node: 20 || >=22} + minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} @@ -5358,6 +5365,10 @@ snapshots: min-indent@1.0.1: {} + minimatch@10.0.1: + dependencies: + brace-expansion: 2.0.1 + minimatch@3.1.2: dependencies: brace-expansion: 1.1.11 From 02dbf8d7dd1fe64c84cd1ee8b00099031d8a512f Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Wed, 11 Dec 2024 00:16:07 +0100 Subject: [PATCH 19/49] clean up --- apps/svelte.dev/src/lib/server/content.ts | 66 +++++++++---------- .../routes/docs/[...path]/llms.txt/+server.ts | 6 +- .../src/routes/llms-full.txt/+server.ts | 7 +- .../svelte.dev/src/routes/llms.txt/+server.ts | 14 ++-- 4 files changed, 45 insertions(+), 48 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index d99520c481..74b2632daf 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -284,54 +284,48 @@ function shouldIncludeFile(filename: string, ignore: string[] = []): boolean { return true; } -export function generateLlmContent( - filteredDocs: Record, - type: Package, - minimizeOptions?: Partial -): string { - let content = `${getDocumentationTitle(type)}\n\n`; - - const paths = sortPaths(Object.keys(filteredDocs)); - - for (const path of paths) { - content += `# ${path.replace('../../../content/', '')}\n\n`; - const docContent = minimizeOptions - ? minimizeContent(filteredDocs[path], minimizeOptions) - : filteredDocs[path]; - content += docContent; - content += '\n'; - } - - return content; +interface GenerateContentOptions { + prefix?: string; + ignore?: string[]; + minimize?: Partial; + package?: Package; } -export function generateCombinedContent( - documentsContent: Record, - ignore: string[] = [], - minimizeOptions?: Partial +export function generateContent( + docs: Record, + options: GenerateContentOptions = {} ): string { + const { prefix, ignore = [], minimize: minimizeOptions, package: pkg } = options; + let content = ''; + if (prefix) { + content = `${prefix}\n\n`; + } + let currentSection = ''; - const paths = sortPaths(Object.keys(documentsContent)); + const paths = sortPaths(Object.keys(docs)); for (const path of paths) { - // Skip files that match ignore patterns if (!shouldIncludeFile(path, ignore)) continue; - const docType = packages.find((pkg) => path.includes(`/docs/${pkg}/`)); - if (!docType) continue; - - const section = getDocumentationStartTitle(docType); - if (section !== currentSection) { - if (currentSection) content += '\n'; - content += `${section}\n\n`; - currentSection = section; + // If a specific package is provided, only include its docs + if (pkg) { + if (!path.includes(`/docs/${pkg}/`)) continue; + } else { + // For combined content, only include paths that match any package + const docType = packages.find((p) => path.includes(`/docs/${p}/`)); + if (!docType) continue; + + const section = getDocumentationStartTitle(docType); + if (section !== currentSection) { + if (currentSection) content += '\n'; + content += `${section}\n\n`; + currentSection = section; + } } content += `## ${path.replace('../../../content/', '')}\n\n`; - const docContent = minimizeOptions - ? minimizeContent(documentsContent[path], minimizeOptions) - : documentsContent[path]; + const docContent = minimizeOptions ? minimizeContent(docs[path], minimizeOptions) : docs[path]; content += docContent; content += '\n'; } diff --git a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts index 137c0f3135..d1c667e43b 100644 --- a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts @@ -4,7 +4,8 @@ import { error } from '@sveltejs/kit'; import { documentsContent, filterDocsByPackage, - generateLlmContent, + generateContent, + getDocumentationTitle, packages, type Package } from '$lib/server/content'; @@ -28,7 +29,8 @@ export const GET: RequestHandler = async ({ params }) => { error(404, 'No documentation found for this package'); } - const content = generateLlmContent(filteredDocs, packageType as Package); + const PREFIX = `${getDocumentationTitle(packageType)}`; + const content = `${PREFIX}\n\n${generateContent(filteredDocs)}`; return new Response(content, { status: 200, diff --git a/apps/svelte.dev/src/routes/llms-full.txt/+server.ts b/apps/svelte.dev/src/routes/llms-full.txt/+server.ts index fca5eb6168..aa6d4d5e3d 100644 --- a/apps/svelte.dev/src/routes/llms-full.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms-full.txt/+server.ts @@ -1,10 +1,11 @@ import type { RequestHandler } from './$types'; -import { documentsContent, generateCombinedContent } from '$lib/server/content'; +import { documentsContent, generateContent } from '$lib/server/content'; -const PREFIX = 'This is the full developer documentation for Svelte and SvelteKit.'; +const PREFIX = + 'This is the full developer documentation for Svelte and SvelteKit.'; export const GET: RequestHandler = async () => { - const content = `${PREFIX}\n\n${generateCombinedContent(documentsContent)}`; + const content = `${PREFIX}\n\n${generateContent(documentsContent)}`; return new Response(content, { status: 200, diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 05b344a870..0c04159b3d 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,12 +1,12 @@ import type { RequestHandler } from './$types'; -import { documentsContent, generateCombinedContent } from '$lib/server/content'; +import { documentsContent, generateContent } from '$lib/server/content'; -const PREFIX = 'This is the abridged developer documentation for Svelte and SvelteKit.'; +const PREFIX = + 'This is the abridged developer documentation for Svelte and SvelteKit.'; export const GET: RequestHandler = async () => { - const content = `${PREFIX}\n\n${generateCombinedContent( - documentsContent, - [ + const content = `${PREFIX}\n\n${generateContent(documentsContent, { + ignore: [ // Svelte ignores '../../../content/docs/svelte/07-misc/04-custom-elements.md', '../../../content/docs/svelte/07-misc/06-v4-migration-guide.md', @@ -28,7 +28,7 @@ export const GET: RequestHandler = async () => { '../../../content/docs/kit/40-best-practices/05-performance.md', '../../../content/docs/kit/60-appendix/**/*.md' ], - { + minimize: { removeLegacy: true, removeNoteBlocks: true, removeDetailsBlocks: true, @@ -36,7 +36,7 @@ export const GET: RequestHandler = async () => { removePrettierIgnore: true, normalizeWhitespace: true } - )}`; + })}`; return new Response(content, { status: 200, From 7d894031471a80b13855895e509dbb20e63b869b Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Wed, 11 Dec 2024 20:24:51 +0100 Subject: [PATCH 20/49] package --- pnpm-lock.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 92b888e62c..7e5b02febd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2582,6 +2582,10 @@ packages: engines: {node: '>=10.0.0'} hasBin: true + minimatch@10.0.1: + resolution: {integrity: sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==} + engines: {node: 20 || >=22} + minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} @@ -5643,6 +5647,10 @@ snapshots: mime@3.0.0: {} + minimatch@10.0.1: + dependencies: + brace-expansion: 2.0.1 + minimatch@3.1.2: dependencies: brace-expansion: 1.1.11 From 918b6277f98a9b18b7e1a03475f54d3833f93cf1 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Wed, 11 Dec 2024 20:29:35 +0100 Subject: [PATCH 21/49] chore: naming --- apps/svelte.dev/src/lib/server/content.ts | 20 +++++++++---------- .../routes/docs/[...path]/llms.txt/+server.ts | 4 ++-- .../src/routes/llms-full.txt/+server.ts | 4 ++-- .../svelte.dev/src/routes/llms.txt/+server.ts | 4 ++-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index 74b2632daf..08d03107c2 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -132,18 +132,18 @@ export const docs = create_docs(); export const examples = index.examples.children; -function getSectionPriority(path: string): number { +function getDocumentationSectionPriority(path: string): number { if (path.includes('/docs/svelte/')) return 0; if (path.includes('/docs/kit/')) return 1; if (path.includes('/docs/cli/')) return 2; return 3; } -export function sortPaths(paths: string[]): string[] { +export function sortDocumentationPaths(paths: string[]): string[] { return paths.sort((a, b) => { // First compare by section priority - const priorityA = getSectionPriority(a); - const priorityB = getSectionPriority(b); + const priorityA = getDocumentationSectionPriority(a); + const priorityB = getDocumentationSectionPriority(b); if (priorityA !== priorityB) return priorityA - priorityB; // Get directory paths @@ -274,7 +274,7 @@ function minimizeContent(content: string, options?: Partial): s return minimized; } -function shouldIncludeFile(filename: string, ignore: string[] = []): boolean { +function shouldIncludeFileLlmDocs(filename: string, ignore: string[] = []): boolean { const shouldIgnore = ignore.some((pattern) => minimatch(filename, pattern)); if (shouldIgnore) { if (dev) console.log(`❌ Ignored by pattern: ${filename}`); @@ -284,16 +284,16 @@ function shouldIncludeFile(filename: string, ignore: string[] = []): boolean { return true; } -interface GenerateContentOptions { +interface GenerateLlmContentOptions { prefix?: string; ignore?: string[]; minimize?: Partial; package?: Package; } -export function generateContent( +export function generateLlmContent( docs: Record, - options: GenerateContentOptions = {} + options: GenerateLlmContentOptions = {} ): string { const { prefix, ignore = [], minimize: minimizeOptions, package: pkg } = options; @@ -303,10 +303,10 @@ export function generateContent( } let currentSection = ''; - const paths = sortPaths(Object.keys(docs)); + const paths = sortDocumentationPaths(Object.keys(docs)); for (const path of paths) { - if (!shouldIncludeFile(path, ignore)) continue; + if (!shouldIncludeFileLlmDocs(path, ignore)) continue; // If a specific package is provided, only include its docs if (pkg) { diff --git a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts index d1c667e43b..9a1b1447df 100644 --- a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts @@ -4,7 +4,7 @@ import { error } from '@sveltejs/kit'; import { documentsContent, filterDocsByPackage, - generateContent, + generateLlmContent, getDocumentationTitle, packages, type Package @@ -30,7 +30,7 @@ export const GET: RequestHandler = async ({ params }) => { } const PREFIX = `${getDocumentationTitle(packageType)}`; - const content = `${PREFIX}\n\n${generateContent(filteredDocs)}`; + const content = `${PREFIX}\n\n${generateLlmContent(filteredDocs)}`; return new Response(content, { status: 200, diff --git a/apps/svelte.dev/src/routes/llms-full.txt/+server.ts b/apps/svelte.dev/src/routes/llms-full.txt/+server.ts index aa6d4d5e3d..510b9442df 100644 --- a/apps/svelte.dev/src/routes/llms-full.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms-full.txt/+server.ts @@ -1,11 +1,11 @@ import type { RequestHandler } from './$types'; -import { documentsContent, generateContent } from '$lib/server/content'; +import { documentsContent, generateLlmContent } from '$lib/server/content'; const PREFIX = 'This is the full developer documentation for Svelte and SvelteKit.'; export const GET: RequestHandler = async () => { - const content = `${PREFIX}\n\n${generateContent(documentsContent)}`; + const content = `${PREFIX}\n\n${generateLlmContent(documentsContent)}`; return new Response(content, { status: 200, diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 0c04159b3d..75cad6de7d 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,11 +1,11 @@ import type { RequestHandler } from './$types'; -import { documentsContent, generateContent } from '$lib/server/content'; +import { documentsContent, generateLlmContent } from '$lib/server/content'; const PREFIX = 'This is the abridged developer documentation for Svelte and SvelteKit.'; export const GET: RequestHandler = async () => { - const content = `${PREFIX}\n\n${generateContent(documentsContent, { + const content = `${PREFIX}\n\n${generateLlmContent(documentsContent, { ignore: [ // Svelte ignores '../../../content/docs/svelte/07-misc/04-custom-elements.md', From b327991455edd4010499d97941677733d4a73704 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Wed, 11 Dec 2024 20:59:31 +0100 Subject: [PATCH 22/49] Update +server.ts --- apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts index 9a1b1447df..4fef4b2f2d 100644 --- a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts @@ -29,7 +29,7 @@ export const GET: RequestHandler = async ({ params }) => { error(404, 'No documentation found for this package'); } - const PREFIX = `${getDocumentationTitle(packageType)}`; + const PREFIX = `${getDocumentationTitle(packageType as Package)}`; const content = `${PREFIX}\n\n${generateLlmContent(filteredDocs)}`; return new Response(content, { From e51cf30ae47e12cba3fb073df15eb104369e223e Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Wed, 11 Dec 2024 21:06:29 +0100 Subject: [PATCH 23/49] Dynamic path names --- apps/svelte.dev/src/lib/server/content.ts | 70 ++++++++++++----------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index 08d03107c2..ffba77597e 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -130,39 +130,15 @@ function create_docs() { export const docs = create_docs(); -export const examples = index.examples.children; +// Automatically determine packages from the docs directory structure +export const packages = Array.from( + new Set( + Object.keys(docs.topics) + .map((topic) => topic.split('/')[1]) + .filter(Boolean) + ) +) as const; -function getDocumentationSectionPriority(path: string): number { - if (path.includes('/docs/svelte/')) return 0; - if (path.includes('/docs/kit/')) return 1; - if (path.includes('/docs/cli/')) return 2; - return 3; -} - -export function sortDocumentationPaths(paths: string[]): string[] { - return paths.sort((a, b) => { - // First compare by section priority - const priorityA = getDocumentationSectionPriority(a); - const priorityB = getDocumentationSectionPriority(b); - if (priorityA !== priorityB) return priorityA - priorityB; - - // Get directory paths - const dirA = a.split('/').slice(0, -1).join('/'); - const dirB = b.split('/').slice(0, -1).join('/'); - - // If in the same directory, prioritize index.md - if (dirA === dirB) { - if (a.endsWith('index.md')) return -1; - if (b.endsWith('index.md')) return 1; - return a.localeCompare(b); - } - - // Otherwise sort by directory path - return dirA.localeCompare(dirB); - }); -} - -export const packages = ['svelte', 'kit', 'cli'] as const; export type Package = (typeof packages)[number]; const DOCUMENTATION_NAMES: Record = { @@ -332,3 +308,33 @@ export function generateLlmContent( return content; } + +function getDocumentationSectionPriority(path: string): number { + if (path.includes('/docs/svelte/')) return 0; + if (path.includes('/docs/kit/')) return 1; + if (path.includes('/docs/cli/')) return 2; + return 3; +} + +export function sortDocumentationPaths(paths: string[]): string[] { + return paths.sort((a, b) => { + // First compare by section priority + const priorityA = getDocumentationSectionPriority(a); + const priorityB = getDocumentationSectionPriority(b); + if (priorityA !== priorityB) return priorityA - priorityB; + + // Get directory paths + const dirA = a.split('/').slice(0, -1).join('/'); + const dirB = b.split('/').slice(0, -1).join('/'); + + // If in the same directory, prioritize index.md + if (dirA === dirB) { + if (a.endsWith('index.md')) return -1; + if (b.endsWith('index.md')) return 1; + return a.localeCompare(b); + } + + // Otherwise sort by directory path + return dirA.localeCompare(dirB); + }); +} From ea0c646270f4348b28a4ba095389a47dab44ad35 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Wed, 11 Dec 2024 21:09:23 +0100 Subject: [PATCH 24/49] clean up --- apps/svelte.dev/src/lib/server/content.ts | 13 +++++-------- .../src/routes/docs/[...path]/llms.txt/+server.ts | 9 ++++----- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index ffba77597e..e245037a6d 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -130,34 +130,31 @@ function create_docs() { export const docs = create_docs(); -// Automatically determine packages from the docs directory structure export const packages = Array.from( new Set( Object.keys(docs.topics) .map((topic) => topic.split('/')[1]) .filter(Boolean) ) -) as const; +); -export type Package = (typeof packages)[number]; - -const DOCUMENTATION_NAMES: Record = { +const DOCUMENTATION_NAMES: Record = { svelte: 'Svelte', kit: 'SvelteKit', cli: 'Svelte CLI' }; -export function getDocumentationTitle(type: Package): string { +export function getDocumentationTitle(type: string): string { return `This is the developer documentation for ${DOCUMENTATION_NAMES[type]}.`; } -export function getDocumentationStartTitle(type: Package): string { +export function getDocumentationStartTitle(type: string): string { return `# Start of ${DOCUMENTATION_NAMES[type]} documentation`; } export function filterDocsByPackage( allDocs: Record, - type: Package + type: string ): Record { const filtered: Record = {}; diff --git a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts index 4fef4b2f2d..7b947aaca5 100644 --- a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts @@ -6,8 +6,7 @@ import { filterDocsByPackage, generateLlmContent, getDocumentationTitle, - packages, - type Package + packages } from '$lib/server/content'; export const prerender = true; @@ -19,17 +18,17 @@ export const entries: EntryGenerator = () => { export const GET: RequestHandler = async ({ params }) => { const packageType = params.path; - if (!packages.includes(packageType as Package)) { + if (!packages.includes(packageType)) { error(404, 'Not Found'); } - const filteredDocs = filterDocsByPackage(documentsContent, packageType as Package); + const filteredDocs = filterDocsByPackage(documentsContent, packageType); if (Object.keys(filteredDocs).length === 0) { error(404, 'No documentation found for this package'); } - const PREFIX = `${getDocumentationTitle(packageType as Package)}`; + const PREFIX = `${getDocumentationTitle(packageType)}`; const content = `${PREFIX}\n\n${generateLlmContent(filteredDocs)}`; return new Response(content, { From 08f1aea6307366a425abcce11267b7a8938a0d17 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Wed, 11 Dec 2024 21:12:28 +0100 Subject: [PATCH 25/49] fix --- apps/svelte.dev/src/lib/server/content.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index e245037a6d..8949589cfd 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -129,6 +129,7 @@ function create_docs() { } export const docs = create_docs(); +export const examples = index.examples.children; export const packages = Array.from( new Set( @@ -261,7 +262,7 @@ interface GenerateLlmContentOptions { prefix?: string; ignore?: string[]; minimize?: Partial; - package?: Package; + package?: string; } export function generateLlmContent( From c1e57a2f8339c21b06d2346802ce72326df8a831 Mon Sep 17 00:00:00 2001 From: Simon Holthausen Date: Thu, 12 Dec 2024 10:09:53 +0100 Subject: [PATCH 26/49] under_score --- apps/svelte.dev/src/lib/server/content.ts | 38 +++++++++---------- .../routes/docs/[...path]/llms.txt/+server.ts | 14 +++---- .../src/routes/llms-full.txt/+server.ts | 4 +- .../svelte.dev/src/routes/llms.txt/+server.ts | 4 +- 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index 4812a68881..cbac0db452 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -19,7 +19,7 @@ const assets = import.meta.glob( } ); -export const documentsContent = import.meta.glob('../../../content/**/*.md', { +export const documents_content = import.meta.glob('../../../content/**/*.md', { eager: true, query: '?raw', import: 'default' @@ -148,15 +148,15 @@ const DOCUMENTATION_NAMES: Record = { cli: 'Svelte CLI' }; -export function getDocumentationTitle(type: string): string { +export function get_documentation_title(type: string): string { return `This is the developer documentation for ${DOCUMENTATION_NAMES[type]}.`; } -export function getDocumentationStartTitle(type: string): string { +export function get_documentation_start_title(type: string): string { return `# Start of ${DOCUMENTATION_NAMES[type]} documentation`; } -export function filterDocsByPackage( +export function filter_docs_by_package( allDocs: Record, type: string ): Record { @@ -189,7 +189,7 @@ const defaultOptions: MinimizeOptions = { normalizeWhitespace: false }; -function removeQuoteBlocks(content: string, blockType: string): string { +function remove_quote_blocks(content: string, blockType: string): string { return content .split('\n') .reduce((acc: string[], line: string, index: number, lines: string[]) => { @@ -212,22 +212,22 @@ function removeQuoteBlocks(content: string, blockType: string): string { .join('\n'); } -function minimizeContent(content: string, options?: Partial): string { +function minimize_content(content: string, options?: Partial): string { // Merge with defaults, but only for properties that are defined const settings: MinimizeOptions = options ? { ...defaultOptions, ...options } : defaultOptions; let minimized = content; if (settings.removeLegacy) { - minimized = removeQuoteBlocks(minimized, 'LEGACY'); + minimized = remove_quote_blocks(minimized, 'LEGACY'); } if (settings.removeNoteBlocks) { - minimized = removeQuoteBlocks(minimized, 'NOTE'); + minimized = remove_quote_blocks(minimized, 'NOTE'); } if (settings.removeDetailsBlocks) { - minimized = removeQuoteBlocks(minimized, 'DETAILS'); + minimized = remove_quote_blocks(minimized, 'DETAILS'); } if (settings.removePlaygroundLinks) { @@ -251,7 +251,7 @@ function minimizeContent(content: string, options?: Partial): s return minimized; } -function shouldIncludeFileLlmDocs(filename: string, ignore: string[] = []): boolean { +function should_include_file_llm_docs(filename: string, ignore: string[] = []): boolean { const shouldIgnore = ignore.some((pattern) => minimatch(filename, pattern)); if (shouldIgnore) { if (dev) console.log(`❌ Ignored by pattern: ${filename}`); @@ -268,7 +268,7 @@ interface GenerateLlmContentOptions { package?: string; } -export function generateLlmContent( +export function generate_llm_content( docs: Record, options: GenerateLlmContentOptions = {} ): string { @@ -280,10 +280,10 @@ export function generateLlmContent( } let currentSection = ''; - const paths = sortDocumentationPaths(Object.keys(docs)); + const paths = sort_documentation_paths(Object.keys(docs)); for (const path of paths) { - if (!shouldIncludeFileLlmDocs(path, ignore)) continue; + if (!should_include_file_llm_docs(path, ignore)) continue; // If a specific package is provided, only include its docs if (pkg) { @@ -293,7 +293,7 @@ export function generateLlmContent( const docType = packages.find((p) => path.includes(`/docs/${p}/`)); if (!docType) continue; - const section = getDocumentationStartTitle(docType); + const section = get_documentation_start_title(docType); if (section !== currentSection) { if (currentSection) content += '\n'; content += `${section}\n\n`; @@ -302,7 +302,7 @@ export function generateLlmContent( } content += `## ${path.replace('../../../content/', '')}\n\n`; - const docContent = minimizeOptions ? minimizeContent(docs[path], minimizeOptions) : docs[path]; + const docContent = minimizeOptions ? minimize_content(docs[path], minimizeOptions) : docs[path]; content += docContent; content += '\n'; } @@ -310,18 +310,18 @@ export function generateLlmContent( return content; } -function getDocumentationSectionPriority(path: string): number { +function get_documentation_section_priority(path: string): number { if (path.includes('/docs/svelte/')) return 0; if (path.includes('/docs/kit/')) return 1; if (path.includes('/docs/cli/')) return 2; return 3; } -export function sortDocumentationPaths(paths: string[]): string[] { +export function sort_documentation_paths(paths: string[]): string[] { return paths.sort((a, b) => { // First compare by section priority - const priorityA = getDocumentationSectionPriority(a); - const priorityB = getDocumentationSectionPriority(b); + const priorityA = get_documentation_section_priority(a); + const priorityB = get_documentation_section_priority(b); if (priorityA !== priorityB) return priorityA - priorityB; // Get directory paths diff --git a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts index 7b947aaca5..af963d07eb 100644 --- a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts @@ -2,10 +2,10 @@ import type { RequestHandler } from './$types'; import type { EntryGenerator } from './$types'; import { error } from '@sveltejs/kit'; import { - documentsContent, - filterDocsByPackage, - generateLlmContent, - getDocumentationTitle, + documents_content, + filter_docs_by_package, + generate_llm_content, + get_documentation_title, packages } from '$lib/server/content'; @@ -22,14 +22,14 @@ export const GET: RequestHandler = async ({ params }) => { error(404, 'Not Found'); } - const filteredDocs = filterDocsByPackage(documentsContent, packageType); + const filteredDocs = filter_docs_by_package(documents_content, packageType); if (Object.keys(filteredDocs).length === 0) { error(404, 'No documentation found for this package'); } - const PREFIX = `${getDocumentationTitle(packageType)}`; - const content = `${PREFIX}\n\n${generateLlmContent(filteredDocs)}`; + const PREFIX = `${get_documentation_title(packageType)}`; + const content = `${PREFIX}\n\n${generate_llm_content(filteredDocs)}`; return new Response(content, { status: 200, diff --git a/apps/svelte.dev/src/routes/llms-full.txt/+server.ts b/apps/svelte.dev/src/routes/llms-full.txt/+server.ts index 510b9442df..e3553b73c6 100644 --- a/apps/svelte.dev/src/routes/llms-full.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms-full.txt/+server.ts @@ -1,11 +1,11 @@ import type { RequestHandler } from './$types'; -import { documentsContent, generateLlmContent } from '$lib/server/content'; +import { documents_content, generate_llm_content } from '$lib/server/content'; const PREFIX = 'This is the full developer documentation for Svelte and SvelteKit.'; export const GET: RequestHandler = async () => { - const content = `${PREFIX}\n\n${generateLlmContent(documentsContent)}`; + const content = `${PREFIX}\n\n${generate_llm_content(documents_content)}`; return new Response(content, { status: 200, diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 75cad6de7d..47d7a0f1f8 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,11 +1,11 @@ import type { RequestHandler } from './$types'; -import { documentsContent, generateLlmContent } from '$lib/server/content'; +import { documents_content, generate_llm_content } from '$lib/server/content'; const PREFIX = 'This is the abridged developer documentation for Svelte and SvelteKit.'; export const GET: RequestHandler = async () => { - const content = `${PREFIX}\n\n${generateLlmContent(documentsContent, { + const content = `${PREFIX}\n\n${generate_llm_content(documents_content, { ignore: [ // Svelte ignores '../../../content/docs/svelte/07-misc/04-custom-elements.md', From 190ff052f228022b5e9ac9f6fdaaaab90b6a9415 Mon Sep 17 00:00:00 2001 From: Simon Holthausen Date: Thu, 12 Dec 2024 10:17:05 +0100 Subject: [PATCH 27/49] code style --- .../routes/docs/[...path]/llms.txt/+server.ts | 22 +++++++++---------- .../src/routes/llms-full.txt/+server.ts | 12 ++++------ .../svelte.dev/src/routes/llms.txt/+server.ts | 13 +++++------ 3 files changed, 19 insertions(+), 28 deletions(-) diff --git a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts index af963d07eb..08d99f139a 100644 --- a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts @@ -1,5 +1,3 @@ -import type { RequestHandler } from './$types'; -import type { EntryGenerator } from './$types'; import { error } from '@sveltejs/kit'; import { documents_content, @@ -11,25 +9,25 @@ import { export const prerender = true; -export const entries: EntryGenerator = () => { +export function entries() { return packages.map((type) => ({ path: type })); -}; +} -export const GET: RequestHandler = async ({ params }) => { - const packageType = params.path; +export function GET({ params }) { + const package_type = params.path; - if (!packages.includes(packageType)) { + if (!packages.includes(package_type)) { error(404, 'Not Found'); } - const filteredDocs = filter_docs_by_package(documents_content, packageType); + const filtered_docs = filter_docs_by_package(documents_content, package_type); - if (Object.keys(filteredDocs).length === 0) { + if (Object.keys(filtered_docs).length === 0) { error(404, 'No documentation found for this package'); } - const PREFIX = `${get_documentation_title(packageType)}`; - const content = `${PREFIX}\n\n${generate_llm_content(filteredDocs)}`; + const prefix = `${get_documentation_title(package_type)}`; + const content = `${prefix}\n\n${generate_llm_content(filtered_docs)}`; return new Response(content, { status: 200, @@ -38,4 +36,4 @@ export const GET: RequestHandler = async ({ params }) => { 'Cache-Control': 'public, max-age=3600' } }); -}; +} diff --git a/apps/svelte.dev/src/routes/llms-full.txt/+server.ts b/apps/svelte.dev/src/routes/llms-full.txt/+server.ts index e3553b73c6..9a5b8a1747 100644 --- a/apps/svelte.dev/src/routes/llms-full.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms-full.txt/+server.ts @@ -1,11 +1,9 @@ -import type { RequestHandler } from './$types'; import { documents_content, generate_llm_content } from '$lib/server/content'; -const PREFIX = - 'This is the full developer documentation for Svelte and SvelteKit.'; +export const prerender = true; -export const GET: RequestHandler = async () => { - const content = `${PREFIX}\n\n${generate_llm_content(documents_content)}`; +export function GET() { + const content = `This is the full developer documentation for Svelte and SvelteKit.\n\n${generate_llm_content(documents_content)}`; return new Response(content, { status: 200, @@ -14,6 +12,4 @@ export const GET: RequestHandler = async () => { 'Cache-Control': 'public, max-age=3600' } }); -}; - -export const prerender = true; +} diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 47d7a0f1f8..d6c2e19c2e 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,11 +1,7 @@ -import type { RequestHandler } from './$types'; import { documents_content, generate_llm_content } from '$lib/server/content'; -const PREFIX = - 'This is the abridged developer documentation for Svelte and SvelteKit.'; - -export const GET: RequestHandler = async () => { - const content = `${PREFIX}\n\n${generate_llm_content(documents_content, { +export function GET() { + const main_content = generate_llm_content(documents_content, { ignore: [ // Svelte ignores '../../../content/docs/svelte/07-misc/04-custom-elements.md', @@ -36,7 +32,8 @@ export const GET: RequestHandler = async () => { removePrettierIgnore: true, normalizeWhitespace: true } - })}`; + }); + const content = `This is the abridged developer documentation for Svelte and SvelteKit.\n\n${main_content}`; return new Response(content, { status: 200, @@ -45,6 +42,6 @@ export const GET: RequestHandler = async () => { 'Cache-Control': 'public, max-age=3600' } }); -}; +} export const prerender = true; From 72a1fd18ff32415363fe32036f0f77bb440b1c7e Mon Sep 17 00:00:00 2001 From: Simon Holthausen Date: Thu, 12 Dec 2024 11:02:17 +0100 Subject: [PATCH 28/49] use real document titles, filter out empty files --- apps/svelte.dev/src/lib/server/content.ts | 64 +++++++------------ .../routes/docs/[...path]/llms.txt/+server.ts | 22 ++----- .../src/routes/llms-full.txt/+server.ts | 4 +- .../svelte.dev/src/routes/llms.txt/+server.ts | 4 +- 4 files changed, 32 insertions(+), 62 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index cbac0db452..0ea10dfb03 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -19,12 +19,6 @@ const assets = import.meta.glob( } ); -export const documents_content = import.meta.glob('../../../content/**/*.md', { - eager: true, - query: '?raw', - import: 'default' -}); - // https://github.com/vitejs/vite/issues/17453 export const index = await create_index(documents, assets, '../../../content', read); @@ -156,21 +150,6 @@ export function get_documentation_start_title(type: string): string { return `# Start of ${DOCUMENTATION_NAMES[type]} documentation`; } -export function filter_docs_by_package( - allDocs: Record, - type: string -): Record { - const filtered: Record = {}; - - for (const [path, content] of Object.entries(allDocs)) { - if (path.toLowerCase().includes(`/docs/${type}/`)) { - filtered[path] = content; - } - } - - return filtered; -} - interface MinimizeOptions { removeLegacy: boolean; removeNoteBlocks: boolean; @@ -268,10 +247,7 @@ interface GenerateLlmContentOptions { package?: string; } -export function generate_llm_content( - docs: Record, - options: GenerateLlmContentOptions = {} -): string { +export function generate_llm_content(options: GenerateLlmContentOptions = {}): string { const { prefix, ignore = [], minimize: minimizeOptions, package: pkg } = options; let content = ''; @@ -279,30 +255,34 @@ export function generate_llm_content( content = `${prefix}\n\n`; } - let currentSection = ''; - const paths = sort_documentation_paths(Object.keys(docs)); + let current_section = ''; + const paths = sort_documentation_paths(); for (const path of paths) { if (!should_include_file_llm_docs(path, ignore)) continue; // If a specific package is provided, only include its docs if (pkg) { - if (!path.includes(`/docs/${pkg}/`)) continue; + if (!path.includes(`docs/${pkg}/`)) continue; } else { // For combined content, only include paths that match any package - const docType = packages.find((p) => path.includes(`/docs/${p}/`)); - if (!docType) continue; + const doc_type = packages.find((p) => path.includes(`docs/${p}/`)); + if (!doc_type) continue; - const section = get_documentation_start_title(docType); - if (section !== currentSection) { - if (currentSection) content += '\n'; + const section = get_documentation_start_title(doc_type); + if (section !== current_section) { + if (current_section) content += '\n'; content += `${section}\n\n`; - currentSection = section; + current_section = section; } } - content += `## ${path.replace('../../../content/', '')}\n\n`; - const docContent = minimizeOptions ? minimize_content(docs[path], minimizeOptions) : docs[path]; + const docContent = minimizeOptions + ? minimize_content(index[path].body, minimizeOptions) + : index[path].body; + if (docContent.trim() === '') continue; + + content += `\n# ${index[path].metadata.title}\n\n`; content += docContent; content += '\n'; } @@ -311,14 +291,16 @@ export function generate_llm_content( } function get_documentation_section_priority(path: string): number { - if (path.includes('/docs/svelte/')) return 0; - if (path.includes('/docs/kit/')) return 1; - if (path.includes('/docs/cli/')) return 2; + if (path.includes('docs/svelte/')) return 0; + if (path.includes('docs/kit/')) return 1; + if (path.includes('docs/cli/')) return 2; return 3; } -export function sort_documentation_paths(paths: string[]): string[] { - return paths.sort((a, b) => { +function sort_documentation_paths(): string[] { + return Object.keys(index).sort((a, b) => { + a = index[a].file; + b = index[b].file; // First compare by section priority const priorityA = get_documentation_section_priority(a); const priorityB = get_documentation_section_priority(b); diff --git a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts index 08d99f139a..b66d43318a 100644 --- a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts @@ -1,11 +1,5 @@ import { error } from '@sveltejs/kit'; -import { - documents_content, - filter_docs_by_package, - generate_llm_content, - get_documentation_title, - packages -} from '$lib/server/content'; +import { generate_llm_content, get_documentation_title, packages } from '$lib/server/content'; export const prerender = true; @@ -14,20 +8,14 @@ export function entries() { } export function GET({ params }) { - const package_type = params.path; + const pkg = params.path; - if (!packages.includes(package_type)) { + if (!packages.includes(pkg)) { error(404, 'Not Found'); } - const filtered_docs = filter_docs_by_package(documents_content, package_type); - - if (Object.keys(filtered_docs).length === 0) { - error(404, 'No documentation found for this package'); - } - - const prefix = `${get_documentation_title(package_type)}`; - const content = `${prefix}\n\n${generate_llm_content(filtered_docs)}`; + const prefix = `${get_documentation_title(pkg)}`; + const content = `${prefix}\n\n${generate_llm_content({ package: pkg })}`; return new Response(content, { status: 200, diff --git a/apps/svelte.dev/src/routes/llms-full.txt/+server.ts b/apps/svelte.dev/src/routes/llms-full.txt/+server.ts index 9a5b8a1747..b5f9bcae4b 100644 --- a/apps/svelte.dev/src/routes/llms-full.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms-full.txt/+server.ts @@ -1,9 +1,9 @@ -import { documents_content, generate_llm_content } from '$lib/server/content'; +import { generate_llm_content } from '$lib/server/content'; export const prerender = true; export function GET() { - const content = `This is the full developer documentation for Svelte and SvelteKit.\n\n${generate_llm_content(documents_content)}`; + const content = `This is the full developer documentation for Svelte and SvelteKit.\n\n${generate_llm_content()}`; return new Response(content, { status: 200, diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index d6c2e19c2e..3c583fd755 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,7 +1,7 @@ -import { documents_content, generate_llm_content } from '$lib/server/content'; +import { generate_llm_content } from '$lib/server/content'; export function GET() { - const main_content = generate_llm_content(documents_content, { + const main_content = generate_llm_content({ ignore: [ // Svelte ignores '../../../content/docs/svelte/07-misc/04-custom-elements.md', From aaee7c6a74a24368708c26f8210fab0576c3f053 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Thu, 12 Dec 2024 23:18:20 +0100 Subject: [PATCH 29/49] move llms.txt to llms-small.txt --- .../svelte.dev/src/routes/{llms.txt => llms-small.txt}/+server.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename apps/svelte.dev/src/routes/{llms.txt => llms-small.txt}/+server.ts (100%) diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms-small.txt/+server.ts similarity index 100% rename from apps/svelte.dev/src/routes/llms.txt/+server.ts rename to apps/svelte.dev/src/routes/llms-small.txt/+server.ts From a7f0f8df25c467d0bfe8d4d4c099599f9a10fdab Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Fri, 13 Dec 2024 00:09:40 +0100 Subject: [PATCH 30/49] llms.txt index --- apps/svelte.dev/src/lib/server/content.ts | 2 +- .../svelte.dev/src/routes/llms.txt/+server.ts | 39 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 apps/svelte.dev/src/routes/llms.txt/+server.ts diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index 0ea10dfb03..2f0c3b7d91 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -136,7 +136,7 @@ export const packages = Array.from( ) ); -const DOCUMENTATION_NAMES: Record = { +export const DOCUMENTATION_NAMES: Record = { svelte: 'Svelte', kit: 'SvelteKit', cli: 'Svelte CLI' diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts new file mode 100644 index 0000000000..7d554957f5 --- /dev/null +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -0,0 +1,39 @@ +import { get_documentation_title, packages, DOCUMENTATION_NAMES } from '$lib/server/content'; + +export const prerender = true; + +export function GET() { + const package_docs = packages + .map( + (pkg) => + `- [${DOCUMENTATION_NAMES[pkg]} documentation](../docs/${pkg}/llms.txt): ${get_documentation_title(pkg)}` + ) + .join('\n'); + + const content = `# Svelte Documentation for LLMs + +> This directory contains various forms of the Svelte documentation formatted for Large Language Models (LLMs). + +## Documentation Sets + +- [Abridged documentation](../llms-small.txt): A minimal version of the Svelte and SvelteKit documentation, with examples and non-essential content removed +- [Complete documentation](../llms-full.txt): The complete Svelte and SvelteKit documentation including all examples and additional content + +## Individual Package Documentation + +${package_docs} + +## Notes + +- The abridged documentation excludes legacy compatibility notes, detailed examples, and supplementary information +- The complete documentation includes all content from the official documentation +- Package-specific documentation files contain only the content relevant to that package +- The content is automatically generated from the same source as the official documentation`; + + return new Response(content, { + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + 'Cache-Control': 'public, max-age=3600' + } + }); +} From c5fac955d438c5d170f82c2c715a4855db4173ac Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Fri, 13 Dec 2024 00:10:15 +0100 Subject: [PATCH 31/49] Update +server.ts --- apps/svelte.dev/src/routes/llms.txt/+server.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 7d554957f5..c743592bc9 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,12 +1,14 @@ import { get_documentation_title, packages, DOCUMENTATION_NAMES } from '$lib/server/content'; +const DOMAIN = 'https://svelte.dev'; + export const prerender = true; export function GET() { const package_docs = packages .map( (pkg) => - `- [${DOCUMENTATION_NAMES[pkg]} documentation](../docs/${pkg}/llms.txt): ${get_documentation_title(pkg)}` + `- [${DOCUMENTATION_NAMES[pkg]} documentation](${DOMAIN}/docs/${pkg}/llms.txt): ${get_documentation_title(pkg)}` ) .join('\n'); @@ -16,8 +18,8 @@ export function GET() { ## Documentation Sets -- [Abridged documentation](../llms-small.txt): A minimal version of the Svelte and SvelteKit documentation, with examples and non-essential content removed -- [Complete documentation](../llms-full.txt): The complete Svelte and SvelteKit documentation including all examples and additional content +- [Abridged documentation](${DOMAIN}/llms-small.txt): A minimal version of the Svelte and SvelteKit documentation, with examples and non-essential content removed +- [Complete documentation](${DOMAIN}/llms-full.txt): The complete Svelte and SvelteKit documentation including all examples and additional content ## Individual Package Documentation From 29b472696aad07e94f03fa93d966369ff67f487c Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Fri, 13 Dec 2024 00:10:47 +0100 Subject: [PATCH 32/49] Update +server.ts --- apps/svelte.dev/src/routes/llms.txt/+server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index c743592bc9..0068b6adb5 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -14,7 +14,7 @@ export function GET() { const content = `# Svelte Documentation for LLMs -> This directory contains various forms of the Svelte documentation formatted for Large Language Models (LLMs). +> Svelte is a UI framework that uses a compiler to let you write breathtakingly concise components that do minimal work in the browser, using languages you already know — HTML, CSS and JavaScript. ## Documentation Sets From 9eeaca69bb0e1d11852ba4e3d216b71c312bb663 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Fri, 13 Dec 2024 00:14:58 +0100 Subject: [PATCH 33/49] Fix index --- apps/svelte.dev/.env | 2 ++ apps/svelte.dev/src/routes/llms.txt/+server.ts | 9 ++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/svelte.dev/.env b/apps/svelte.dev/.env index 9b1319a6f1..616fecf879 100644 --- a/apps/svelte.dev/.env +++ b/apps/svelte.dev/.env @@ -3,3 +3,5 @@ SUPABASE_KEY= GITHUB_CLIENT_ID= GITHUB_CLIENT_SECRET= + +VERCEL_URL=http://localhost:5173 \ No newline at end of file diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 0068b6adb5..f5788d3811 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,14 +1,13 @@ +import { VERCEL_URL } from '$env/static/private'; import { get_documentation_title, packages, DOCUMENTATION_NAMES } from '$lib/server/content'; -const DOMAIN = 'https://svelte.dev'; - export const prerender = true; export function GET() { const package_docs = packages .map( (pkg) => - `- [${DOCUMENTATION_NAMES[pkg]} documentation](${DOMAIN}/docs/${pkg}/llms.txt): ${get_documentation_title(pkg)}` + `- [${DOCUMENTATION_NAMES[pkg]} documentation](${VERCEL_URL}/docs/${pkg}/llms.txt): ${get_documentation_title(pkg)}` ) .join('\n'); @@ -18,8 +17,8 @@ export function GET() { ## Documentation Sets -- [Abridged documentation](${DOMAIN}/llms-small.txt): A minimal version of the Svelte and SvelteKit documentation, with examples and non-essential content removed -- [Complete documentation](${DOMAIN}/llms-full.txt): The complete Svelte and SvelteKit documentation including all examples and additional content +- [Abridged documentation](${VERCEL_URL}/llms-small.txt): A minimal version of the Svelte and SvelteKit documentation, with examples and non-essential content removed +- [Complete documentation](${VERCEL_URL}/llms-full.txt): The complete Svelte and SvelteKit documentation including all examples and additional content ## Individual Package Documentation From 9c8469cbed2f05f8fc2ef1231e92ce33b3fc9df9 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Fri, 13 Dec 2024 00:22:10 +0100 Subject: [PATCH 34/49] fix --- apps/svelte.dev/.env | 2 +- apps/svelte.dev/src/routes/llms.txt/+server.ts | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/svelte.dev/.env b/apps/svelte.dev/.env index 616fecf879..7f5f4507ba 100644 --- a/apps/svelte.dev/.env +++ b/apps/svelte.dev/.env @@ -4,4 +4,4 @@ SUPABASE_KEY= GITHUB_CLIENT_ID= GITHUB_CLIENT_SECRET= -VERCEL_URL=http://localhost:5173 \ No newline at end of file +VERCEL_URL= \ No newline at end of file diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index f5788d3811..59c3b346a7 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,13 +1,15 @@ import { VERCEL_URL } from '$env/static/private'; import { get_documentation_title, packages, DOCUMENTATION_NAMES } from '$lib/server/content'; +const DOMAIN = VERCEL_URL ? `https://${VERCEL_URL}` : ''; + export const prerender = true; export function GET() { const package_docs = packages .map( (pkg) => - `- [${DOCUMENTATION_NAMES[pkg]} documentation](${VERCEL_URL}/docs/${pkg}/llms.txt): ${get_documentation_title(pkg)}` + `- [${DOCUMENTATION_NAMES[pkg]} documentation](${DOMAIN}/docs/${pkg}/llms.txt): ${get_documentation_title(pkg)}` ) .join('\n'); @@ -17,8 +19,8 @@ export function GET() { ## Documentation Sets -- [Abridged documentation](${VERCEL_URL}/llms-small.txt): A minimal version of the Svelte and SvelteKit documentation, with examples and non-essential content removed -- [Complete documentation](${VERCEL_URL}/llms-full.txt): The complete Svelte and SvelteKit documentation including all examples and additional content +- [Abridged documentation](${DOMAIN}/llms-small.txt): A minimal version of the Svelte and SvelteKit documentation, with examples and non-essential content removed +- [Complete documentation](${DOMAIN}/llms-full.txt): The complete Svelte and SvelteKit documentation including all examples and additional content ## Individual Package Documentation From e9d7e70fdd72682b215ee3f0b21e78f30c75b26d Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Fri, 13 Dec 2024 00:50:14 +0100 Subject: [PATCH 35/49] revert VERCEL_URL usage --- apps/svelte.dev/.env | 4 +--- apps/svelte.dev/src/routes/llms.txt/+server.ts | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/apps/svelte.dev/.env b/apps/svelte.dev/.env index 7f5f4507ba..31fc509a46 100644 --- a/apps/svelte.dev/.env +++ b/apps/svelte.dev/.env @@ -2,6 +2,4 @@ SUPABASE_URL= SUPABASE_KEY= GITHUB_CLIENT_ID= -GITHUB_CLIENT_SECRET= - -VERCEL_URL= \ No newline at end of file +GITHUB_CLIENT_SECRET= \ No newline at end of file diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 59c3b346a7..59362f229e 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,7 +1,6 @@ -import { VERCEL_URL } from '$env/static/private'; import { get_documentation_title, packages, DOCUMENTATION_NAMES } from '$lib/server/content'; -const DOMAIN = VERCEL_URL ? `https://${VERCEL_URL}` : ''; +const DOMAIN = `https://svelte.dev`; export const prerender = true; From 855a19784467ca53e14c788a9705368c9a493b31 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Fri, 13 Dec 2024 00:52:32 +0100 Subject: [PATCH 36/49] Update apps/svelte.dev/src/lib/server/content.ts Co-authored-by: Rich Harris --- apps/svelte.dev/src/lib/server/content.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index 2f0c3b7d91..bc72415ef7 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -128,13 +128,7 @@ function create_docs() { export const docs = create_docs(); export const examples = index.examples.children; -export const packages = Array.from( - new Set( - Object.keys(docs.topics) - .map((topic) => topic.split('/')[1]) - .filter(Boolean) - ) -); +export const packages = Object.keys(docs.topics).map((topic) => topic.split('/')[1]); export const DOCUMENTATION_NAMES: Record = { svelte: 'Svelte', From f0c91cc55a78c4d5cf9c40f9af0f75188bedd421 Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Thu, 12 Dec 2024 19:18:29 -0500 Subject: [PATCH 37/49] move llm stuff into its own module --- apps/svelte.dev/src/lib/server/content.ts | 190 ----------------- apps/svelte.dev/src/lib/server/llms.ts | 191 ++++++++++++++++++ .../routes/docs/[...path]/llms.txt/+server.ts | 2 +- .../src/routes/llms-full.txt/+server.ts | 2 +- .../src/routes/llms-small.txt/+server.ts | 2 +- .../svelte.dev/src/routes/llms.txt/+server.ts | 2 +- 6 files changed, 195 insertions(+), 194 deletions(-) create mode 100644 apps/svelte.dev/src/lib/server/llms.ts diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index bc72415ef7..fed156751e 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -1,8 +1,6 @@ -import { dev } from '$app/environment'; import { read } from '$app/server'; import type { Document } from '@sveltejs/site-kit'; import { create_index } from '@sveltejs/site-kit/server/content'; -import { minimatch } from 'minimatch'; const documents = import.meta.glob('../../../content/**/*.md', { eager: true, @@ -127,191 +125,3 @@ function create_docs() { export const docs = create_docs(); export const examples = index.examples.children; - -export const packages = Object.keys(docs.topics).map((topic) => topic.split('/')[1]); - -export const DOCUMENTATION_NAMES: Record = { - svelte: 'Svelte', - kit: 'SvelteKit', - cli: 'Svelte CLI' -}; - -export function get_documentation_title(type: string): string { - return `This is the developer documentation for ${DOCUMENTATION_NAMES[type]}.`; -} - -export function get_documentation_start_title(type: string): string { - return `# Start of ${DOCUMENTATION_NAMES[type]} documentation`; -} - -interface MinimizeOptions { - removeLegacy: boolean; - removeNoteBlocks: boolean; - removeDetailsBlocks: boolean; - removePlaygroundLinks: boolean; - removePrettierIgnore: boolean; - normalizeWhitespace: boolean; -} - -const defaultOptions: MinimizeOptions = { - removeLegacy: false, - removeNoteBlocks: false, - removeDetailsBlocks: false, - removePlaygroundLinks: false, - removePrettierIgnore: false, - normalizeWhitespace: false -}; - -function remove_quote_blocks(content: string, blockType: string): string { - return content - .split('\n') - .reduce((acc: string[], line: string, index: number, lines: string[]) => { - // If we find a block (with or without additional text), skip it and all subsequent blockquote lines - if (line.trim().startsWith(`> [!${blockType}]`)) { - // Skip all subsequent lines that are part of the blockquote - let i = index; - while (i < lines.length && (lines[i].startsWith('>') || lines[i].trim() === '')) { - i++; - } - // Update the index to skip all these lines - index = i - 1; - return acc; - } - - // Only add the line if it's not being skipped - acc.push(line); - return acc; - }, []) - .join('\n'); -} - -function minimize_content(content: string, options?: Partial): string { - // Merge with defaults, but only for properties that are defined - const settings: MinimizeOptions = options ? { ...defaultOptions, ...options } : defaultOptions; - - let minimized = content; - - if (settings.removeLegacy) { - minimized = remove_quote_blocks(minimized, 'LEGACY'); - } - - if (settings.removeNoteBlocks) { - minimized = remove_quote_blocks(minimized, 'NOTE'); - } - - if (settings.removeDetailsBlocks) { - minimized = remove_quote_blocks(minimized, 'DETAILS'); - } - - if (settings.removePlaygroundLinks) { - // Replace playground URLs with /[link] but keep the original link text - minimized = minimized.replace(/\[([^\]]+)\]\(\/playground[^)]+\)/g, '[$1](/REMOVED)'); - } - - if (settings.removePrettierIgnore) { - minimized = minimized - .split('\n') - .filter((line) => line.trim() !== '') - .join('\n'); - } - - if (settings.normalizeWhitespace) { - minimized = minimized.replace(/\s+/g, ' '); - } - - minimized = minimized.trim(); - - return minimized; -} - -function should_include_file_llm_docs(filename: string, ignore: string[] = []): boolean { - const shouldIgnore = ignore.some((pattern) => minimatch(filename, pattern)); - if (shouldIgnore) { - if (dev) console.log(`❌ Ignored by pattern: ${filename}`); - return false; - } - - return true; -} - -interface GenerateLlmContentOptions { - prefix?: string; - ignore?: string[]; - minimize?: Partial; - package?: string; -} - -export function generate_llm_content(options: GenerateLlmContentOptions = {}): string { - const { prefix, ignore = [], minimize: minimizeOptions, package: pkg } = options; - - let content = ''; - if (prefix) { - content = `${prefix}\n\n`; - } - - let current_section = ''; - const paths = sort_documentation_paths(); - - for (const path of paths) { - if (!should_include_file_llm_docs(path, ignore)) continue; - - // If a specific package is provided, only include its docs - if (pkg) { - if (!path.includes(`docs/${pkg}/`)) continue; - } else { - // For combined content, only include paths that match any package - const doc_type = packages.find((p) => path.includes(`docs/${p}/`)); - if (!doc_type) continue; - - const section = get_documentation_start_title(doc_type); - if (section !== current_section) { - if (current_section) content += '\n'; - content += `${section}\n\n`; - current_section = section; - } - } - - const docContent = minimizeOptions - ? minimize_content(index[path].body, minimizeOptions) - : index[path].body; - if (docContent.trim() === '') continue; - - content += `\n# ${index[path].metadata.title}\n\n`; - content += docContent; - content += '\n'; - } - - return content; -} - -function get_documentation_section_priority(path: string): number { - if (path.includes('docs/svelte/')) return 0; - if (path.includes('docs/kit/')) return 1; - if (path.includes('docs/cli/')) return 2; - return 3; -} - -function sort_documentation_paths(): string[] { - return Object.keys(index).sort((a, b) => { - a = index[a].file; - b = index[b].file; - // First compare by section priority - const priorityA = get_documentation_section_priority(a); - const priorityB = get_documentation_section_priority(b); - if (priorityA !== priorityB) return priorityA - priorityB; - - // Get directory paths - const dirA = a.split('/').slice(0, -1).join('/'); - const dirB = b.split('/').slice(0, -1).join('/'); - - // If in the same directory, prioritize index.md - if (dirA === dirB) { - if (a.endsWith('index.md')) return -1; - if (b.endsWith('index.md')) return 1; - return a.localeCompare(b); - } - - // Otherwise sort by directory path - return dirA.localeCompare(dirB); - }); -} diff --git a/apps/svelte.dev/src/lib/server/llms.ts b/apps/svelte.dev/src/lib/server/llms.ts new file mode 100644 index 0000000000..9bc4a6199e --- /dev/null +++ b/apps/svelte.dev/src/lib/server/llms.ts @@ -0,0 +1,191 @@ +import { minimatch } from 'minimatch'; +import { dev } from '$app/environment'; +import { docs, index } from './content'; + +interface GenerateLlmContentOptions { + prefix?: string; + ignore?: string[]; + minimize?: Partial; + package?: string; +} + +export function generate_llm_content(options: GenerateLlmContentOptions = {}): string { + const { prefix, ignore = [], minimize: minimizeOptions, package: pkg } = options; + + let content = ''; + if (prefix) { + content = `${prefix}\n\n`; + } + + let current_section = ''; + const paths = sort_documentation_paths(); + + for (const path of paths) { + if (!should_include_file_llm_docs(path, ignore)) continue; + + // If a specific package is provided, only include its docs + if (pkg) { + if (!path.includes(`docs/${pkg}/`)) continue; + } else { + // For combined content, only include paths that match any package + const doc_type = packages.find((p) => path.includes(`docs/${p}/`)); + if (!doc_type) continue; + + const section = get_documentation_start_title(doc_type); + if (section !== current_section) { + if (current_section) content += '\n'; + content += `${section}\n\n`; + current_section = section; + } + } + + const docContent = minimizeOptions + ? minimize_content(index[path].body, minimizeOptions) + : index[path].body; + if (docContent.trim() === '') continue; + + content += `\n# ${index[path].metadata.title}\n\n`; + content += docContent; + content += '\n'; + } + + return content; +} + +export const packages = Object.keys(docs.topics).map((topic) => topic.split('/')[1]); + +export const DOCUMENTATION_NAMES: Record = { + svelte: 'Svelte', + kit: 'SvelteKit', + cli: 'Svelte CLI' +}; + +export function get_documentation_title(type: string): string { + return `This is the developer documentation for ${DOCUMENTATION_NAMES[type]}.`; +} + +export function get_documentation_start_title(type: string): string { + return `# Start of ${DOCUMENTATION_NAMES[type]} documentation`; +} + +function minimize_content(content: string, options?: Partial): string { + // Merge with defaults, but only for properties that are defined + const settings: MinimizeOptions = options ? { ...defaultOptions, ...options } : defaultOptions; + + let minimized = content; + + if (settings.removeLegacy) { + minimized = remove_quote_blocks(minimized, 'LEGACY'); + } + + if (settings.removeNoteBlocks) { + minimized = remove_quote_blocks(minimized, 'NOTE'); + } + + if (settings.removeDetailsBlocks) { + minimized = remove_quote_blocks(minimized, 'DETAILS'); + } + + if (settings.removePlaygroundLinks) { + // Replace playground URLs with /[link] but keep the original link text + minimized = minimized.replace(/\[([^\]]+)\]\(\/playground[^)]+\)/g, '[$1](/REMOVED)'); + } + + if (settings.removePrettierIgnore) { + minimized = minimized + .split('\n') + .filter((line) => line.trim() !== '') + .join('\n'); + } + + if (settings.normalizeWhitespace) { + minimized = minimized.replace(/\s+/g, ' '); + } + + minimized = minimized.trim(); + + return minimized; +} + +function should_include_file_llm_docs(filename: string, ignore: string[] = []): boolean { + const shouldIgnore = ignore.some((pattern) => minimatch(filename, pattern)); + if (shouldIgnore) { + if (dev) console.log(`❌ Ignored by pattern: ${filename}`); + return false; + } + + return true; +} + +function get_documentation_section_priority(path: string): number { + if (path.includes('docs/svelte/')) return 0; + if (path.includes('docs/kit/')) return 1; + if (path.includes('docs/cli/')) return 2; + return 3; +} + +function sort_documentation_paths(): string[] { + return Object.keys(index).sort((a, b) => { + a = index[a].file; + b = index[b].file; + // First compare by section priority + const priorityA = get_documentation_section_priority(a); + const priorityB = get_documentation_section_priority(b); + if (priorityA !== priorityB) return priorityA - priorityB; + + // Get directory paths + const dirA = a.split('/').slice(0, -1).join('/'); + const dirB = b.split('/').slice(0, -1).join('/'); + + // If in the same directory, prioritize index.md + if (dirA === dirB) { + if (a.endsWith('index.md')) return -1; + if (b.endsWith('index.md')) return 1; + return a.localeCompare(b); + } + + // Otherwise sort by directory path + return dirA.localeCompare(dirB); + }); +} + +interface MinimizeOptions { + removeLegacy: boolean; + removeNoteBlocks: boolean; + removeDetailsBlocks: boolean; + removePlaygroundLinks: boolean; + removePrettierIgnore: boolean; + normalizeWhitespace: boolean; +} + +const defaultOptions: MinimizeOptions = { + removeLegacy: false, + removeNoteBlocks: false, + removeDetailsBlocks: false, + removePlaygroundLinks: false, + removePrettierIgnore: false, + normalizeWhitespace: false +}; + +function remove_quote_blocks(content: string, blockType: string): string { + return content + .split('\n') + .reduce((acc: string[], line: string, index: number, lines: string[]) => { + // If we find a block (with or without additional text), skip it and all subsequent blockquote lines + if (line.trim().startsWith(`> [!${blockType}]`)) { + // Skip all subsequent lines that are part of the blockquote + let i = index; + while (i < lines.length && (lines[i].startsWith('>') || lines[i].trim() === '')) { + i++; + } + // Update the index to skip all these lines + index = i - 1; + return acc; + } + + // Only add the line if it's not being skipped + acc.push(line); + return acc; + }, []) + .join('\n'); +} diff --git a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts index b66d43318a..64632b2f4d 100644 --- a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts @@ -1,5 +1,5 @@ import { error } from '@sveltejs/kit'; -import { generate_llm_content, get_documentation_title, packages } from '$lib/server/content'; +import { generate_llm_content, get_documentation_title, packages } from '$lib/server/llms'; export const prerender = true; diff --git a/apps/svelte.dev/src/routes/llms-full.txt/+server.ts b/apps/svelte.dev/src/routes/llms-full.txt/+server.ts index b5f9bcae4b..60f22c2a7b 100644 --- a/apps/svelte.dev/src/routes/llms-full.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms-full.txt/+server.ts @@ -1,4 +1,4 @@ -import { generate_llm_content } from '$lib/server/content'; +import { generate_llm_content } from '$lib/server/llms'; export const prerender = true; diff --git a/apps/svelte.dev/src/routes/llms-small.txt/+server.ts b/apps/svelte.dev/src/routes/llms-small.txt/+server.ts index 3c583fd755..cbd9c90f9d 100644 --- a/apps/svelte.dev/src/routes/llms-small.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms-small.txt/+server.ts @@ -1,4 +1,4 @@ -import { generate_llm_content } from '$lib/server/content'; +import { generate_llm_content } from '$lib/server/llms'; export function GET() { const main_content = generate_llm_content({ diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 59362f229e..4969c4f765 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,4 +1,4 @@ -import { get_documentation_title, packages, DOCUMENTATION_NAMES } from '$lib/server/content'; +import { get_documentation_title, packages, DOCUMENTATION_NAMES } from '$lib/server/llms'; const DOMAIN = `https://svelte.dev`; From 086c48fe3b1a9005d8b3777d9471acb2a9144b62 Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Thu, 12 Dec 2024 19:19:36 -0500 Subject: [PATCH 38/49] revert whitespace changes --- apps/svelte.dev/.env | 2 +- apps/svelte.dev/src/lib/server/content.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/svelte.dev/.env b/apps/svelte.dev/.env index 31fc509a46..9b1319a6f1 100644 --- a/apps/svelte.dev/.env +++ b/apps/svelte.dev/.env @@ -2,4 +2,4 @@ SUPABASE_URL= SUPABASE_KEY= GITHUB_CLIENT_ID= -GITHUB_CLIENT_SECRET= \ No newline at end of file +GITHUB_CLIENT_SECRET= diff --git a/apps/svelte.dev/src/lib/server/content.ts b/apps/svelte.dev/src/lib/server/content.ts index fed156751e..01c84c2ad6 100644 --- a/apps/svelte.dev/src/lib/server/content.ts +++ b/apps/svelte.dev/src/lib/server/content.ts @@ -124,4 +124,5 @@ function create_docs() { } export const docs = create_docs(); + export const examples = index.examples.children; From d7f9180fba724fc401912ac880e79674cacd312f Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Thu, 12 Dec 2024 19:21:04 -0500 Subject: [PATCH 39/49] snake_case --- apps/svelte.dev/src/lib/server/llms.ts | 42 +++++++++---------- .../src/routes/llms-small.txt/+server.ts | 12 +++--- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/llms.ts b/apps/svelte.dev/src/lib/server/llms.ts index 9bc4a6199e..3a54e82635 100644 --- a/apps/svelte.dev/src/lib/server/llms.ts +++ b/apps/svelte.dev/src/lib/server/llms.ts @@ -9,6 +9,15 @@ interface GenerateLlmContentOptions { package?: string; } +interface MinimizeOptions { + remove_legacy: boolean; + remove_note_blocks: boolean; + remove_details_blocks: boolean; + remove_playground_links: boolean; + remove_prettier_ignore: boolean; + normalize_whitespace: boolean; +} + export function generate_llm_content(options: GenerateLlmContentOptions = {}): string { const { prefix, ignore = [], minimize: minimizeOptions, package: pkg } = options; @@ -74,31 +83,31 @@ function minimize_content(content: string, options?: Partial): let minimized = content; - if (settings.removeLegacy) { + if (settings.remove_legacy) { minimized = remove_quote_blocks(minimized, 'LEGACY'); } - if (settings.removeNoteBlocks) { + if (settings.remove_note_blocks) { minimized = remove_quote_blocks(minimized, 'NOTE'); } - if (settings.removeDetailsBlocks) { + if (settings.remove_details_blocks) { minimized = remove_quote_blocks(minimized, 'DETAILS'); } - if (settings.removePlaygroundLinks) { + if (settings.remove_playground_links) { // Replace playground URLs with /[link] but keep the original link text minimized = minimized.replace(/\[([^\]]+)\]\(\/playground[^)]+\)/g, '[$1](/REMOVED)'); } - if (settings.removePrettierIgnore) { + if (settings.remove_prettier_ignore) { minimized = minimized .split('\n') .filter((line) => line.trim() !== '') .join('\n'); } - if (settings.normalizeWhitespace) { + if (settings.normalize_whitespace) { minimized = minimized.replace(/\s+/g, ' '); } @@ -149,22 +158,13 @@ function sort_documentation_paths(): string[] { }); } -interface MinimizeOptions { - removeLegacy: boolean; - removeNoteBlocks: boolean; - removeDetailsBlocks: boolean; - removePlaygroundLinks: boolean; - removePrettierIgnore: boolean; - normalizeWhitespace: boolean; -} - const defaultOptions: MinimizeOptions = { - removeLegacy: false, - removeNoteBlocks: false, - removeDetailsBlocks: false, - removePlaygroundLinks: false, - removePrettierIgnore: false, - normalizeWhitespace: false + remove_legacy: false, + remove_note_blocks: false, + remove_details_blocks: false, + remove_playground_links: false, + remove_prettier_ignore: false, + normalize_whitespace: false }; function remove_quote_blocks(content: string, blockType: string): string { diff --git a/apps/svelte.dev/src/routes/llms-small.txt/+server.ts b/apps/svelte.dev/src/routes/llms-small.txt/+server.ts index cbd9c90f9d..133a7f53ff 100644 --- a/apps/svelte.dev/src/routes/llms-small.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms-small.txt/+server.ts @@ -25,12 +25,12 @@ export function GET() { '../../../content/docs/kit/60-appendix/**/*.md' ], minimize: { - removeLegacy: true, - removeNoteBlocks: true, - removeDetailsBlocks: true, - removePlaygroundLinks: true, - removePrettierIgnore: true, - normalizeWhitespace: true + remove_legacy: true, + remove_note_blocks: true, + remove_details_blocks: true, + remove_playground_links: true, + remove_prettier_ignore: true, + normalize_whitespace: true } }); const content = `This is the abridged developer documentation for Svelte and SvelteKit.\n\n${main_content}`; From d541799c24511f414ade9624a0a402e7e04f384e Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Thu, 12 Dec 2024 19:22:12 -0500 Subject: [PATCH 40/49] tweak --- apps/svelte.dev/src/lib/server/llms.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/llms.ts b/apps/svelte.dev/src/lib/server/llms.ts index 3a54e82635..177bdf8350 100644 --- a/apps/svelte.dev/src/lib/server/llms.ts +++ b/apps/svelte.dev/src/lib/server/llms.ts @@ -19,22 +19,21 @@ interface MinimizeOptions { } export function generate_llm_content(options: GenerateLlmContentOptions = {}): string { - const { prefix, ignore = [], minimize: minimizeOptions, package: pkg } = options; - let content = ''; - if (prefix) { - content = `${prefix}\n\n`; + + if (options.prefix) { + content = `${options.prefix}\n\n`; } let current_section = ''; const paths = sort_documentation_paths(); for (const path of paths) { - if (!should_include_file_llm_docs(path, ignore)) continue; + if (!should_include_file_llm_docs(path, options.ignore)) continue; // If a specific package is provided, only include its docs - if (pkg) { - if (!path.includes(`docs/${pkg}/`)) continue; + if (options.package) { + if (!path.includes(`docs/${options.package}/`)) continue; } else { // For combined content, only include paths that match any package const doc_type = packages.find((p) => path.includes(`docs/${p}/`)); @@ -48,8 +47,8 @@ export function generate_llm_content(options: GenerateLlmContentOptions = {}): s } } - const docContent = minimizeOptions - ? minimize_content(index[path].body, minimizeOptions) + const docContent = options.minimize + ? minimize_content(index[path].body, options.minimize) : index[path].body; if (docContent.trim() === '') continue; From 8137d82fbae0fc41a4e4b56b7fcbff6fe8dfcb95 Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Thu, 12 Dec 2024 19:26:10 -0500 Subject: [PATCH 41/49] snake_case etc --- apps/svelte.dev/src/lib/server/llms.ts | 29 +++++++++++++------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/llms.ts b/apps/svelte.dev/src/lib/server/llms.ts index 177bdf8350..56c026b1e8 100644 --- a/apps/svelte.dev/src/lib/server/llms.ts +++ b/apps/svelte.dev/src/lib/server/llms.ts @@ -18,6 +18,15 @@ interface MinimizeOptions { normalize_whitespace: boolean; } +const defaults: MinimizeOptions = { + remove_legacy: false, + remove_note_blocks: false, + remove_details_blocks: false, + remove_playground_links: false, + remove_prettier_ignore: false, + normalize_whitespace: false +}; + export function generate_llm_content(options: GenerateLlmContentOptions = {}): string { let content = ''; @@ -47,13 +56,13 @@ export function generate_llm_content(options: GenerateLlmContentOptions = {}): s } } - const docContent = options.minimize + const doc_content = options.minimize ? minimize_content(index[path].body, options.minimize) : index[path].body; - if (docContent.trim() === '') continue; + if (doc_content.trim() === '') continue; content += `\n# ${index[path].metadata.title}\n\n`; - content += docContent; + content += doc_content; content += '\n'; } @@ -78,7 +87,7 @@ export function get_documentation_start_title(type: string): string { function minimize_content(content: string, options?: Partial): string { // Merge with defaults, but only for properties that are defined - const settings: MinimizeOptions = options ? { ...defaultOptions, ...options } : defaultOptions; + const settings: MinimizeOptions = { ...defaults, ...options }; let minimized = content; @@ -116,8 +125,7 @@ function minimize_content(content: string, options?: Partial): } function should_include_file_llm_docs(filename: string, ignore: string[] = []): boolean { - const shouldIgnore = ignore.some((pattern) => minimatch(filename, pattern)); - if (shouldIgnore) { + if (ignore.some((pattern) => minimatch(filename, pattern))) { if (dev) console.log(`❌ Ignored by pattern: ${filename}`); return false; } @@ -157,15 +165,6 @@ function sort_documentation_paths(): string[] { }); } -const defaultOptions: MinimizeOptions = { - remove_legacy: false, - remove_note_blocks: false, - remove_details_blocks: false, - remove_playground_links: false, - remove_prettier_ignore: false, - normalize_whitespace: false -}; - function remove_quote_blocks(content: string, blockType: string): string { return content .split('\n') From edc5d43273e23aa05fce6f7c1e6800d280dcee1c Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Thu, 12 Dec 2024 19:36:35 -0500 Subject: [PATCH 42/49] make ignores work --- apps/svelte.dev/src/lib/server/llms.ts | 6 ++-- .../src/routes/llms-small.txt/+server.ts | 33 +++++++++---------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/llms.ts b/apps/svelte.dev/src/lib/server/llms.ts index 56c026b1e8..f2d677737a 100644 --- a/apps/svelte.dev/src/lib/server/llms.ts +++ b/apps/svelte.dev/src/lib/server/llms.ts @@ -124,9 +124,9 @@ function minimize_content(content: string, options?: Partial): return minimized; } -function should_include_file_llm_docs(filename: string, ignore: string[] = []): boolean { - if (ignore.some((pattern) => minimatch(filename, pattern))) { - if (dev) console.log(`❌ Ignored by pattern: ${filename}`); +function should_include_file_llm_docs(path: string, ignore: string[] = []): boolean { + if (ignore.some((pattern) => minimatch(path, pattern))) { + if (dev) console.log(`❌ Ignored by pattern: ${path}`); return false; } diff --git a/apps/svelte.dev/src/routes/llms-small.txt/+server.ts b/apps/svelte.dev/src/routes/llms-small.txt/+server.ts index 133a7f53ff..cc613ce008 100644 --- a/apps/svelte.dev/src/routes/llms-small.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms-small.txt/+server.ts @@ -4,25 +4,24 @@ export function GET() { const main_content = generate_llm_content({ ignore: [ // Svelte ignores - '../../../content/docs/svelte/07-misc/04-custom-elements.md', - '../../../content/docs/svelte/07-misc/06-v4-migration-guide.md', - '../../../content/docs/svelte/07-misc/07-v5-migration-guide.md', - '../../../content/docs/svelte/07-misc/99-faq.md', - '../../../content/docs/svelte/07-misc/xx-reactivity-indepth.md', - '../../../content/docs/svelte/98-reference/21-svelte-legacy.md', - '../../../content/docs/svelte/99-legacy/**/*.md', - '../../../content/docs/svelte/98-reference/30-runtime-errors.md', - '../../../content/docs/svelte/98-reference/30-runtime-warnings.md', - '../../../content/docs/svelte/98-reference/30-compiler-errors.md', - '../../../content/docs/svelte/98-reference/30-compiler-warnings.md', - '**/xx-*.md', + 'docs/svelte/legacy/**/*', + 'docs/svelte/misc/custom-elements', + 'docs/svelte/misc/v4-migration-guide', + 'docs/svelte/misc/v5-migration-guide', + 'docs/svelte/misc/faq', + 'docs/svelte/reference/compiler-errors', + 'docs/svelte/reference/compiler-warnings', + 'docs/svelte/reference/runtime-errors', + 'docs/svelte/reference/runtime-warnings', + 'docs/svelte/reference/svelte-legacy', + '**/xx-*', // SvelteKit ignores - '../../../content/docs/kit/25-build-and-deploy/*adapter-*.md', - '../../../content/docs/kit/25-build-and-deploy/99-writing-adapters.md', - '../../../content/docs/kit/30-advanced/70-packaging.md', - '../../../content/docs/kit/40-best-practices/05-performance.md', - '../../../content/docs/kit/60-appendix/**/*.md' + 'docs/kit/advanced/packaging', + 'docs/kit/appendix/**/*', + 'docs/kit/best-practices/performance', + 'docs/kit/build-and-deploy/*adapter-*', + 'docs/kit/build-and-deploy/writing-adapters' ], minimize: { remove_legacy: true, From bdf0381b90eb9195af6f434175e5d81bb7d63966 Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Thu, 12 Dec 2024 19:54:27 -0500 Subject: [PATCH 43/49] simplify --- apps/svelte.dev/src/lib/server/llms.ts | 105 ++++++------------ .../routes/docs/[...path]/llms.txt/+server.ts | 12 +- .../src/routes/llms-full.txt/+server.ts | 4 +- .../src/routes/llms-small.txt/+server.ts | 3 +- .../svelte.dev/src/routes/llms.txt/+server.ts | 8 +- 5 files changed, 47 insertions(+), 85 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/llms.ts b/apps/svelte.dev/src/lib/server/llms.ts index f2d677737a..2626eeca79 100644 --- a/apps/svelte.dev/src/lib/server/llms.ts +++ b/apps/svelte.dev/src/lib/server/llms.ts @@ -1,12 +1,12 @@ import { minimatch } from 'minimatch'; import { dev } from '$app/environment'; -import { docs, index } from './content'; +import { index } from './content'; interface GenerateLlmContentOptions { prefix?: string; ignore?: string[]; minimize?: Partial; - package?: string; + sections: Section[]; } interface MinimizeOptions { @@ -18,6 +18,11 @@ interface MinimizeOptions { normalize_whitespace: boolean; } +interface Section { + slug: string; + title: string; +} + const defaults: MinimizeOptions = { remove_legacy: false, remove_note_blocks: false, @@ -27,62 +32,48 @@ const defaults: MinimizeOptions = { normalize_whitespace: false }; -export function generate_llm_content(options: GenerateLlmContentOptions = {}): string { +export function generate_llm_content(options: GenerateLlmContentOptions): string { let content = ''; if (options.prefix) { content = `${options.prefix}\n\n`; } - let current_section = ''; - const paths = sort_documentation_paths(); - - for (const path of paths) { - if (!should_include_file_llm_docs(path, options.ignore)) continue; - - // If a specific package is provided, only include its docs - if (options.package) { - if (!path.includes(`docs/${options.package}/`)) continue; - } else { - // For combined content, only include paths that match any package - const doc_type = packages.find((p) => path.includes(`docs/${p}/`)); - if (!doc_type) continue; - - const section = get_documentation_start_title(doc_type); - if (section !== current_section) { - if (current_section) content += '\n'; - content += `${section}\n\n`; - current_section = section; - } + for (const section of options.sections) { + if (options.sections.length > 1) { + content += `${get_documentation_start_title(section)}\n\n`; } - const doc_content = options.minimize - ? minimize_content(index[path].body, options.minimize) - : index[path].body; - if (doc_content.trim() === '') continue; + for (const [path, document] of Object.entries(index)) { + if (!path.startsWith(`docs/${section.slug}`)) continue; + if (!should_include_file_llm_docs(path, options.ignore)) continue; + + const doc_content = options.minimize + ? minimize_content(document.body, options.minimize) + : document.body; + if (doc_content.trim() === '') continue; - content += `\n# ${index[path].metadata.title}\n\n`; - content += doc_content; - content += '\n'; + content += `\n# ${document.metadata.title}\n\n`; + content += doc_content; + content += '\n'; + } } return content; } -export const packages = Object.keys(docs.topics).map((topic) => topic.split('/')[1]); - -export const DOCUMENTATION_NAMES: Record = { - svelte: 'Svelte', - kit: 'SvelteKit', - cli: 'Svelte CLI' -}; +export const sections: Section[] = [ + { slug: 'svelte', title: 'Svelte' }, + { slug: 'kit', title: 'SvelteKit' }, + { slug: 'cli', title: 'the Svelte CLI' } +]; -export function get_documentation_title(type: string): string { - return `This is the developer documentation for ${DOCUMENTATION_NAMES[type]}.`; +export function get_documentation_title(section: Section): string { + return `This is the developer documentation for ${section.title}.`; } -export function get_documentation_start_title(type: string): string { - return `# Start of ${DOCUMENTATION_NAMES[type]} documentation`; +export function get_documentation_start_title(section: Section): string { + return `# Start of ${section.title} documentation`; } function minimize_content(content: string, options?: Partial): string { @@ -133,38 +124,6 @@ function should_include_file_llm_docs(path: string, ignore: string[] = []): bool return true; } -function get_documentation_section_priority(path: string): number { - if (path.includes('docs/svelte/')) return 0; - if (path.includes('docs/kit/')) return 1; - if (path.includes('docs/cli/')) return 2; - return 3; -} - -function sort_documentation_paths(): string[] { - return Object.keys(index).sort((a, b) => { - a = index[a].file; - b = index[b].file; - // First compare by section priority - const priorityA = get_documentation_section_priority(a); - const priorityB = get_documentation_section_priority(b); - if (priorityA !== priorityB) return priorityA - priorityB; - - // Get directory paths - const dirA = a.split('/').slice(0, -1).join('/'); - const dirB = b.split('/').slice(0, -1).join('/'); - - // If in the same directory, prioritize index.md - if (dirA === dirB) { - if (a.endsWith('index.md')) return -1; - if (b.endsWith('index.md')) return 1; - return a.localeCompare(b); - } - - // Otherwise sort by directory path - return dirA.localeCompare(dirB); - }); -} - function remove_quote_blocks(content: string, blockType: string): string { return content .split('\n') diff --git a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts index 64632b2f4d..d757e502e0 100644 --- a/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts @@ -1,21 +1,23 @@ import { error } from '@sveltejs/kit'; -import { generate_llm_content, get_documentation_title, packages } from '$lib/server/llms'; +import { generate_llm_content, get_documentation_title, sections } from '$lib/server/llms'; export const prerender = true; export function entries() { - return packages.map((type) => ({ path: type })); + return sections.map((section) => ({ path: section.slug })); } export function GET({ params }) { const pkg = params.path; - if (!packages.includes(pkg)) { + const section = sections.find((s) => s.slug === pkg); + + if (!section) { error(404, 'Not Found'); } - const prefix = `${get_documentation_title(pkg)}`; - const content = `${prefix}\n\n${generate_llm_content({ package: pkg })}`; + const prefix = `${get_documentation_title(section)}`; + const content = `${prefix}\n\n${generate_llm_content({ sections: [section] })}`; return new Response(content, { status: 200, diff --git a/apps/svelte.dev/src/routes/llms-full.txt/+server.ts b/apps/svelte.dev/src/routes/llms-full.txt/+server.ts index 60f22c2a7b..18f2518797 100644 --- a/apps/svelte.dev/src/routes/llms-full.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms-full.txt/+server.ts @@ -1,9 +1,9 @@ -import { generate_llm_content } from '$lib/server/llms'; +import { generate_llm_content, sections } from '$lib/server/llms'; export const prerender = true; export function GET() { - const content = `This is the full developer documentation for Svelte and SvelteKit.\n\n${generate_llm_content()}`; + const content = `This is the full developer documentation for Svelte and SvelteKit.\n\n${generate_llm_content({ sections })}`; return new Response(content, { status: 200, diff --git a/apps/svelte.dev/src/routes/llms-small.txt/+server.ts b/apps/svelte.dev/src/routes/llms-small.txt/+server.ts index cc613ce008..63eb51f77e 100644 --- a/apps/svelte.dev/src/routes/llms-small.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms-small.txt/+server.ts @@ -1,7 +1,8 @@ -import { generate_llm_content } from '$lib/server/llms'; +import { generate_llm_content, sections } from '$lib/server/llms'; export function GET() { const main_content = generate_llm_content({ + sections, ignore: [ // Svelte ignores 'docs/svelte/legacy/**/*', diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index 4969c4f765..a67010735b 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,14 +1,14 @@ -import { get_documentation_title, packages, DOCUMENTATION_NAMES } from '$lib/server/llms'; +import { get_documentation_title, sections } from '$lib/server/llms'; const DOMAIN = `https://svelte.dev`; export const prerender = true; export function GET() { - const package_docs = packages + const package_docs = sections .map( - (pkg) => - `- [${DOCUMENTATION_NAMES[pkg]} documentation](${DOMAIN}/docs/${pkg}/llms.txt): ${get_documentation_title(pkg)}` + (section) => + `- [${section.title} documentation](${DOMAIN}/docs/${section.slug}/llms.txt): ${get_documentation_title(section)}` ) .join('\n'); From e7bd8d952db2667169c7b815fe3054d4d583cbd2 Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Thu, 12 Dec 2024 19:57:14 -0500 Subject: [PATCH 44/49] unused --- apps/svelte.dev/src/lib/server/llms.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/llms.ts b/apps/svelte.dev/src/lib/server/llms.ts index 2626eeca79..9a2a9ceeb2 100644 --- a/apps/svelte.dev/src/lib/server/llms.ts +++ b/apps/svelte.dev/src/lib/server/llms.ts @@ -3,7 +3,6 @@ import { dev } from '$app/environment'; import { index } from './content'; interface GenerateLlmContentOptions { - prefix?: string; ignore?: string[]; minimize?: Partial; sections: Section[]; @@ -35,10 +34,6 @@ const defaults: MinimizeOptions = { export function generate_llm_content(options: GenerateLlmContentOptions): string { let content = ''; - if (options.prefix) { - content = `${options.prefix}\n\n`; - } - for (const section of options.sections) { if (options.sections.length > 1) { content += `${get_documentation_start_title(section)}\n\n`; From f5df78225bbbf60e366acb88960952d07064eedc Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Thu, 12 Dec 2024 20:02:33 -0500 Subject: [PATCH 45/49] reduce indirection --- apps/svelte.dev/src/lib/server/llms.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/llms.ts b/apps/svelte.dev/src/lib/server/llms.ts index 9a2a9ceeb2..c01e41944e 100644 --- a/apps/svelte.dev/src/lib/server/llms.ts +++ b/apps/svelte.dev/src/lib/server/llms.ts @@ -36,7 +36,7 @@ export function generate_llm_content(options: GenerateLlmContentOptions): string for (const section of options.sections) { if (options.sections.length > 1) { - content += `${get_documentation_start_title(section)}\n\n`; + content += `# Start of ${section.title} documentation\n\n`; } for (const [path, document] of Object.entries(index)) { @@ -67,10 +67,6 @@ export function get_documentation_title(section: Section): string { return `This is the developer documentation for ${section.title}.`; } -export function get_documentation_start_title(section: Section): string { - return `# Start of ${section.title} documentation`; -} - function minimize_content(content: string, options?: Partial): string { // Merge with defaults, but only for properties that are defined const settings: MinimizeOptions = { ...defaults, ...options }; From aa3a4e6fa9ca117afb74bb554283fda570407262 Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Thu, 12 Dec 2024 20:04:34 -0500 Subject: [PATCH 46/49] more --- apps/svelte.dev/src/lib/server/llms.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/llms.ts b/apps/svelte.dev/src/lib/server/llms.ts index c01e41944e..3008cff916 100644 --- a/apps/svelte.dev/src/lib/server/llms.ts +++ b/apps/svelte.dev/src/lib/server/llms.ts @@ -41,7 +41,11 @@ export function generate_llm_content(options: GenerateLlmContentOptions): string for (const [path, document] of Object.entries(index)) { if (!path.startsWith(`docs/${section.slug}`)) continue; - if (!should_include_file_llm_docs(path, options.ignore)) continue; + + if (options.ignore?.some((pattern) => minimatch(path, pattern))) { + if (dev) console.log(`❌ Ignored by pattern: ${path}`); + continue; + } const doc_content = options.minimize ? minimize_content(document.body, options.minimize) @@ -106,15 +110,6 @@ function minimize_content(content: string, options?: Partial): return minimized; } -function should_include_file_llm_docs(path: string, ignore: string[] = []): boolean { - if (ignore.some((pattern) => minimatch(path, pattern))) { - if (dev) console.log(`❌ Ignored by pattern: ${path}`); - return false; - } - - return true; -} - function remove_quote_blocks(content: string, blockType: string): string { return content .split('\n') From 84ed768b2a85afa694c4d02b685f5229a7c7277d Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Thu, 12 Dec 2024 20:09:58 -0500 Subject: [PATCH 47/49] move template into separate .md file --- .../svelte.dev/src/routes/llms.txt/+server.ts | 31 ++++--------------- .../src/routes/llms.txt/template.md | 19 ++++++++++++ 2 files changed, 25 insertions(+), 25 deletions(-) create mode 100644 apps/svelte.dev/src/routes/llms.txt/template.md diff --git a/apps/svelte.dev/src/routes/llms.txt/+server.ts b/apps/svelte.dev/src/routes/llms.txt/+server.ts index a67010735b..9a961557c2 100644 --- a/apps/svelte.dev/src/routes/llms.txt/+server.ts +++ b/apps/svelte.dev/src/routes/llms.txt/+server.ts @@ -1,36 +1,17 @@ import { get_documentation_title, sections } from '$lib/server/llms'; +import template from './template.md?raw'; const DOMAIN = `https://svelte.dev`; export const prerender = true; export function GET() { - const package_docs = sections - .map( - (section) => - `- [${section.title} documentation](${DOMAIN}/docs/${section.slug}/llms.txt): ${get_documentation_title(section)}` - ) - .join('\n'); + const package_docs = sections.map( + (section) => + `- [${section.title} documentation](${DOMAIN}/docs/${section.slug}/llms.txt): ${get_documentation_title(section)}` + ); - const content = `# Svelte Documentation for LLMs - -> Svelte is a UI framework that uses a compiler to let you write breathtakingly concise components that do minimal work in the browser, using languages you already know — HTML, CSS and JavaScript. - -## Documentation Sets - -- [Abridged documentation](${DOMAIN}/llms-small.txt): A minimal version of the Svelte and SvelteKit documentation, with examples and non-essential content removed -- [Complete documentation](${DOMAIN}/llms-full.txt): The complete Svelte and SvelteKit documentation including all examples and additional content - -## Individual Package Documentation - -${package_docs} - -## Notes - -- The abridged documentation excludes legacy compatibility notes, detailed examples, and supplementary information -- The complete documentation includes all content from the official documentation -- Package-specific documentation files contain only the content relevant to that package -- The content is automatically generated from the same source as the official documentation`; + const content = template.replace('%PACKAGE_DOCS%', package_docs.join('\n')); return new Response(content, { headers: { diff --git a/apps/svelte.dev/src/routes/llms.txt/template.md b/apps/svelte.dev/src/routes/llms.txt/template.md new file mode 100644 index 0000000000..aa4b32e1ac --- /dev/null +++ b/apps/svelte.dev/src/routes/llms.txt/template.md @@ -0,0 +1,19 @@ +# Svelte Documentation for LLMs + +> Svelte is a UI framework that uses a compiler to let you write breathtakingly concise components that do minimal work in the browser, using languages you already know — HTML, CSS and JavaScript. + +## Documentation Sets + +- [Abridged documentation](https://svelte.dev/llms-small.txt): A minimal version of the Svelte and SvelteKit documentation, with examples and non-essential content removed +- [Complete documentation](https://svelte.dev/llms-full.txt): The complete Svelte and SvelteKit documentation including all examples and additional content + +## Individual Package Documentation + +%PACKAGE_DOCS% + +## Notes + +- The abridged documentation excludes legacy compatibility notes, detailed examples, and supplementary information +- The complete documentation includes all content from the official documentation +- Package-specific documentation files contain only the content relevant to that package +- The content is automatically generated from the same source as the official documentation From b2233b848d89e8f405d1ec5723e253f0a31ab863 Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Thu, 12 Dec 2024 20:33:57 -0500 Subject: [PATCH 48/49] add a section to /docs --- apps/svelte.dev/src/routes/docs/+page.svelte | 8 ++++ .../src/routes/docs/llms/+page.svelte | 46 +++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 apps/svelte.dev/src/routes/docs/llms/+page.svelte diff --git a/apps/svelte.dev/src/routes/docs/+page.svelte b/apps/svelte.dev/src/routes/docs/+page.svelte index 06929a50e5..6691bf3986 100644 --- a/apps/svelte.dev/src/routes/docs/+page.svelte +++ b/apps/svelte.dev/src/routes/docs/+page.svelte @@ -90,6 +90,14 @@

+ +

I'm a Large Language Model (LLM)

+

+ If you're an artificial intelligence, or trying to teach one how to use Svelte, we offer the + documentation in plaintext format. Beep boop. +

+ +

Help! I'm stuck

diff --git a/apps/svelte.dev/src/routes/docs/llms/+page.svelte b/apps/svelte.dev/src/routes/docs/llms/+page.svelte new file mode 100644 index 0000000000..2d07e49cf2 --- /dev/null +++ b/apps/svelte.dev/src/routes/docs/llms/+page.svelte @@ -0,0 +1,46 @@ + + +

+ +

Docs for LLMs

+ +

+ We support the llms.txt convention for making documentation + available to large language models and the applications that make use of them. +

+ +

Currently, we have the following root-level files...

+ +
    +
  • /llms.txt — a listing of the available files
    • +
  • + /llms-full.txt — complete documentation for Svelte, SvelteKit and + the CLI +
    • +
  • + /llms-small.txt — compressed documentation for use with smaller + context windows +
    • +
+ +

...and package-level documentation:

+ + + +
+ + From a521cc97b193645e1c45eecfa0ed997c50f9a3b3 Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Fri, 13 Dec 2024 06:22:29 -0500 Subject: [PATCH 49/49] advent of svelte --- .../content/blog/2024-12-01-advent-of-svelte.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/svelte.dev/content/blog/2024-12-01-advent-of-svelte.md b/apps/svelte.dev/content/blog/2024-12-01-advent-of-svelte.md index 5a81ef26a5..f569196f7e 100644 --- a/apps/svelte.dev/content/blog/2024-12-01-advent-of-svelte.md +++ b/apps/svelte.dev/content/blog/2024-12-01-advent-of-svelte.md @@ -99,9 +99,13 @@ As of today, you can also return things that _aren't_ built in to the language, - [docs](/docs/kit/hooks#Universal-hooks-transport) - [demo](https://stackblitz.com/edit/sveltejs-kit-template-default-b5zbxomg?file=src%2Fhooks.js) -## Day 13 +## Day 13: rise of the robots -Coming soon! +For those of you using LLMs to help you write code — via Cursor or Copilot or Claude or Bolt or v0 or some other interface — we now publish the documentation in a selection of robot-friendly `llms.txt` files. This is experimental and will evolve over time, but by way of example here's a [snake game](/playground/0de3c1c1a31d47bdbb7c4aa3477a6b46) built by Sonnet 3.5 with no additional prompting. + +Thanks to [Didier Catz](https://x.com/didiercatz) and [Stanislav Khromov](https://bsky.app/profile/khromov.se) for building this! + +- [docs](/docs/llms) ## Day 14