feat: llms.txt

apps/svelte.dev/package.json

@@ -68,6 +68,7 @@
 		"lightningcss": "^1.25.1",
 		"magic-string": "^0.30.11",
 		"marked": "^14.1.2",
+		"minimatch": "^10.0.1",
 		"prettier": "^3.3.2",
 		"prettier-plugin-svelte": "^3.2.4",
 		"satori": "^0.10.13",

apps/svelte.dev/src/lib/server/content.ts

@@ -1,6 +1,8 @@
+import { dev } from '$app/environment';
 import { read } from '$app/server';
 import type { Document } from '@sveltejs/site-kit';
 import { create_index } from '@sveltejs/site-kit/server/content';
+import { minimatch } from 'minimatch';
 
 const documents = import.meta.glob<string>('../../../content/**/*.md', {
 	eager: true,
@@ -14,6 +16,12 @@ const assets = import.meta.glob<string>('../../../content/**/+assets/**', {
 	import: 'default'
 });
 
+export const documentsContent = import.meta.glob<string>('../../../content/**/*.md', {
+	eager: true,
+	query: '?raw',
+	import: 'default'
+});
+
 // https://github.com/vitejs/vite/issues/17453
 export const index = await create_index(documents, assets, '../../../content', read);
 
@@ -123,3 +131,210 @@ function create_docs() {
 export const docs = create_docs();
 
 export const examples = index.examples.children;
+
+function getSectionPriority(path: string): number {
+	if (path.includes('/docs/svelte/')) return 0;
+	if (path.includes('/docs/kit/')) return 1;
+	if (path.includes('/docs/cli/')) return 2;
+	return 3;
+}
+
+export function sortPaths(paths: string[]): string[] {
+	return paths.sort((a, b) => {
+		// First compare by section priority
+		const priorityA = getSectionPriority(a);
+		const priorityB = getSectionPriority(b);
+		if (priorityA !== priorityB) return priorityA - priorityB;
+
+		// Get directory paths
+		const dirA = a.split('/').slice(0, -1).join('/');
+		const dirB = b.split('/').slice(0, -1).join('/');
+
+		// If in the same directory, prioritize index.md
+		if (dirA === dirB) {
+			if (a.endsWith('index.md')) return -1;
+			if (b.endsWith('index.md')) return 1;
+			return a.localeCompare(b);
+		}
+
+		// Otherwise sort by directory path
+		return dirA.localeCompare(dirB);
+	});
+}
+
+export const packages = ['svelte', 'kit', 'cli'] as const;
+export type Package = (typeof packages)[number];
+
+const DOCUMENTATION_NAMES: Record<Package, string> = {
+	svelte: 'Svelte',
+	kit: 'SvelteKit',
+	cli: 'Svelte CLI'
+};
+
+export function getDocumentationTitle(type: Package): string {
+	return `This is the developer documentation for ${DOCUMENTATION_NAMES[type]}.`;
+}
+
+export function getDocumentationStartTitle(type: Package): string {
+	return `# Start of ${DOCUMENTATION_NAMES[type]} documentation`;
+}
+
+export function filterDocsByPackage(
+	allDocs: Record<string, string>,
+	type: Package
+): Record<string, string> {
+	const filtered: Record<string, string> = {};
+
+	for (const [path, content] of Object.entries(allDocs)) {
+		if (path.toLowerCase().includes(`/docs/${type}/`)) {
+			filtered[path] = content;
+		}
+	}
+
+	return filtered;
+}
+
+interface MinimizeOptions {
+	removeLegacy: boolean;
+	removeNoteBlocks: boolean;
+	removeDetailsBlocks: boolean;
+	removePlaygroundLinks: boolean;
+	removePrettierIgnore: boolean;
+	normalizeWhitespace: boolean;
+}
+
+const defaultOptions: MinimizeOptions = {
+	removeLegacy: false,
+	removeNoteBlocks: false,
+	removeDetailsBlocks: false,
+	removePlaygroundLinks: false,
+	removePrettierIgnore: false,
+	normalizeWhitespace: false
+};
+
+function removeQuoteBlocks(content: string, blockType: string): string {
+	return content
+		.split('\n')
+		.reduce((acc: string[], line: string, index: number, lines: string[]) => {
+			// If we find a block (with or without additional text), skip it and all subsequent blockquote lines
+			if (line.trim().startsWith(`> [!${blockType}]`)) {
+				// Skip all subsequent lines that are part of the blockquote
+				let i = index;
+				while (i < lines.length && (lines[i].startsWith('>') || lines[i].trim() === '')) {
+					i++;
+				}
+				// Update the index to skip all these lines
+				index = i - 1;
+				return acc;
+			}
+
+			// Only add the line if it's not being skipped
+			acc.push(line);
+			return acc;
+		}, [])
+		.join('\n');
+}
+
+function minimizeContent(content: string, options?: Partial<MinimizeOptions>): string {
+	// Merge with defaults, but only for properties that are defined
+	const settings: MinimizeOptions = options ? { ...defaultOptions, ...options } : defaultOptions;
+
+	let minimized = content;
+
+	if (settings.removeLegacy) {
+		minimized = removeQuoteBlocks(minimized, 'LEGACY');
+	}
+
+	if (settings.removeNoteBlocks) {
+		minimized = removeQuoteBlocks(minimized, 'NOTE');
+	}
+
+	if (settings.removeDetailsBlocks) {
+		minimized = removeQuoteBlocks(minimized, 'DETAILS');
+	}
+
+	if (settings.removePlaygroundLinks) {
+		// Replace playground URLs with /[link] but keep the original link text
+		minimized = minimized.replace(/\[([^\]]+)\]\(\/playground[^)]+\)/g, '[$1](/REMOVED)');
+	}
+
+	if (settings.removePrettierIgnore) {
+		minimized = minimized
+			.split('\n')
+			.filter((line) => line.trim() !== '<!-- prettier-ignore -->')
+			.join('\n');
+	}
+
+	if (settings.normalizeWhitespace) {
+		minimized = minimized.replace(/\s+/g, ' ');
+	}
+
+	minimized = minimized.trim();
+
+	return minimized;
+}
+
+function shouldIncludeFile(filename: string, ignore: string[] = []): boolean {
+	const shouldIgnore = ignore.some((pattern) => minimatch(filename, pattern));
+	if (shouldIgnore) {
+		if (dev) console.log(`❌ Ignored by pattern: ${filename}`);
+		return false;
+	}
+
+	return true;
+}
+
+export function generateLlmContent(
+	filteredDocs: Record<string, string>,
+	type: Package,
+	minimizeOptions?: Partial<MinimizeOptions>
+): string {
+	let content = `<SYSTEM>${getDocumentationTitle(type)}</SYSTEM>\n\n`;
+
+	const paths = sortPaths(Object.keys(filteredDocs));
+
+	for (const path of paths) {
+		content += `# ${path.replace('../../../content/', '')}\n\n`;
+		const docContent = minimizeOptions
+			? minimizeContent(filteredDocs[path], minimizeOptions)
+			: filteredDocs[path];
+		content += docContent;
+		content += '\n';
+	}
+
+	return content;
+}
+
+export function generateCombinedContent(
+	documentsContent: Record<string, string>,
+	ignore: string[] = [],
+	minimizeOptions?: Partial<MinimizeOptions>
+): string {
+	let content = '';
+	let currentSection = '';
+	const paths = sortPaths(Object.keys(documentsContent));
+
+	for (const path of paths) {
+		// Skip files that match ignore patterns
+		if (!shouldIncludeFile(path, ignore)) continue;
+
+		const docType = packages.find((pkg) => path.includes(`/docs/${pkg}/`));
+		if (!docType) continue;
+
+		const section = getDocumentationStartTitle(docType);
+		if (section !== currentSection) {
+			if (currentSection) content += '\n';
+			content += `${section}\n\n`;
+			currentSection = section;
+		}
+
+		content += `## ${path.replace('../../../content/', '')}\n\n`;
+		const docContent = minimizeOptions
+			? minimizeContent(documentsContent[path], minimizeOptions)
+			: documentsContent[path];
+		content += docContent;
+		content += '\n';
+	}
+
+	return content;
+}

apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts

@@ -0,0 +1,40 @@
+import type { RequestHandler } from './$types';
+import type { EntryGenerator } from './$types';
+import { error } from '@sveltejs/kit';
+import {
+	documentsContent,
+	filterDocsByPackage,
+	generateLlmContent,
+	packages,
+	type Package
+} from '$lib/server/content';
+
+export const prerender = true;
+
+export const entries: EntryGenerator = () => {
+	return packages.map((type) => ({ path: type }));
+};
+
+export const GET: RequestHandler = async ({ params }) => {
+	const packageType = params.path;
+
+	if (!packages.includes(packageType as Package)) {
+		error(404, 'Not Found');
+	}
+
+	const filteredDocs = filterDocsByPackage(documentsContent, packageType as Package);
+
+	if (Object.keys(filteredDocs).length === 0) {
+		error(404, 'No documentation found for this package');
+	}
+
+	const content = generateLlmContent(filteredDocs, packageType as Package);
+
+	return new Response(content, {
+		status: 200,
+		headers: {
+			'Content-Type': 'text/plain; charset=utf-8',
+			'Cache-Control': 'public, max-age=3600'
+		}
+	});
+};

apps/svelte.dev/src/routes/llms-full.txt/+server.ts

@@ -0,0 +1,18 @@
+import type { RequestHandler } from './$types';
+import { documentsContent, generateCombinedContent } from '$lib/server/content';
+
+const PREFIX = 'This is the full developer documentation for Svelte and SvelteKit.';
+
+export const GET: RequestHandler = async () => {
+	const content = `${PREFIX}\n\n${generateCombinedContent(documentsContent)}`;
+
+	return new Response(content, {
+		status: 200,
+		headers: {
+			'Content-Type': 'text/plain; charset=utf-8',
+			'Cache-Control': 'public, max-age=3600'
+		}
+	});
+};
+
+export const prerender = true;

apps/svelte.dev/src/routes/llms.txt/+server.ts

@@ -0,0 +1,50 @@
+import type { RequestHandler } from './$types';
+import { documentsContent, generateCombinedContent } from '$lib/server/content';
+
+const PREFIX = 'This is the abridged developer documentation for Svelte and SvelteKit.';
+
+export const GET: RequestHandler = async () => {
+	const content = `${PREFIX}\n\n${generateCombinedContent(
+		documentsContent,
+		[
+			// Svelte ignores
+			'../../../content/docs/svelte/07-misc/04-custom-elements.md',
+			'../../../content/docs/svelte/07-misc/06-v4-migration-guide.md',
+			'../../../content/docs/svelte/07-misc/07-v5-migration-guide.md',
+			'../../../content/docs/svelte/07-misc/99-faq.md',
+			'../../../content/docs/svelte/07-misc/xx-reactivity-indepth.md',
+			'../../../content/docs/svelte/98-reference/21-svelte-legacy.md',
+			'../../../content/docs/svelte/99-legacy/**/*.md',
+			'../../../content/docs/svelte/98-reference/30-runtime-errors.md',
+			'../../../content/docs/svelte/98-reference/30-runtime-warnings.md',
+			'../../../content/docs/svelte/98-reference/30-compiler-errors.md',
+			'../../../content/docs/svelte/98-reference/30-compiler-warnings.md',
+			'**/xx-*.md',
+
+			// SvelteKit ignores
+			'../../../content/docs/kit/25-build-and-deploy/*adapter-*.md',
+			'../../../content/docs/kit/25-build-and-deploy/99-writing-adapters.md',
+			'../../../content/docs/kit/30-advanced/70-packaging.md',
+			'../../../content/docs/kit/40-best-practices/05-performance.md',
+			'../../../content/docs/kit/60-appendix/**/*.md'
+		],
+		{
+			removeLegacy: true,
+			removeNoteBlocks: true,
+			removeDetailsBlocks: true,
+			removePlaygroundLinks: true,
+			removePrettierIgnore: true,
+			normalizeWhitespace: true
+		}
+	)}`;
+
+	return new Response(content, {
+		status: 200,
+		headers: {
+			'Content-Type': 'text/plain; charset=utf-8',
+			'Cache-Control': 'public, max-age=3600'
+		}
+	});
+};
+
+export const prerender = true;