feat: llms.txt

apps/svelte.dev/.env

@@ -3,3 +3,5 @@ SUPABASE_KEY=
 
 GITHUB_CLIENT_ID=
 GITHUB_CLIENT_SECRET=
+
+VERCEL_URL=

apps/svelte.dev/package.json

@@ -67,6 +67,7 @@
 		"lightningcss": "^1.25.1",
 		"magic-string": "^0.30.11",
 		"marked": "^14.1.2",
+		"minimatch": "^10.0.1",
 		"prettier": "^3.3.2",
 		"prettier-plugin-svelte": "^3.2.4",
 		"satori": "^0.10.13",

apps/svelte.dev/src/lib/server/content.ts

@@ -1,6 +1,8 @@
+import { dev } from '$app/environment';
 import { read } from '$app/server';
 import type { Document } from '@sveltejs/site-kit';
 import { create_index } from '@sveltejs/site-kit/server/content';
+import { minimatch } from 'minimatch';
 
 const documents = import.meta.glob<string>('../../../content/**/*.md', {
 	eager: true,
@@ -124,5 +126,198 @@ function create_docs() {
 }
 
 export const docs = create_docs();
-
 export const examples = index.examples.children;
+
+export const packages = Array.from(
+	new Set(
+		Object.keys(docs.topics)
+			.map((topic) => topic.split('/')[1])
+			.filter(Boolean)
+	)
+);
+
+export const DOCUMENTATION_NAMES: Record<string, string> = {
+	svelte: 'Svelte',
+	kit: 'SvelteKit',
+	cli: 'Svelte CLI'
+};
+
+export function get_documentation_title(type: string): string {
+	return `This is the developer documentation for ${DOCUMENTATION_NAMES[type]}.`;
+}
+
+export function get_documentation_start_title(type: string): string {
+	return `# Start of ${DOCUMENTATION_NAMES[type]} documentation`;
+}
+
+interface MinimizeOptions {
+	removeLegacy: boolean;
+	removeNoteBlocks: boolean;
+	removeDetailsBlocks: boolean;
+	removePlaygroundLinks: boolean;
+	removePrettierIgnore: boolean;
+	normalizeWhitespace: boolean;
+}
+
+const defaultOptions: MinimizeOptions = {
+	removeLegacy: false,
+	removeNoteBlocks: false,
+	removeDetailsBlocks: false,
+	removePlaygroundLinks: false,
+	removePrettierIgnore: false,
+	normalizeWhitespace: false
+};
+
+function remove_quote_blocks(content: string, blockType: string): string {
+	return content
+		.split('\n')
+		.reduce((acc: string[], line: string, index: number, lines: string[]) => {
+			// If we find a block (with or without additional text), skip it and all subsequent blockquote lines
+			if (line.trim().startsWith(`> [!${blockType}]`)) {
+				// Skip all subsequent lines that are part of the blockquote
+				let i = index;
+				while (i < lines.length && (lines[i].startsWith('>') || lines[i].trim() === '')) {
+					i++;
+				}
+				// Update the index to skip all these lines
+				index = i - 1;
+				return acc;
+			}
+
+			// Only add the line if it's not being skipped
+			acc.push(line);
+			return acc;
+		}, [])
+		.join('\n');
+}
+
+function minimize_content(content: string, options?: Partial<MinimizeOptions>): string {
+	// Merge with defaults, but only for properties that are defined
+	const settings: MinimizeOptions = options ? { ...defaultOptions, ...options } : defaultOptions;
+
+	let minimized = content;
+
+	if (settings.removeLegacy) {
+		minimized = remove_quote_blocks(minimized, 'LEGACY');
+	}
+
+	if (settings.removeNoteBlocks) {
+		minimized = remove_quote_blocks(minimized, 'NOTE');
+	}
+
+	if (settings.removeDetailsBlocks) {
+		minimized = remove_quote_blocks(minimized, 'DETAILS');
+	}
+
+	if (settings.removePlaygroundLinks) {
+		// Replace playground URLs with /[link] but keep the original link text
+		minimized = minimized.replace(/\[([^\]]+)\]\(\/playground[^)]+\)/g, '[$1](/REMOVED)');
+	}
+
+	if (settings.removePrettierIgnore) {
+		minimized = minimized
+			.split('\n')
+			.filter((line) => line.trim() !== '<!-- prettier-ignore -->')
+			.join('\n');
+	}
+
+	if (settings.normalizeWhitespace) {
+		minimized = minimized.replace(/\s+/g, ' ');
+	}
+
+	minimized = minimized.trim();
+
+	return minimized;
+}
+
+function should_include_file_llm_docs(filename: string, ignore: string[] = []): boolean {
+	const shouldIgnore = ignore.some((pattern) => minimatch(filename, pattern));
+	if (shouldIgnore) {
+		if (dev) console.log(`❌ Ignored by pattern: ${filename}`);
+		return false;
+	}
+
+	return true;
+}
+
+interface GenerateLlmContentOptions {
+	prefix?: string;
+	ignore?: string[];
+	minimize?: Partial<MinimizeOptions>;
+	package?: string;
+}
+
+export function generate_llm_content(options: GenerateLlmContentOptions = {}): string {
+	const { prefix, ignore = [], minimize: minimizeOptions, package: pkg } = options;
+
+	let content = '';
+	if (prefix) {
+		content = `${prefix}\n\n`;
+	}
+
+	let current_section = '';
+	const paths = sort_documentation_paths();
+
+	for (const path of paths) {
+		if (!should_include_file_llm_docs(path, ignore)) continue;
+
+		// If a specific package is provided, only include its docs
+		if (pkg) {
+			if (!path.includes(`docs/${pkg}/`)) continue;
+		} else {
+			// For combined content, only include paths that match any package
+			const doc_type = packages.find((p) => path.includes(`docs/${p}/`));
+			if (!doc_type) continue;
+
+			const section = get_documentation_start_title(doc_type);
+			if (section !== current_section) {
+				if (current_section) content += '\n';
+				content += `${section}\n\n`;
+				current_section = section;
+			}
+		}
+
+		const docContent = minimizeOptions
+			? minimize_content(index[path].body, minimizeOptions)
+			: index[path].body;
+		if (docContent.trim() === '') continue;
+
+		content += `\n# ${index[path].metadata.title}\n\n`;
+		content += docContent;
+		content += '\n';
+	}
+
+	return content;
+}
+
+function get_documentation_section_priority(path: string): number {
+	if (path.includes('docs/svelte/')) return 0;
+	if (path.includes('docs/kit/')) return 1;
+	if (path.includes('docs/cli/')) return 2;
+	return 3;
+}
+
+function sort_documentation_paths(): string[] {
+	return Object.keys(index).sort((a, b) => {
+		a = index[a].file;
+		b = index[b].file;
+		// First compare by section priority
+		const priorityA = get_documentation_section_priority(a);
+		const priorityB = get_documentation_section_priority(b);
+		if (priorityA !== priorityB) return priorityA - priorityB;
+
+		// Get directory paths
+		const dirA = a.split('/').slice(0, -1).join('/');
+		const dirB = b.split('/').slice(0, -1).join('/');
+
+		// If in the same directory, prioritize index.md
+		if (dirA === dirB) {
+			if (a.endsWith('index.md')) return -1;
+			if (b.endsWith('index.md')) return 1;
+			return a.localeCompare(b);
+		}
+
+		// Otherwise sort by directory path
+		return dirA.localeCompare(dirB);
+	});
+}

apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts

@@ -0,0 +1,27 @@
+import { error } from '@sveltejs/kit';
+import { generate_llm_content, get_documentation_title, packages } from '$lib/server/content';
+
+export const prerender = true;
+
+export function entries() {
+	return packages.map((type) => ({ path: type }));
+}
+
+export function GET({ params }) {
+	const pkg = params.path;
+
+	if (!packages.includes(pkg)) {
+		error(404, 'Not Found');
+	}
+
+	const prefix = `<SYSTEM>${get_documentation_title(pkg)}</SYSTEM>`;
+	const content = `${prefix}\n\n${generate_llm_content({ package: pkg })}`;
+
+	return new Response(content, {
+		status: 200,
+		headers: {
+			'Content-Type': 'text/plain; charset=utf-8',
+			'Cache-Control': 'public, max-age=3600'
+		}
+	});
+}

apps/svelte.dev/src/routes/llms-full.txt/+server.ts

@@ -0,0 +1,15 @@
+import { generate_llm_content } from '$lib/server/content';
+
+export const prerender = true;
+
+export function GET() {
+	const content = `<SYSTEM>This is the full developer documentation for Svelte and SvelteKit.</SYSTEM>\n\n${generate_llm_content()}`;
+
+	return new Response(content, {
+		status: 200,
+		headers: {
+			'Content-Type': 'text/plain; charset=utf-8',
+			'Cache-Control': 'public, max-age=3600'
+		}
+	});
+}

apps/svelte.dev/src/routes/llms-small.txt/+server.ts

@@ -0,0 +1,47 @@
+import { generate_llm_content } from '$lib/server/content';
+
+export function GET() {
+	const main_content = generate_llm_content({
+		ignore: [
+			// Svelte ignores
+			'../../../content/docs/svelte/07-misc/04-custom-elements.md',
+			'../../../content/docs/svelte/07-misc/06-v4-migration-guide.md',
+			'../../../content/docs/svelte/07-misc/07-v5-migration-guide.md',
+			'../../../content/docs/svelte/07-misc/99-faq.md',
+			'../../../content/docs/svelte/07-misc/xx-reactivity-indepth.md',
+			'../../../content/docs/svelte/98-reference/21-svelte-legacy.md',
+			'../../../content/docs/svelte/99-legacy/**/*.md',
+			'../../../content/docs/svelte/98-reference/30-runtime-errors.md',
+			'../../../content/docs/svelte/98-reference/30-runtime-warnings.md',
+			'../../../content/docs/svelte/98-reference/30-compiler-errors.md',
+			'../../../content/docs/svelte/98-reference/30-compiler-warnings.md',
+			'**/xx-*.md',
+
+			// SvelteKit ignores
+			'../../../content/docs/kit/25-build-and-deploy/*adapter-*.md',
+			'../../../content/docs/kit/25-build-and-deploy/99-writing-adapters.md',
+			'../../../content/docs/kit/30-advanced/70-packaging.md',
+			'../../../content/docs/kit/40-best-practices/05-performance.md',
+			'../../../content/docs/kit/60-appendix/**/*.md'
+		],
+		minimize: {
+			removeLegacy: true,
+			removeNoteBlocks: true,
+			removeDetailsBlocks: true,
+			removePlaygroundLinks: true,
+			removePrettierIgnore: true,
+			normalizeWhitespace: true
+		}
+	});
+	const content = `<SYSTEM>This is the abridged developer documentation for Svelte and SvelteKit.</SYSTEM>\n\n${main_content}`;
+
+	return new Response(content, {
+		status: 200,
+		headers: {
+			'Content-Type': 'text/plain; charset=utf-8',
+			'Cache-Control': 'public, max-age=3600'
+		}
+	});
+}
+
+export const prerender = true;

apps/svelte.dev/src/routes/llms.txt/+server.ts

@@ -0,0 +1,42 @@
+import { VERCEL_URL } from '$env/static/private';
+import { get_documentation_title, packages, DOCUMENTATION_NAMES } from '$lib/server/content';
+
+const DOMAIN = VERCEL_URL ? `https://${VERCEL_URL}` : '';
+
+export const prerender = true;
+
+export function GET() {
+	const package_docs = packages
+		.map(
+			(pkg) =>
+				`- [${DOCUMENTATION_NAMES[pkg]} documentation](${DOMAIN}/docs/${pkg}/llms.txt): ${get_documentation_title(pkg)}`
+		)
+		.join('\n');
+
+	const content = `# Svelte Documentation for LLMs
+
+> Svelte is a UI framework that uses a compiler to let you write breathtakingly concise components that do minimal work in the browser, using languages you already know — HTML, CSS and JavaScript.
+
+## Documentation Sets
+
+- [Abridged documentation](${DOMAIN}/llms-small.txt): A minimal version of the Svelte and SvelteKit documentation, with examples and non-essential content removed
+- [Complete documentation](${DOMAIN}/llms-full.txt): The complete Svelte and SvelteKit documentation including all examples and additional content
+
+## Individual Package Documentation
+
+${package_docs}
+
+## Notes
+
+- The abridged documentation excludes legacy compatibility notes, detailed examples, and supplementary information
+- The complete documentation includes all content from the official documentation
+- Package-specific documentation files contain only the content relevant to that package
+- The content is automatically generated from the same source as the official documentation`;
+
+	return new Response(content, {
+		headers: {
+			'Content-Type': 'text/plain; charset=utf-8',
+			'Cache-Control': 'public, max-age=3600'
+		}
+	});
+}