Filter llms.txt

Author: khromov

apps/svelte.dev/package.json

@@ -68,6 +68,7 @@
 		"lightningcss": "^1.25.1",
 		"magic-string": "^0.30.11",
 		"marked": "^14.1.2",
+		"minimatch": "^10.0.1",
 		"prettier": "^3.3.2",
 		"prettier-plugin-svelte": "^3.2.4",
 		"satori": "^0.10.13",

apps/svelte.dev/src/lib/server/content.ts

@@ -1,6 +1,8 @@
+import { dev } from '$app/environment';
 import { read } from '$app/server';
 import type { Document } from '@sveltejs/site-kit';
 import { create_index } from '@sveltejs/site-kit/server/content';
+import { minimatch } from 'minimatch';
 
 const documents = import.meta.glob<string>('../../../content/**/*.md', {
 	eager: true,
@@ -272,6 +274,16 @@ function minimizeContent(content: string, options?: Partial<MinimizeOptions>): s
 	return minimized;
 }
 
+function shouldIncludeFile(filename: string, ignore: string[] = []): boolean {
+	const shouldIgnore = ignore.some((pattern) => minimatch(filename, pattern));
+	if (shouldIgnore) {
+		if (dev) console.log(`❌ Ignored by pattern: ${filename}`);
+		return false;
+	}
+
+	return true;
+}
+
 export function generateLlmContent(
 	filteredDocs: Record<string, string>,
 	type: Package,
@@ -295,13 +307,17 @@ export function generateLlmContent(
 
 export function generateCombinedContent(
 	documentsContent: Record<string, string>,
+	ignore: string[] = [],
 	minimizeOptions?: Partial<MinimizeOptions>
 ): string {
 	let content = '';
 	let currentSection = '';
 	const paths = sortPaths(Object.keys(documentsContent));
 
 	for (const path of paths) {
+		// Skip files that match ignore patterns
+		if (!shouldIncludeFile(path, ignore)) continue;
+
 		const docType = packages.find((pkg) => path.includes(`/docs/${pkg}/`));
 		if (!docType) continue;
 

apps/svelte.dev/src/routes/llms.txt/+server.ts

@@ -4,14 +4,39 @@ import { documentsContent, generateCombinedContent } from '$lib/server/content';
 const PREFIX = 'This is the abridged developer documentation for Svelte and SvelteKit.';
 
 export const GET: RequestHandler = async () => {
-	const content = `${PREFIX}\n\n${generateCombinedContent(documentsContent, {
-		removeLegacy: true,
-		removeNoteBlocks: true,
-		removeDetailsBlocks: true,
-		removePlaygroundLinks: true,
-		removePrettierIgnore: true,
-		normalizeWhitespace: true
-	})}`;
+	const content = `${PREFIX}\n\n${generateCombinedContent(
+		documentsContent,
+		[
+			// Svelte ignores
+			'../../../content/docs/svelte/07-misc/04-custom-elements.md',
+			'../../../content/docs/svelte/07-misc/06-v4-migration-guide.md',
+			'../../../content/docs/svelte/07-misc/07-v5-migration-guide.md',
+			'../../../content/docs/svelte/07-misc/99-faq.md',
+			'../../../content/docs/svelte/07-misc/xx-reactivity-indepth.md',
+			'../../../content/docs/svelte/98-reference/21-svelte-legacy.md',
+			'../../../content/docs/svelte/99-legacy/**/*.md',
+			'../../../content/docs/svelte/98-reference/30-runtime-errors.md',
+			'../../../content/docs/svelte/98-reference/30-runtime-warnings.md',
+			'../../../content/docs/svelte/98-reference/30-compiler-errors.md',
+			'../../../content/docs/svelte/98-reference/30-compiler-warnings.md',
+			'**/xx-*.md',
+
+			// SvelteKit ignores
+			'../../../content/docs/kit/25-build-and-deploy/*adapter-*.md',
+			'../../../content/docs/kit/25-build-and-deploy/99-writing-adapters.md',
+			'../../../content/docs/kit/30-advanced/70-packaging.md',
+			'../../../content/docs/kit/40-best-practices/05-performance.md',
+			'../../../content/docs/kit/60-appendix/**/*.md'
+		],
+		{
+			removeLegacy: true,
+			removeNoteBlocks: true,
+			removeDetailsBlocks: true,
+			removePlaygroundLinks: true,
+			removePrettierIgnore: true,
+			normalizeWhitespace: true
+		}
+	)}`;
 
 	return new Response(content, {
 		status: 200,