chore: add /llms.txt containg raw markdown for each docs page (#1322)

That way LLMs have it easier getting a clean version of the docs

Author: dummdidumm

apps/svelte.dev/src/lib/components/PageControls.svelte

@@ -1,8 +1,10 @@
 <script lang="ts">
+	import { page } from '$app/state';
 	import { Icon } from '@sveltejs/site-kit/components';
 
 	interface Props {
 		repo: string;
+		llms?: boolean;
 		prev: null | {
 			path: string;
 			title: string;
@@ -13,13 +15,18 @@
 		};
 	}
 
-	let { repo, prev, next }: Props = $props();
+	let { repo, prev, next, llms }: Props = $props();
 </script>
 
 <p class="edit">
 	<a href={repo}>
 		<Icon name="edit" /> Edit this page on GitHub
 	</a>
+	{#if llms}
+		<a href={page.url.pathname.replace(/\/$/, '') + '/llms.txt'}>
+			<Icon name="contents" /> llms.txt
+		</a>
+	{/if}
 </p>
 
 <div class="controls">
@@ -44,9 +51,13 @@
 		position: relative;
 		margin: 6rem 0 2rem 0;
 		font: var(--sk-font-ui-small);
+		display: flex;
 
 		a {
 			text-decoration: none;
+			&:first-of-type {
+				margin-right: 5rem;
+			}
 		}
 
 		:global(svg) {

apps/svelte.dev/src/lib/server/llms.ts

@@ -5,7 +5,7 @@ import { index } from './content';
 interface GenerateLlmContentOptions {
 	ignore?: string[];
 	minimize?: Partial<MinimizeOptions>;
-	sections: Section[];
+	topics: Topic[];
 }
 
 interface MinimizeOptions {
@@ -17,7 +17,7 @@ interface MinimizeOptions {
 	normalize_whitespace: boolean;
 }
 
-interface Section {
+interface Topic {
 	slug: string;
 	title: string;
 }
@@ -34,13 +34,13 @@ const defaults: MinimizeOptions = {
 export function generate_llm_content(options: GenerateLlmContentOptions): string {
 	let content = '';
 
-	for (const section of options.sections) {
-		if (options.sections.length > 1) {
-			content += `# Start of ${section.title} documentation\n\n`;
+	for (const topic of options.topics) {
+		if (options.topics.length > 1) {
+			content += `# Start of ${topic.title} documentation\n\n`;
 		}
 
 		for (const [path, document] of Object.entries(index)) {
-			if (!path.startsWith(`docs/${section.slug}`)) continue;
+			if (!path.startsWith(`docs/${topic.slug}`)) continue;
 
 			if (options.ignore?.some((pattern) => minimatch(path, pattern))) {
 				if (dev) console.log(`❌ Ignored by pattern: ${path}`);
@@ -61,14 +61,14 @@ export function generate_llm_content(options: GenerateLlmContentOptions): string
 	return content;
 }
 
-export const sections: Section[] = [
+export const topics: Topic[] = [
 	{ slug: 'svelte', title: 'Svelte' },
 	{ slug: 'kit', title: 'SvelteKit' },
 	{ slug: 'cli', title: 'the Svelte CLI' }
 ];
 
-export function get_documentation_title(section: Section): string {
-	return `This is the developer documentation for ${section.title}.`;
+export function get_documentation_title(topic: Topic): string {
+	return `This is the developer documentation for ${topic.title}.`;
 }
 
 function minimize_content(content: string, options?: Partial<MinimizeOptions>): string {

apps/svelte.dev/src/routes/docs/[topic]/[...path]/+page.svelte

@@ -83,6 +83,7 @@
 	</div>
 
 	<PageControls
+		llms
 		{repo}
 		prev={data.document.prev && {
 			title: data.document.prev.title,

apps/svelte.dev/src/routes/docs/[topic]/[...path]/llms.txt/+server.ts

@@ -1,32 +1,46 @@
 import { error } from '@sveltejs/kit';
-import { generate_llm_content, get_documentation_title, sections } from '$lib/server/llms';
+import { docs } from '$lib/server/content.js';
+import { generate_llm_content, get_documentation_title, topics } from '$lib/server/llms';
 
 export const prerender = true;
 
 export function entries() {
-	return sections.map((section) => {
-		const [topic, ...rest] = section.slug.split('/');
-		return { topic, path: rest.join('/') };
+	return topics.map((topic) => {
+		return { topic: topic.slug, path: '' };
 	});
 }
 
 export function GET({ params }) {
-	const pkg = params.path ? `${params.topic}/${params.path}` : params.topic;
+	if (params.path) {
+		const page = docs.pages[`docs/${params.topic}/${params.path}`];
 
-	const section = sections.find((s) => s.slug === pkg);
-
-	if (!section) {
-		error(404, 'Not Found');
-	}
+		if (!page) {
+			error(404, 'Not Found');
+		}
 
-	const prefix = `<SYSTEM>${get_documentation_title(section)}</SYSTEM>`;
-	const content = `${prefix}\n\n${generate_llm_content({ sections: [section] })}`;
+		return new Response(page.body, {
+			status: 200,
+			headers: {
+				'Content-Type': 'text/plain; charset=utf-8',
+				'Cache-Control': 'public, max-age=3600'
+			}
+		});
+	} else {
+		const topic = topics.find((s) => s.slug === params.topic);
 
-	return new Response(content, {
-		status: 200,
-		headers: {
-			'Content-Type': 'text/plain; charset=utf-8',
-			'Cache-Control': 'public, max-age=3600'
+		if (!topic) {
+			error(404, 'Not Found');
 		}
-	});
+
+		const prefix = `<SYSTEM>${get_documentation_title(topic)}</SYSTEM>`;
+		const content = `${prefix}\n\n${generate_llm_content({ topics: [topic] })}`;
+
+		return new Response(content, {
+			status: 200,
+			headers: {
+				'Content-Type': 'text/plain; charset=utf-8',
+				'Cache-Control': 'public, max-age=3600'
+			}
+		});
+	}
 }

apps/svelte.dev/src/routes/llms-full.txt/+server.ts

@@ -1,9 +1,9 @@
-import { generate_llm_content, sections } from '$lib/server/llms';
+import { generate_llm_content, topics } from '$lib/server/llms';
 
 export const prerender = true;
 
 export function GET() {
-	const content = `<SYSTEM>This is the full developer documentation for Svelte and SvelteKit.</SYSTEM>\n\n${generate_llm_content({ sections })}`;
+	const content = `<SYSTEM>This is the full developer documentation for Svelte and SvelteKit.</SYSTEM>\n\n${generate_llm_content({ topics })}`;
 
 	return new Response(content, {
 		status: 200,

apps/svelte.dev/src/routes/llms-medium.txt/+server.ts

@@ -1,8 +1,8 @@
-import { generate_llm_content, sections } from '$lib/server/llms';
+import { generate_llm_content, topics } from '$lib/server/llms';
 
 export function GET() {
 	const main_content = generate_llm_content({
-		sections,
+		topics,
 		ignore: [
 			// Svelte ignores
 			'docs/svelte/legacy/**/*',

apps/svelte.dev/src/routes/llms.txt/+server.ts

@@ -1,14 +1,14 @@
-import { get_documentation_title, sections } from '$lib/server/llms';
+import { get_documentation_title, topics } from '$lib/server/llms';
 import template from './template.md?raw';
 
 const DOMAIN = `https://svelte.dev`;
 
 export const prerender = true;
 
 export function GET() {
-	const package_docs = sections.map(
-		(section) =>
-			`- [${section.title} documentation](${DOMAIN}/docs/${section.slug}/llms.txt): ${get_documentation_title(section)}`
+	const package_docs = topics.map(
+		(topic) =>
+			`- [${topic.title} documentation](${DOMAIN}/docs/${topic.slug}/llms.txt): ${get_documentation_title(topic)}`
 	);
 
 	const content = template.replace('%PACKAGE_DOCS%', package_docs.join('\n'));