Skip to content

Commit 99b16ef

Browse files
khromovdummdidummRich-Harris
authored
feat: llms.txt (#979)
* wip * Update +server.ts * Update +server.ts * wip * wip * Update +server.ts * Update +server.ts * wip * cleanup * wip * refactor * cleanupo * Update +server.ts * Update content.ts * wip * Create +server.ts * minimize llms.txt * Filter llms.txt * clean up * package * chore: naming * Update +server.ts * Dynamic path names * clean up * fix * under_score * code style * use real document titles, filter out empty files * move llms.txt to llms-small.txt * llms.txt index * Update +server.ts * Update +server.ts * Fix index * fix * revert VERCEL_URL usage * Update apps/svelte.dev/src/lib/server/content.ts Co-authored-by: Rich Harris <[email protected]> * move llm stuff into its own module * revert whitespace changes * snake_case * tweak * snake_case etc * make ignores work * simplify * unused * reduce indirection * more * move template into separate .md file * add a section to /docs * advent of svelte --------- Co-authored-by: Simon Holthausen <[email protected]> Co-authored-by: Rich Harris <[email protected]>
1 parent d2c90f6 commit 99b16ef

File tree

11 files changed

+338
-2
lines changed

11 files changed

+338
-2
lines changed

‎apps/svelte.dev/content/blog/2024-12-01-advent-of-svelte.md‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,9 +99,13 @@ As of today, you can also return things that _aren't_ built in to the language,
9999
- [docs](/docs/kit/hooks#Universal-hooks-transport)
100100
- [demo](https://stackblitz.com/edit/sveltejs-kit-template-default-b5zbxomg?file=src%2Fhooks.js)
101101

102-
## Day 13
102+
## Day 13: rise of the robots
103103

104-
Coming soon!
104+
For those of you using LLMs to help you write code — via Cursor or Copilot or Claude or Bolt or v0 or some other interface — we now publish the documentation in a selection of robot-friendly `llms.txt` files. This is experimental and will evolve over time, but by way of example here's a [snake game](/playground/0de3c1c1a31d47bdbb7c4aa3477a6b46) built by Sonnet 3.5 with no additional prompting.
105+
106+
Thanks to [Didier Catz](https://x.com/didiercatz) and [Stanislav Khromov](https://bsky.app/profile/khromov.se) for building this!
107+
108+
- [docs](/docs/llms)
105109

106110
## Day 14
107111

‎apps/svelte.dev/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
"lightningcss": "^1.25.1",
6868
"magic-string": "^0.30.11",
6969
"marked": "^14.1.2",
70+
"minimatch": "^10.0.1",
7071
"prettier": "^3.3.2",
7172
"prettier-plugin-svelte": "^3.2.4",
7273
"satori": "^0.10.13",

‎apps/svelte.dev/src/lib/server/llms.ts‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
import { minimatch } from 'minimatch';
2+
import { dev } from '$app/environment';
3+
import { index } from './content';
4+
5+
interface GenerateLlmContentOptions {
6+
ignore?: string[];
7+
minimize?: Partial<MinimizeOptions>;
8+
sections: Section[];
9+
}
10+
11+
interface MinimizeOptions {
12+
remove_legacy: boolean;
13+
remove_note_blocks: boolean;
14+
remove_details_blocks: boolean;
15+
remove_playground_links: boolean;
16+
remove_prettier_ignore: boolean;
17+
normalize_whitespace: boolean;
18+
}
19+
20+
interface Section {
21+
slug: string;
22+
title: string;
23+
}
24+
25+
const defaults: MinimizeOptions = {
26+
remove_legacy: false,
27+
remove_note_blocks: false,
28+
remove_details_blocks: false,
29+
remove_playground_links: false,
30+
remove_prettier_ignore: false,
31+
normalize_whitespace: false
32+
};
33+
34+
export function generate_llm_content(options: GenerateLlmContentOptions): string {
35+
let content = '';
36+
37+
for (const section of options.sections) {
38+
if (options.sections.length > 1) {
39+
content += `# Start of ${section.title} documentation\n\n`;
40+
}
41+
42+
for (const [path, document] of Object.entries(index)) {
43+
if (!path.startsWith(`docs/${section.slug}`)) continue;
44+
45+
if (options.ignore?.some((pattern) => minimatch(path, pattern))) {
46+
if (dev) console.log(`❌ Ignored by pattern: ${path}`);
47+
continue;
48+
}
49+
50+
const doc_content = options.minimize
51+
? minimize_content(document.body, options.minimize)
52+
: document.body;
53+
if (doc_content.trim() === '') continue;
54+
55+
content += `\n# ${document.metadata.title}\n\n`;
56+
content += doc_content;
57+
content += '\n';
58+
}
59+
}
60+
61+
return content;
62+
}
63+
64+
export const sections: Section[] = [
65+
{ slug: 'svelte', title: 'Svelte' },
66+
{ slug: 'kit', title: 'SvelteKit' },
67+
{ slug: 'cli', title: 'the Svelte CLI' }
68+
];
69+
70+
export function get_documentation_title(section: Section): string {
71+
return `This is the developer documentation for ${section.title}.`;
72+
}
73+
74+
function minimize_content(content: string, options?: Partial<MinimizeOptions>): string {
75+
// Merge with defaults, but only for properties that are defined
76+
const settings: MinimizeOptions = { ...defaults, ...options };
77+
78+
let minimized = content;
79+
80+
if (settings.remove_legacy) {
81+
minimized = remove_quote_blocks(minimized, 'LEGACY');
82+
}
83+
84+
if (settings.remove_note_blocks) {
85+
minimized = remove_quote_blocks(minimized, 'NOTE');
86+
}
87+
88+
if (settings.remove_details_blocks) {
89+
minimized = remove_quote_blocks(minimized, 'DETAILS');
90+
}
91+
92+
if (settings.remove_playground_links) {
93+
// Replace playground URLs with /[link] but keep the original link text
94+
minimized = minimized.replace(/\[([^\]]+)\]\(\/playground[^)]+\)/g, '[$1](/REMOVED)');
95+
}
96+
97+
if (settings.remove_prettier_ignore) {
98+
minimized = minimized
99+
.split('\n')
100+
.filter((line) => line.trim() !== '<!-- prettier-ignore -->')
101+
.join('\n');
102+
}
103+
104+
if (settings.normalize_whitespace) {
105+
minimized = minimized.replace(/\s+/g, ' ');
106+
}
107+
108+
minimized = minimized.trim();
109+
110+
return minimized;
111+
}
112+
113+
function remove_quote_blocks(content: string, blockType: string): string {
114+
return content
115+
.split('\n')
116+
.reduce((acc: string[], line: string, index: number, lines: string[]) => {
117+
// If we find a block (with or without additional text), skip it and all subsequent blockquote lines
118+
if (line.trim().startsWith(`> [!${blockType}]`)) {
119+
// Skip all subsequent lines that are part of the blockquote
120+
let i = index;
121+
while (i < lines.length && (lines[i].startsWith('>') || lines[i].trim() === '')) {
122+
i++;
123+
}
124+
// Update the index to skip all these lines
125+
index = i - 1;
126+
return acc;
127+
}
128+
129+
// Only add the line if it's not being skipped
130+
acc.push(line);
131+
return acc;
132+
}, [])
133+
.join('\n');
134+
}

‎apps/svelte.dev/src/routes/docs/+page.svelte‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,14 @@
9090
</p>
9191
</a>
9292

93+
<a href="/docs/llms">
94+
<h2>I'm a Large Language Model (LLM)</h2>
95+
<p>
96+
If you're an artificial intelligence, or trying to teach one how to use Svelte, we offer the
97+
documentation in plaintext format. Beep boop.
98+
</p>
99+
</a>
100+
93101
<a href="/chat" class="external">
94102
<h2>Help! I'm stuck</h2>
95103
<p>

‎apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import { error } from '@sveltejs/kit';
2+
import { generate_llm_content, get_documentation_title, sections } from '$lib/server/llms';
3+
4+
export const prerender = true;
5+
6+
export function entries() {
7+
return sections.map((section) => ({ path: section.slug }));
8+
}
9+
10+
export function GET({ params }) {
11+
const pkg = params.path;
12+
13+
const section = sections.find((s) => s.slug === pkg);
14+
15+
if (!section) {
16+
error(404, 'Not Found');
17+
}
18+
19+
const prefix = `<SYSTEM>${get_documentation_title(section)}</SYSTEM>`;
20+
const content = `${prefix}\n\n${generate_llm_content({ sections: [section] })}`;
21+
22+
return new Response(content, {
23+
status: 200,
24+
headers: {
25+
'Content-Type': 'text/plain; charset=utf-8',
26+
'Cache-Control': 'public, max-age=3600'
27+
}
28+
});
29+
}

‎apps/svelte.dev/src/routes/docs/llms/+page.svelte‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
<script>
2+
import { Text } from '@sveltejs/site-kit/components';
3+
</script>
4+
5+
<div class="page">
6+
<Text>
7+
<h1>Docs for LLMs</h1>
8+
9+
<p>
10+
We support the <a href="https://llmstxt.org/">llms.txt</a> convention for making documentation
11+
available to large language models and the applications that make use of them.
12+
</p>
13+
14+
<p>Currently, we have the following root-level files...</p>
15+
16+
<ul>
17+
<li><a href="/llms.txt">/llms.txt</a> — a listing of the available files</li>
18+
<li>
19+
<a href="/llms-full.txt">/llms-full.txt</a> — complete documentation for Svelte, SvelteKit and
20+
the CLI
21+
</li>
22+
<li>
23+
<a href="/llms-small.txt">/llms-small.txt</a> — compressed documentation for use with smaller
24+
context windows
25+
</li>
26+
</ul>
27+
28+
<p>...and package-level documentation:</p>
29+
30+
<ul>
31+
<li><a href="/docs/svelte/llms.txt">/docs/svelte/llms.txt</a></li>
32+
<li><a href="/docs/kit/llms.txt">/docs/kit/llms.txt</a></li>
33+
<li><a href="/docs/cli/llms.txt">/docs/cli/llms.txt</a></li>
34+
</ul>
35+
</Text>
36+
</div>
37+
38+
<style>
39+
.page {
40+
padding: var(--sk-page-padding-top) var(--sk-page-padding-side);
41+
max-width: var(--sk-page-content-width);
42+
box-sizing: content-box;
43+
margin: auto;
44+
text-wrap: balance;
45+
}
46+
</style>

‎apps/svelte.dev/src/routes/llms-full.txt/+server.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import { generate_llm_content, sections } from '$lib/server/llms';
2+
3+
export const prerender = true;
4+
5+
export function GET() {
6+
const content = `<SYSTEM>This is the full developer documentation for Svelte and SvelteKit.</SYSTEM>\n\n${generate_llm_content({ sections })}`;
7+
8+
return new Response(content, {
9+
status: 200,
10+
headers: {
11+
'Content-Type': 'text/plain; charset=utf-8',
12+
'Cache-Control': 'public, max-age=3600'
13+
}
14+
});
15+
}

‎apps/svelte.dev/src/routes/llms-small.txt/+server.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import { generate_llm_content, sections } from '$lib/server/llms';
2+
3+
export function GET() {
4+
const main_content = generate_llm_content({
5+
sections,
6+
ignore: [
7+
// Svelte ignores
8+
'docs/svelte/legacy/**/*',
9+
'docs/svelte/misc/custom-elements',
10+
'docs/svelte/misc/v4-migration-guide',
11+
'docs/svelte/misc/v5-migration-guide',
12+
'docs/svelte/misc/faq',
13+
'docs/svelte/reference/compiler-errors',
14+
'docs/svelte/reference/compiler-warnings',
15+
'docs/svelte/reference/runtime-errors',
16+
'docs/svelte/reference/runtime-warnings',
17+
'docs/svelte/reference/svelte-legacy',
18+
'**/xx-*',
19+
20+
// SvelteKit ignores
21+
'docs/kit/advanced/packaging',
22+
'docs/kit/appendix/**/*',
23+
'docs/kit/best-practices/performance',
24+
'docs/kit/build-and-deploy/*adapter-*',
25+
'docs/kit/build-and-deploy/writing-adapters'
26+
],
27+
minimize: {
28+
remove_legacy: true,
29+
remove_note_blocks: true,
30+
remove_details_blocks: true,
31+
remove_playground_links: true,
32+
remove_prettier_ignore: true,
33+
normalize_whitespace: true
34+
}
35+
});
36+
const content = `<SYSTEM>This is the abridged developer documentation for Svelte and SvelteKit.</SYSTEM>\n\n${main_content}`;
37+
38+
return new Response(content, {
39+
status: 200,
40+
headers: {
41+
'Content-Type': 'text/plain; charset=utf-8',
42+
'Cache-Control': 'public, max-age=3600'
43+
}
44+
});
45+
}
46+
47+
export const prerender = true;

‎apps/svelte.dev/src/routes/llms.txt/+server.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import { get_documentation_title, sections } from '$lib/server/llms';
2+
import template from './template.md?raw';
3+
4+
const DOMAIN = `https://svelte.dev`;
5+
6+
export const prerender = true;
7+
8+
export function GET() {
9+
const package_docs = sections.map(
10+
(section) =>
11+
`- [${section.title} documentation](${DOMAIN}/docs/${section.slug}/llms.txt): ${get_documentation_title(section)}`
12+
);
13+
14+
const content = template.replace('%PACKAGE_DOCS%', package_docs.join('\n'));
15+
16+
return new Response(content, {
17+
headers: {
18+
'Content-Type': 'text/plain; charset=utf-8',
19+
'Cache-Control': 'public, max-age=3600'
20+
}
21+
});
22+
}

‎apps/svelte.dev/src/routes/llms.txt/template.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Svelte Documentation for LLMs
2+
3+
> Svelte is a UI framework that uses a compiler to let you write breathtakingly concise components that do minimal work in the browser, using languages you already know — HTML, CSS and JavaScript.
4+
5+
## Documentation Sets
6+
7+
- [Abridged documentation](https://svelte.dev/llms-small.txt): A minimal version of the Svelte and SvelteKit documentation, with examples and non-essential content removed
8+
- [Complete documentation](https://svelte.dev/llms-full.txt): The complete Svelte and SvelteKit documentation including all examples and additional content
9+
10+
## Individual Package Documentation
11+
12+
%PACKAGE_DOCS%
13+
14+
## Notes
15+
16+
- The abridged documentation excludes legacy compatibility notes, detailed examples, and supplementary information
17+
- The complete documentation includes all content from the official documentation
18+
- Package-specific documentation files contain only the content relevant to that package
19+
- The content is automatically generated from the same source as the official documentation

0 commit comments

Comments
 (0)