Skip to content

Commit f0c91cc

Browse files
Rich-HarrisRich-Harris
committed
move llm stuff into its own module
1 parent 855a197 commit f0c91cc

File tree

6 files changed

+195
-194
lines changed

6 files changed

+195
-194
lines changed

‎apps/svelte.dev/src/lib/server/content.ts‎

Lines changed: 0 additions & 190 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
1-
import { dev } from '$app/environment';
21
import { read } from '$app/server';
32
import type { Document } from '@sveltejs/site-kit';
43
import { create_index } from '@sveltejs/site-kit/server/content';
5-
import { minimatch } from 'minimatch';
64

75
const documents = import.meta.glob<string>('../../../content/**/*.md', {
86
eager: true,
@@ -127,191 +125,3 @@ function create_docs() {
127125

128126
export const docs = create_docs();
129127
export const examples = index.examples.children;
130-
131-
export const packages = Object.keys(docs.topics).map((topic) => topic.split('/')[1]);
132-
133-
export const DOCUMENTATION_NAMES: Record<string, string> = {
134-
svelte: 'Svelte',
135-
kit: 'SvelteKit',
136-
cli: 'Svelte CLI'
137-
};
138-
139-
export function get_documentation_title(type: string): string {
140-
return `This is the developer documentation for ${DOCUMENTATION_NAMES[type]}.`;
141-
}
142-
143-
export function get_documentation_start_title(type: string): string {
144-
return `# Start of ${DOCUMENTATION_NAMES[type]} documentation`;
145-
}
146-
147-
interface MinimizeOptions {
148-
removeLegacy: boolean;
149-
removeNoteBlocks: boolean;
150-
removeDetailsBlocks: boolean;
151-
removePlaygroundLinks: boolean;
152-
removePrettierIgnore: boolean;
153-
normalizeWhitespace: boolean;
154-
}
155-
156-
const defaultOptions: MinimizeOptions = {
157-
removeLegacy: false,
158-
removeNoteBlocks: false,
159-
removeDetailsBlocks: false,
160-
removePlaygroundLinks: false,
161-
removePrettierIgnore: false,
162-
normalizeWhitespace: false
163-
};
164-
165-
function remove_quote_blocks(content: string, blockType: string): string {
166-
return content
167-
.split('\n')
168-
.reduce((acc: string[], line: string, index: number, lines: string[]) => {
169-
// If we find a block (with or without additional text), skip it and all subsequent blockquote lines
170-
if (line.trim().startsWith(`> [!${blockType}]`)) {
171-
// Skip all subsequent lines that are part of the blockquote
172-
let i = index;
173-
while (i < lines.length && (lines[i].startsWith('>') || lines[i].trim() === '')) {
174-
i++;
175-
}
176-
// Update the index to skip all these lines
177-
index = i - 1;
178-
return acc;
179-
}
180-
181-
// Only add the line if it's not being skipped
182-
acc.push(line);
183-
return acc;
184-
}, [])
185-
.join('\n');
186-
}
187-
188-
function minimize_content(content: string, options?: Partial<MinimizeOptions>): string {
189-
// Merge with defaults, but only for properties that are defined
190-
const settings: MinimizeOptions = options ? { ...defaultOptions, ...options } : defaultOptions;
191-
192-
let minimized = content;
193-
194-
if (settings.removeLegacy) {
195-
minimized = remove_quote_blocks(minimized, 'LEGACY');
196-
}
197-
198-
if (settings.removeNoteBlocks) {
199-
minimized = remove_quote_blocks(minimized, 'NOTE');
200-
}
201-
202-
if (settings.removeDetailsBlocks) {
203-
minimized = remove_quote_blocks(minimized, 'DETAILS');
204-
}
205-
206-
if (settings.removePlaygroundLinks) {
207-
// Replace playground URLs with /[link] but keep the original link text
208-
minimized = minimized.replace(/\[([^\]]+)\]\(\/playground[^)]+\)/g, '[$1](/REMOVED)');
209-
}
210-
211-
if (settings.removePrettierIgnore) {
212-
minimized = minimized
213-
.split('\n')
214-
.filter((line) => line.trim() !== '<!-- prettier-ignore -->')
215-
.join('\n');
216-
}
217-
218-
if (settings.normalizeWhitespace) {
219-
minimized = minimized.replace(/\s+/g, ' ');
220-
}
221-
222-
minimized = minimized.trim();
223-
224-
return minimized;
225-
}
226-
227-
function should_include_file_llm_docs(filename: string, ignore: string[] = []): boolean {
228-
const shouldIgnore = ignore.some((pattern) => minimatch(filename, pattern));
229-
if (shouldIgnore) {
230-
if (dev) console.log(`❌ Ignored by pattern: ${filename}`);
231-
return false;
232-
}
233-
234-
return true;
235-
}
236-
237-
interface GenerateLlmContentOptions {
238-
prefix?: string;
239-
ignore?: string[];
240-
minimize?: Partial<MinimizeOptions>;
241-
package?: string;
242-
}
243-
244-
export function generate_llm_content(options: GenerateLlmContentOptions = {}): string {
245-
const { prefix, ignore = [], minimize: minimizeOptions, package: pkg } = options;
246-
247-
let content = '';
248-
if (prefix) {
249-
content = `${prefix}\n\n`;
250-
}
251-
252-
let current_section = '';
253-
const paths = sort_documentation_paths();
254-
255-
for (const path of paths) {
256-
if (!should_include_file_llm_docs(path, ignore)) continue;
257-
258-
// If a specific package is provided, only include its docs
259-
if (pkg) {
260-
if (!path.includes(`docs/${pkg}/`)) continue;
261-
} else {
262-
// For combined content, only include paths that match any package
263-
const doc_type = packages.find((p) => path.includes(`docs/${p}/`));
264-
if (!doc_type) continue;
265-
266-
const section = get_documentation_start_title(doc_type);
267-
if (section !== current_section) {
268-
if (current_section) content += '\n';
269-
content += `${section}\n\n`;
270-
current_section = section;
271-
}
272-
}
273-
274-
const docContent = minimizeOptions
275-
? minimize_content(index[path].body, minimizeOptions)
276-
: index[path].body;
277-
if (docContent.trim() === '') continue;
278-
279-
content += `\n# ${index[path].metadata.title}\n\n`;
280-
content += docContent;
281-
content += '\n';
282-
}
283-
284-
return content;
285-
}
286-
287-
function get_documentation_section_priority(path: string): number {
288-
if (path.includes('docs/svelte/')) return 0;
289-
if (path.includes('docs/kit/')) return 1;
290-
if (path.includes('docs/cli/')) return 2;
291-
return 3;
292-
}
293-
294-
function sort_documentation_paths(): string[] {
295-
return Object.keys(index).sort((a, b) => {
296-
a = index[a].file;
297-
b = index[b].file;
298-
// First compare by section priority
299-
const priorityA = get_documentation_section_priority(a);
300-
const priorityB = get_documentation_section_priority(b);
301-
if (priorityA !== priorityB) return priorityA - priorityB;
302-
303-
// Get directory paths
304-
const dirA = a.split('/').slice(0, -1).join('/');
305-
const dirB = b.split('/').slice(0, -1).join('/');
306-
307-
// If in the same directory, prioritize index.md
308-
if (dirA === dirB) {
309-
if (a.endsWith('index.md')) return -1;
310-
if (b.endsWith('index.md')) return 1;
311-
return a.localeCompare(b);
312-
}
313-
314-
// Otherwise sort by directory path
315-
return dirA.localeCompare(dirB);
316-
});
317-
}

‎apps/svelte.dev/src/lib/server/llms.ts‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
import { minimatch } from 'minimatch';
2+
import { dev } from '$app/environment';
3+
import { docs, index } from './content';
4+
5+
interface GenerateLlmContentOptions {
6+
prefix?: string;
7+
ignore?: string[];
8+
minimize?: Partial<MinimizeOptions>;
9+
package?: string;
10+
}
11+
12+
export function generate_llm_content(options: GenerateLlmContentOptions = {}): string {
13+
const { prefix, ignore = [], minimize: minimizeOptions, package: pkg } = options;
14+
15+
let content = '';
16+
if (prefix) {
17+
content = `${prefix}\n\n`;
18+
}
19+
20+
let current_section = '';
21+
const paths = sort_documentation_paths();
22+
23+
for (const path of paths) {
24+
if (!should_include_file_llm_docs(path, ignore)) continue;
25+
26+
// If a specific package is provided, only include its docs
27+
if (pkg) {
28+
if (!path.includes(`docs/${pkg}/`)) continue;
29+
} else {
30+
// For combined content, only include paths that match any package
31+
const doc_type = packages.find((p) => path.includes(`docs/${p}/`));
32+
if (!doc_type) continue;
33+
34+
const section = get_documentation_start_title(doc_type);
35+
if (section !== current_section) {
36+
if (current_section) content += '\n';
37+
content += `${section}\n\n`;
38+
current_section = section;
39+
}
40+
}
41+
42+
const docContent = minimizeOptions
43+
? minimize_content(index[path].body, minimizeOptions)
44+
: index[path].body;
45+
if (docContent.trim() === '') continue;
46+
47+
content += `\n# ${index[path].metadata.title}\n\n`;
48+
content += docContent;
49+
content += '\n';
50+
}
51+
52+
return content;
53+
}
54+
55+
export const packages = Object.keys(docs.topics).map((topic) => topic.split('/')[1]);
56+
57+
export const DOCUMENTATION_NAMES: Record<string, string> = {
58+
svelte: 'Svelte',
59+
kit: 'SvelteKit',
60+
cli: 'Svelte CLI'
61+
};
62+
63+
export function get_documentation_title(type: string): string {
64+
return `This is the developer documentation for ${DOCUMENTATION_NAMES[type]}.`;
65+
}
66+
67+
export function get_documentation_start_title(type: string): string {
68+
return `# Start of ${DOCUMENTATION_NAMES[type]} documentation`;
69+
}
70+
71+
function minimize_content(content: string, options?: Partial<MinimizeOptions>): string {
72+
// Merge with defaults, but only for properties that are defined
73+
const settings: MinimizeOptions = options ? { ...defaultOptions, ...options } : defaultOptions;
74+
75+
let minimized = content;
76+
77+
if (settings.removeLegacy) {
78+
minimized = remove_quote_blocks(minimized, 'LEGACY');
79+
}
80+
81+
if (settings.removeNoteBlocks) {
82+
minimized = remove_quote_blocks(minimized, 'NOTE');
83+
}
84+
85+
if (settings.removeDetailsBlocks) {
86+
minimized = remove_quote_blocks(minimized, 'DETAILS');
87+
}
88+
89+
if (settings.removePlaygroundLinks) {
90+
// Replace playground URLs with /[link] but keep the original link text
91+
minimized = minimized.replace(/\[([^\]]+)\]\(\/playground[^)]+\)/g, '[$1](/REMOVED)');
92+
}
93+
94+
if (settings.removePrettierIgnore) {
95+
minimized = minimized
96+
.split('\n')
97+
.filter((line) => line.trim() !== '<!-- prettier-ignore -->')
98+
.join('\n');
99+
}
100+
101+
if (settings.normalizeWhitespace) {
102+
minimized = minimized.replace(/\s+/g, ' ');
103+
}
104+
105+
minimized = minimized.trim();
106+
107+
return minimized;
108+
}
109+
110+
function should_include_file_llm_docs(filename: string, ignore: string[] = []): boolean {
111+
const shouldIgnore = ignore.some((pattern) => minimatch(filename, pattern));
112+
if (shouldIgnore) {
113+
if (dev) console.log(`❌ Ignored by pattern: ${filename}`);
114+
return false;
115+
}
116+
117+
return true;
118+
}
119+
120+
function get_documentation_section_priority(path: string): number {
121+
if (path.includes('docs/svelte/')) return 0;
122+
if (path.includes('docs/kit/')) return 1;
123+
if (path.includes('docs/cli/')) return 2;
124+
return 3;
125+
}
126+
127+
function sort_documentation_paths(): string[] {
128+
return Object.keys(index).sort((a, b) => {
129+
a = index[a].file;
130+
b = index[b].file;
131+
// First compare by section priority
132+
const priorityA = get_documentation_section_priority(a);
133+
const priorityB = get_documentation_section_priority(b);
134+
if (priorityA !== priorityB) return priorityA - priorityB;
135+
136+
// Get directory paths
137+
const dirA = a.split('/').slice(0, -1).join('/');
138+
const dirB = b.split('/').slice(0, -1).join('/');
139+
140+
// If in the same directory, prioritize index.md
141+
if (dirA === dirB) {
142+
if (a.endsWith('index.md')) return -1;
143+
if (b.endsWith('index.md')) return 1;
144+
return a.localeCompare(b);
145+
}
146+
147+
// Otherwise sort by directory path
148+
return dirA.localeCompare(dirB);
149+
});
150+
}
151+
152+
interface MinimizeOptions {
153+
removeLegacy: boolean;
154+
removeNoteBlocks: boolean;
155+
removeDetailsBlocks: boolean;
156+
removePlaygroundLinks: boolean;
157+
removePrettierIgnore: boolean;
158+
normalizeWhitespace: boolean;
159+
}
160+
161+
const defaultOptions: MinimizeOptions = {
162+
removeLegacy: false,
163+
removeNoteBlocks: false,
164+
removeDetailsBlocks: false,
165+
removePlaygroundLinks: false,
166+
removePrettierIgnore: false,
167+
normalizeWhitespace: false
168+
};
169+
170+
function remove_quote_blocks(content: string, blockType: string): string {
171+
return content
172+
.split('\n')
173+
.reduce((acc: string[], line: string, index: number, lines: string[]) => {
174+
// If we find a block (with or without additional text), skip it and all subsequent blockquote lines
175+
if (line.trim().startsWith(`> [!${blockType}]`)) {
176+
// Skip all subsequent lines that are part of the blockquote
177+
let i = index;
178+
while (i < lines.length && (lines[i].startsWith('>') || lines[i].trim() === '')) {
179+
i++;
180+
}
181+
// Update the index to skip all these lines
182+
index = i - 1;
183+
return acc;
184+
}
185+
186+
// Only add the line if it's not being skipped
187+
acc.push(line);
188+
return acc;
189+
}, [])
190+
.join('\n');
191+
}

‎apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { error } from '@sveltejs/kit';
2-
import { generate_llm_content, get_documentation_title, packages } from '$lib/server/content';
2+
import { generate_llm_content, get_documentation_title, packages } from '$lib/server/llms';
33

44
export const prerender = true;
55

0 commit comments

Comments
 (0)