Fix llms.txt generation

Author: sibiraj-s

app/llms.txt/route.ts

@@ -1,44 +1,12 @@
-import * as fs from 'node:fs/promises';
-import fg from 'fast-glob';
-import { remarkInclude } from 'fumadocs-mdx/config';
-import matter from 'gray-matter';
-import { remark } from 'remark';
-import remarkGfm from 'remark-gfm';
-import remarkMdx from 'remark-mdx';
-import remarkStringify from 'remark-stringify';
+import { getLLMText } from '@/lib/get-llm-text';
+import { source } from '@/lib/source';
 
+// cached forever
 export const revalidate = false;
 
 export async function GET() {
-  // all scanned content
-  const files = await fg(['./content/docs/**/*.mdx']);
-
-  const scan = files.map(async (file) => {
-    const fileContent = await fs.readFile(file);
-    const { content, data } = matter(fileContent.toString());
-
-    const processed = await processContent(content);
-    return `file: ${file}
-meta: ${JSON.stringify(data, null, 2)}
-
-${processed}`;
-  });
-
+  const scan = source.getPages().map(getLLMText);
   const scanned = await Promise.all(scan);
 
   return new Response(scanned.join('\n\n'));
 }
-
-async function processContent(content: string): Promise<string> {
-  const file = await remark()
-    .use(remarkMdx)
-    // https://fumadocs.vercel.app/docs/mdx/include
-    .use(remarkInclude)
-    // gfm styles
-    .use(remarkGfm)
-    // .use(your remark plugins)
-    .use(remarkStringify) // to string
-    .process(content);
-
-  return String(file);
-}

lib/get-llm-text.ts

@@ -0,0 +1,27 @@
+import type { InferPageType } from 'fumadocs-core/source';
+import { remarkInclude } from 'fumadocs-mdx/config';
+import { remark } from 'remark';
+import remarkGfm from 'remark-gfm';
+import remarkMdx from 'remark-mdx';
+
+import { source } from '@/lib/source';
+
+const processor = remark()
+  .use(remarkMdx)
+  // needed for Fumadocs MDX
+  .use(remarkInclude)
+  .use(remarkGfm);
+
+export async function getLLMText(page: InferPageType<typeof source>) {
+  const processed = await processor.process({
+    path: page.data._file.absolutePath,
+    value: page.data.content,
+  });
+
+  return `# ${page.data.title}
+URL: ${page.url}
+
+${page.data.description}
+
+${processed.value}`;
+}

package-lock.json

@@ -14,11 +14,10 @@
   "dependencies": {
     "fast-glob": "^3.3.3",
     "fumadocs-core": "^15.6.11",
-    "fumadocs-mdx": "^11.7.3",
+    "fumadocs-mdx": "^11.7.5",
     "fumadocs-openapi": "^9.1.13",
     "fumadocs-ui": "^15.6.11",
     "geist": "^1.4.2",
-    "gray-matter": "^4.0.3",
     "next": "^15.4.6",
     "react": "^19.1.1",
     "react-dom": "^19.1.1",