feat: add LLM text processing and route for retrieving formatted content

Author: LuggaPugga

src/app/llms.txt/route.ts

@@ -0,0 +1,11 @@
+import { source } from "@/lib/source"
+import { getLLMText } from "@/lib/get-llm-text"
+
+export const revalidate = false
+
+export async function GET() {
+  const scan = source.getPages().map(getLLMText)
+  const scanned = await Promise.all(scan)
+
+  return new Response(scanned.join("\n\n"))
+}

src/lib/get-llm-text.ts

@@ -0,0 +1,22 @@
+import { remark } from "remark"
+import remarkGfm from "remark-gfm"
+import remarkMdx from "remark-mdx"
+import { remarkInclude } from "fumadocs-mdx/config"
+import { source } from "@/lib/source"
+import type { InferPageType } from "fumadocs-core/source"
+
+const processor = remark().use(remarkMdx).use(remarkInclude).use(remarkGfm)
+
+export async function getLLMText(page: InferPageType<typeof source>) {
+  const processed = await processor.process({
+    path: page.data._file.absolutePath,
+    value: page.data.content,
+  })
+
+  return `# ${page.data.title}
+URL: ${page.url}
+
+${page.data.description}
+
+${processed.value}`
+}