[Docs Site] Use vendored Markdown for llms-full.txt (cloudflare#23686)

* [Docs Site] Use vendored Markdown for llms-full.txt

* remove tests as llms-full is now a post-build thing

* unused import

Author: KianNH

.github/workflows/publish-production.yml

@@ -41,6 +41,11 @@ jobs:
           cd distmd && zip -r markdown.zip .
           npx wrangler r2 object put vendored-markdown/markdown.zip --file=markdown.zip --remote
           rm markdown.zip
+
+          cd distllms
+          for file in $(find . -type f); do
+            npx wrangler r2 object put vendored-markdown/$file --file=$file --remote
+          done
       - name: Upload vendored Markdown files to ZT DevDocs bucket
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.ZT_DEVDOCS_ACCESS_KEY_ID }}

.gitignore

@@ -1,6 +1,7 @@
 # build output
 dist/
 distmd/
+distllms/
 # generated types
 .astro/
 

bin/generate-index-md.ts

@@ -1,12 +1,23 @@
-import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import {
+	readFileSync,
+	writeFileSync,
+	mkdirSync,
+	appendFileSync,
+} from "node:fs";
 
 import glob from "fast-glob";
 import { parse } from "node-html-parser";
 import { htmlToMarkdown } from "~/util/markdown";
 
+import YAML from "yaml";
+
 const files = await glob("dist/**/*.html");
 
 for (const file of files) {
+	if (file === "dist/index.html" || file === "dist/404.html") {
+		continue;
+	}
+
 	const html = readFileSync(file, "utf-8");
 	const dom = parse(html);
 
@@ -24,9 +35,42 @@ for (const file of files) {
 		continue;
 	}
 
+	const product = file.split("/")[1];
 	const path = file.replace("dist/", "distmd/").replace(".html", ".md");
 
 	mkdirSync(path.split("/").slice(0, -1).join("/"), { recursive: true });
-
 	writeFileSync(path, markdown);
+
+	const llmsFullContent = ["<page>", markdown, "</page>\n\n"].join("\n");
+
+	mkdirSync(`distllms/${product}`, { recursive: true });
+	appendFileSync("distllms/llms-full.txt", llmsFullContent);
+	appendFileSync(`distllms/${product}/llms-full.txt`, llmsFullContent);
+
+	try {
+		const path = await glob(`src/content/products/${product}.*`).then((arr) =>
+			arr.at(0),
+		);
+
+		if (!path) {
+			continue;
+		}
+
+		const yaml = YAML.parse(readFileSync(path, "utf-8"));
+		const group = yaml.product?.group?.replaceAll(" ", "-").toLowerCase();
+
+		if (!group) {
+			continue;
+		}
+
+		mkdirSync(`distllms/${group}`, { recursive: true });
+		appendFileSync(`distllms/${group}/llms-full.txt`, llmsFullContent);
+	} catch (error) {
+		if (error instanceof Error) {
+			console.error(
+				`Failed to find a product group for ${product}:`,
+				error.message,
+			);
+		}
+	}
 }

src/pages/[area]/llms-full.txt.ts

@@ -22,6 +22,17 @@ export default class extends WorkerEntrypoint<Env> {
 			});
 		}
 
+		if (request.url.endsWith("/llms-full.txt")) {
+			const { pathname } = new URL(request.url);
+			const res = await this.env.VENDORED_MARKDOWN.get(pathname.slice(1));
+
+			return new Response(res?.body, {
+				headers: {
+					"Content-Type": "text/markdown; charset=utf-8",
+				},
+			});
+		}
+
 		if (request.url.endsWith("/index.md")) {
 			const htmlUrl = request.url.replace("index.md", "");
 			const res = await this.env.ASSETS.fetch(htmlUrl, request);

src/pages/[product]/llms-full.txt.ts

@@ -204,41 +204,6 @@ describe("Cloudflare Docs", () => {
 			const text = await response.text();
 			expect(text).toContain("# Cloudflare Developer Documentation");
 		});
-
-		it("llms-full.txt", async () => {
-			const request = new Request("http://fakehost/llms-full.txt");
-			const response = await SELF.fetch(request);
-
-			expect(response.status).toBe(200);
-
-			const text = await response.text();
-			expect(text).toContain("URL: https://developers.cloudflare.com/");
-			expect(text).toContain('from "~/components"');
-		});
-
-		it("product-specific llms-full.txt", async () => {
-			const request = new Request("http://fakehost/workers/llms-full.txt");
-			const response = await SELF.fetch(request);
-
-			expect(response.status).toBe(200);
-
-			const text = await response.text();
-			expect(text).toContain("URL: https://developers.cloudflare.com/");
-			expect(text).toContain('from "~/components"');
-		});
-
-		it("area-specific llms-full.txt", async () => {
-			const request = new Request(
-				"http://fakehost/developer-platform/llms-full.txt",
-			);
-			const response = await SELF.fetch(request);
-
-			expect(response.status).toBe(200);
-
-			const text = await response.text();
-			expect(text).toContain("URL: https://developers.cloudflare.com/");
-			expect(text).toContain('from "~/components"');
-		});
 	});
 
 	describe("index.md handling", () => {