worker?

Author: KianNH

bin/generate-index-md.ts

@@ -0,0 +1,62 @@
+import type { APIRoute } from "astro";
+import { getCollection } from "astro:content";
+import { slug } from "github-slugger";
+
+export async function getStaticPaths() {
+	const products = await getCollection("products");
+
+	const areas = new Set(
+		products.flatMap((p) => {
+			if (!p.data.product.group) return [];
+
+			return slug(p.data.product.group.toLowerCase());
+		}),
+	);
+
+	return [...areas].map((area) => {
+		return {
+			params: {
+				area,
+			},
+		};
+	});
+}
+
+export const GET: APIRoute = async ({ params }) => {
+	const products = await getCollection("products", (p) => {
+		if (!p.data.product.group) return false;
+
+		return slug(p.data.product.group.toLowerCase()) === params.area;
+	});
+
+	const markdown = await getCollection("docs", (e) => {
+		if (!e.body) return false;
+
+		if (
+			e.id === "warp-client/legal/3rdparty" ||
+			e.id === "magic-wan/legal/3rdparty"
+		)
+			return false;
+
+		return products.some((p) =>
+			e.id.startsWith(p.data.product.url.slice(1, -1)),
+		);
+	})
+		.then((entries) =>
+			entries.map((entry) => {
+				return [
+					`# ${entry.data.title}`,
+					`URL: https://developers.cloudflare.com/${entry.id}/`,
+					`${entry.body?.trim()}`,
+					"---",
+				].join("\n\n");
+			}),
+		)
+		.then((array) => array.join("\n\n"));
+
+	return new Response(markdown, {
+		headers: {
+			"content-type": "text/plain",
+		},
+	});
+};

src/pages/[area]/llms-full.txt.ts

@@ -0,0 +1,50 @@
+import type { APIRoute } from "astro";
+import { getCollection } from "astro:content";
+
+export async function getStaticPaths() {
+	const products = await getCollection("products", (p) => {
+		return p.data.product.group?.toLowerCase() === "developer platform";
+	});
+
+	return products.map((entry) => {
+		return {
+			params: {
+				product: entry.id,
+			},
+			props: {
+				product: entry,
+			},
+		};
+	});
+}
+
+export const GET: APIRoute = async ({ props }) => {
+	const markdown = await getCollection("docs", (e) => {
+		if (
+			e.id === "warp-client/legal/3rdparty" ||
+			e.id === "magic-wan/legal/3rdparty"
+		)
+			return false;
+
+		return (
+			e.id.startsWith(props.product.data.product.url.slice(1, -1)) && e.body
+		);
+	})
+		.then((entries) =>
+			entries.map((entry) => {
+				return [
+					`# ${entry.data.title}`,
+					`URL: https://developers.cloudflare.com/${entry.id}/`,
+					`${entry.body?.trim()}`,
+					"---",
+				].join("\n\n");
+			}),
+		)
+		.then((array) => array.join("\n\n"));
+
+	return new Response(markdown, {
+		headers: {
+			"content-type": "text/plain",
+		},
+	});
+};

src/pages/[product]/llms-full.txt.ts

@@ -0,0 +1,33 @@
+import type { APIRoute } from "astro";
+import { getCollection } from "astro:content";
+
+export const GET: APIRoute = async () => {
+	const markdown = await getCollection("docs", (e) => {
+		if (!e.body) return false;
+
+		if (
+			e.id === "warp-client/legal/3rdparty" ||
+			e.id === "magic-wan/legal/3rdparty"
+		)
+			return false;
+
+		return true;
+	})
+		.then((entries) =>
+			entries.map((entry) => {
+				return [
+					`# ${entry.data.title}`,
+					`URL: https://developers.cloudflare.com/${entry.id}/`,
+					`${entry.body?.trim()}`,
+					"---",
+				].join("\n\n");
+			}),
+		)
+		.then((array) => array.join("\n\n"));
+
+	return new Response(markdown, {
+		headers: {
+			"content-type": "text/plain",
+		},
+	});
+};

src/pages/llms-full.txt.ts

@@ -2,24 +2,58 @@ import { WorkerEntrypoint } from "cloudflare:workers";
 import { generateRedirectsEvaluator } from "redirects-in-workers";
 import redirectsFileContents from "../dist/_redirects";
 
+import { parse } from "node-html-parser";
+import { process } from "../src/util/rehype";
+
+import rehypeParse from "rehype-parse";
+import rehypeBaseUrl from "../src/plugins/rehype/base-url";
+import rehypeFilterElements from "../src/plugins/rehype/filter-elements";
+import remarkGfm from "remark-gfm";
+import rehypeRemark from "rehype-remark";
+import remarkStringify from "remark-stringify";
+
 const redirectsEvaluator = generateRedirectsEvaluator(redirectsFileContents);
 
 export default class extends WorkerEntrypoint<Env> {
-	override async fetch(request: Request, env: Env) {
+	override async fetch(request: Request) {
 		const url = new URL(request.url);
-		if (request.url.endsWith("index.md")) {
-			const path = url.pathname.slice(1) + "index.md";
-			const file = await env.MARKDOWN.get(path);
+		if (request.url.endsWith("/index.md")) {
+			const res = await this.env.ASSETS.fetch(
+				request.url.replace("index.md", ""),
+				request,
+			);
+
+			if (res.status === 404) {
+				return res;
+			}
+
+			if (
+				res.status === 200 &&
+				res.headers.get("content-type")?.startsWith("text/html")
+			) {
+				const html = await res.text();
 
-			if (file) {
-				return new Response(file.body, {
+				const content = parse(html).querySelector(".sl-markdown-content");
+
+				if (!content) {
+					return new Response("Not Found", { status: 404 });
+				}
+
+				const markdown = await process(content.toString(), [
+					rehypeParse,
+					rehypeBaseUrl,
+					rehypeFilterElements,
+					remarkGfm,
+					rehypeRemark,
+					remarkStringify,
+				]);
+
+				return new Response(markdown, {
 					headers: {
-						"content-type": "text/markdown",
+						"content-type": "text/markdown; charset=utf-8",
 					},
 				});
 			}
-
-			return new Response("Not Found", { status: 404 });
 		}
 
 		try {

worker/index.ts

@@ -14,8 +14,4 @@ rules = [{ type = "Text", globs = ["**/_redirects"], fallthrough = true }]
 directory = "./dist"
 binding = "ASSETS"
 not_found_handling = "404-page"
-run_worker_first = true
-
-[[r2_buckets]]
-binding = "MARKDOWN"
-bucket_name = "devdocs-index-md"
+run_worker_first = true