[Docs Site] Improve llms-full.txt (#19040)

* [Docs Site] Improve llms-full.txt

* Ignore third-party legal files

Author: KianNH

src/pages/[area]/llms-full.txt.ts

@@ -0,0 +1,62 @@
+import type { APIRoute } from "astro";
+import { getCollection } from "astro:content";
+import { slug } from "github-slugger";
+
+export async function getStaticPaths() {
+	const products = await getCollection("products");
+
+	const areas = new Set(
+		products.flatMap((p) => {
+			if (!p.data.product.group) return [];
+
+			return slug(p.data.product.group.toLowerCase());
+		}),
+	);
+
+	return [...areas].map((area) => {
+		return {
+			params: {
+				area,
+			},
+		};
+	});
+}
+
+export const GET: APIRoute = async ({ params }) => {
+	const products = await getCollection("products", (p) => {
+		if (!p.data.product.group) return false;
+
+		return slug(p.data.product.group.toLowerCase()) === params.area;
+	});
+
+	const markdown = await getCollection("docs", (e) => {
+		if (!e.body) return false;
+
+		if (
+			e.slug === "warp-client/legal/3rdparty" ||
+			e.slug === "magic-wan/legal/3rdparty"
+		)
+			return false;
+
+		return products.some((p) =>
+			e.slug.startsWith(p.data.product.url.slice(1, -1)),
+		);
+	})
+		.then((entries) =>
+			entries.map((entry) => {
+				return [
+					`# ${entry.data.title}`,
+					`URL: https://developers.cloudflare.com/${entry.slug}/`,
+					`${entry.body.trim()}`,
+					"---",
+				].join("\n\n");
+			}),
+		)
+		.then((array) => array.join("\n\n"));
+
+	return new Response(markdown, {
+		headers: {
+			"content-type": "text/plain",
+		},
+	});
+};

src/pages/[product]/llms-full.txt.ts

@@ -0,0 +1,50 @@
+import type { APIRoute } from "astro";
+import { getCollection } from "astro:content";
+
+export async function getStaticPaths() {
+	const products = await getCollection("products", (p) => {
+		return p.data.product.group?.toLowerCase() === "developer platform";
+	});
+
+	return products.map((entry) => {
+		return {
+			params: {
+				product: entry.id,
+			},
+			props: {
+				product: entry,
+			},
+		};
+	});
+}
+
+export const GET: APIRoute = async ({ props }) => {
+	const markdown = await getCollection("docs", (e) => {
+		if (
+			e.slug === "warp-client/legal/3rdparty" ||
+			e.slug === "magic-wan/legal/3rdparty"
+		)
+			return false;
+
+		return (
+			e.slug.startsWith(props.product.data.product.url.slice(1, -1)) && e.body
+		);
+	})
+		.then((entries) =>
+			entries.map((entry) => {
+				return [
+					`# ${entry.data.title}`,
+					`URL: https://developers.cloudflare.com/${entry.slug}/`,
+					`${entry.body.trim()}`,
+					"---",
+				].join("\n\n");
+			}),
+		)
+		.then((array) => array.join("\n\n"));
+
+	return new Response(markdown, {
+		headers: {
+			"content-type": "text/plain",
+		},
+	});
+};

src/pages/llms-full.txt.ts

@@ -2,8 +2,27 @@ import type { APIRoute } from "astro";
 import { getCollection } from "astro:content";
 
 export const GET: APIRoute = async () => {
-	const markdown = await getCollection("docs")
-		.then((entries) => entries.map((entry) => entry.body))
+	const markdown = await getCollection("docs", (e) => {
+		if (!e.body) return false;
+
+		if (
+			e.slug === "warp-client/legal/3rdparty" ||
+			e.slug === "magic-wan/legal/3rdparty"
+		)
+			return false;
+
+		return true;
+	})
+		.then((entries) =>
+			entries.map((entry) => {
+				return [
+					`# ${entry.data.title}`,
+					`URL: https://developers.cloudflare.com/${entry.slug}/`,
+					`${entry.body.trim()}`,
+					"---",
+				].join("\n\n");
+			}),
+		)
 		.then((array) => array.join("\n\n"));
 
 	return new Response(markdown, {