[Docs Site] Make per-product 404 pages and link to SERP/llms.txt (#23587)

* [Docs Site] Make per-product 404 pages and link to SERP/llms.txt

* route to 404 via worker

* update test

Author: KianNH

astro.config.ts

@@ -161,6 +161,7 @@ export default defineConfig({
 				headingLinks: false,
 			},
 			routeMiddleware: "./src/plugins/starlight/route-data.ts",
+			disable404Route: true,
 		}),
 		liveCode({}),
 		icon(),

src/components/404.astro

@@ -0,0 +1,21 @@
+<div class="flex flex-col items-center justify-center md:py-28">
+	<h1>404</h1>
+	<p>
+		Check the URL, try using our <a id="404-search-link" href="/search/"
+			>search</a
+		> or try our LLM-friendly
+		<a href="/llms.txt"> llms.txt directory</a>.
+	</p>
+</div>
+
+<script>
+	const { pathname } = window.location;
+
+	const anchor = document.getElementById("404-search-link");
+
+	if (anchor) {
+		const pretty = pathname.replaceAll("/", " ").replaceAll("-", " ").trim();
+
+		anchor.setAttribute("href", `/search/?q=${encodeURIComponent(pretty)}`);
+	}
+</script>

src/pages/404.astro

@@ -0,0 +1,21 @@
+---
+import StarlightPage, {
+	type StarlightPageProps,
+} from "@astrojs/starlight/components/StarlightPage.astro";
+import FourOhFour from "~/components/404.astro";
+
+const props = {
+	frontmatter: {
+		title: "404 - Page Not Found",
+		template: "splash",
+		editUrl: false,
+		feedback: false,
+	},
+	hideTitle: true,
+	hideBreadcrumbs: true,
+} as StarlightPageProps;
+---
+
+<StarlightPage {...props}>
+	<FourOhFour />
+</StarlightPage>

src/pages/[product]/404.astro

@@ -0,0 +1,38 @@
+---
+import StarlightPage, {
+	type StarlightPageProps,
+} from "@astrojs/starlight/components/StarlightPage.astro";
+import type { GetStaticPaths } from "astro";
+import { getCollection } from "astro:content";
+
+import FourOhFour from "~/components/404.astro";
+
+export const getStaticPaths = (async () => {
+	const sections = await getCollection("docs", (e) => {
+		return e.id.split("/").length === 1;
+	});
+
+	return sections.map((section) => {
+		return {
+			params: {
+				product: section.id,
+			},
+		};
+	});
+}) satisfies GetStaticPaths;
+
+const props = {
+	frontmatter: {
+		title: "404 - Page Not Found",
+		tableOfContents: false,
+		editUrl: false,
+		feedback: false,
+	},
+	hideTitle: true,
+	hideBreadcrumbs: true,
+} as StarlightPageProps;
+---
+
+<StarlightPage {...props}>
+	<FourOhFour />
+</StarlightPage>

worker/index.ts

@@ -98,6 +98,16 @@ export default class extends WorkerEntrypoint<Env> {
 			console.error("Unknown error", error);
 		}
 
-		return this.env.ASSETS.fetch(request);
+		const response = await this.env.ASSETS.fetch(request);
+
+		if (response.status === 404) {
+			const section = new URL(response.url).pathname.split("/").at(1);
+
+			if (!section) return response;
+
+			return this.env.ASSETS.fetch(`http://fakehost/${section}/404/`);
+		}
+
+		return response;
 	}
 }

worker/index.worker.test.ts

@@ -24,7 +24,7 @@ describe("Cloudflare Docs", () => {
 			const request = new Request("http://fakehost/non-existent");
 			const response = await SELF.fetch(request);
 			expect(response.status).toBe(404);
-			expect(await response.text()).toContain("Page not found.");
+			expect(await response.text()).toContain("Check the URL,");
 		});
 	});
 
@@ -283,7 +283,7 @@ describe("Cloudflare Docs", () => {
 			const request = new Request("http://fakehost/non-existent/index.md");
 			const response = await SELF.fetch(request);
 			expect(response.status).toBe(404);
-			expect(await response.text()).toContain("Page not found.");
+			expect(await response.text()).toContain("Check the URL,");
 		});
 	});