[Docs Site] Generate index.md for all index.html files

astro.config.ts

@@ -90,15 +90,6 @@ export default defineConfig({
 				src: "./src/assets/logo.svg",
 			},
 			favicon: "/favicon.png",
-			head: ["image", "og:image", "twitter:image"].map((name) => {
-				return {
-					tag: "meta",
-					attrs: {
-						name,
-						content: "https://developers.cloudflare.com/cf-twitter-card.png",
-					},
-				};
-			}),
 			social: {
 				github: "https://github.com/cloudflare/cloudflare-docs",
 				"x.com": "https://x.com/cloudflare",

public/__redirects

@@ -1737,6 +1737,7 @@
 /cloudflare-one/tutorials/zsh-env-var/ /cloudflare-one/tutorials/cli/ 301
 
 ### DYNAMIC REDIRECTS ###
+/*/index.html.md /:splat/index.md 301
 /api-next/* /api/:splat 301
 /changelog-next/* /changelog/:splat 301
 /browser-rendering/quick-actions-rest-api/* /browser-rendering/rest-api/:splat 301

public/_headers

@@ -3,3 +3,9 @@
 
 /_astro/*
 	Cache-Control: public, max-age=604800, immutable
+
+/*/llms-full.txt:
+  Content-Type: text/markdown; charset=utf-8
+
+/*/index.md:
+  Content-Type: text/markdown; charset=utf-8

src/components/overrides/Head.astro

@@ -150,6 +150,16 @@ const ogImageUrl = new URL(ogImagePath, Astro.url.origin).toString();
 	});
 });
 
+head.push({
+	tag: "link",
+	attrs: {
+		rel: "alternate",
+		type: "text/markdown",
+		href: Astro.url.pathname + "index.md",
+	},
+	content: "",
+});
+
 metaTags.map((attrs) => {
 	head.push({
 		tag: "meta",

src/content/docs/style-guide/fixtures/index.mdx

@@ -0,0 +1,8 @@
+---
+title: Fixtures
+noindex: true
+sidebar:
+  hidden: true
+---
+
+This folder stores test fixtures to be used in CI.

src/content/docs/style-guide/fixtures/markdown.mdx

@@ -0,0 +1,23 @@
+---
+title: Markdown
+noindex: true
+sidebar:
+  hidden: true
+---
+
+import { Tabs, TabItem } from "~/components";
+
+The HTML generated by this file is used as a test fixture for our Markdown generation.
+
+<Tabs>
+	<TabItem label="mdx">
+		```mdx
+		test
+		```
+	</TabItem>
+	<TabItem label="md">
+		```md
+		test
+		```
+	</TabItem>
+</Tabs>

vitest.workspace.ts

@@ -8,6 +8,14 @@ const workspace = defineWorkspace([
 		test: {
 			name: "Workers",
 			include: ["**/*.worker.test.ts"],
+			deps: {
+				optimizer: {
+					ssr: {
+						enabled: true,
+						include: ["node-html-parser"],
+					},
+				},
+			},
 			poolOptions: {
 				workers: {
 					wrangler: { configPath: "./wrangler.toml" },

worker/index.ts

@@ -2,6 +2,16 @@ import { WorkerEntrypoint } from "cloudflare:workers";
 import { generateRedirectsEvaluator } from "redirects-in-workers";
 import redirectsFileContents from "../dist/__redirects";
 
+import { parse } from "node-html-parser";
+import { process } from "../src/util/rehype";
+
+import rehypeParse from "rehype-parse";
+import rehypeBaseUrl from "../src/plugins/rehype/base-url";
+import rehypeFilterElements from "../src/plugins/rehype/filter-elements";
+import remarkGfm from "remark-gfm";
+import rehypeRemark from "rehype-remark";
+import remarkStringify from "remark-stringify";
+
 const redirectsEvaluator = generateRedirectsEvaluator(redirectsFileContents, {
 	maxLineLength: 10_000, // Usually 2_000
 	maxStaticRules: 10_000, // Usually 2_000
@@ -10,6 +20,45 @@ const redirectsEvaluator = generateRedirectsEvaluator(redirectsFileContents, {
 
 export default class extends WorkerEntrypoint<Env> {
 	override async fetch(request: Request) {
+		if (request.url.endsWith("/index.md")) {
+			const res = await this.env.ASSETS.fetch(
+				request.url.replace("index.md", ""),
+				request,
+			);
+
+			if (res.status === 404) {
+				return res;
+			}
+
+			if (
+				res.status === 200 &&
+				res.headers.get("content-type")?.startsWith("text/html")
+			) {
+				const html = await res.text();
+
+				const content = parse(html).querySelector(".sl-markdown-content");
+
+				if (!content) {
+					return new Response("Not Found", { status: 404 });
+				}
+
+				const markdown = await process(content.toString(), [
+					rehypeParse,
+					rehypeBaseUrl,
+					rehypeFilterElements,
+					remarkGfm,
+					rehypeRemark,
+					remarkStringify,
+				]);
+
+				return new Response(markdown, {
+					headers: {
+						"content-type": "text/markdown; charset=utf-8",
+					},
+				});
+			}
+		}
+
 		try {
 			try {
 				const redirect = await redirectsEvaluator(request, this.env.ASSETS);

worker/index.worker.test.ts

@@ -63,6 +63,14 @@ describe("Cloudflare Docs", () => {
 			expect(response.status).toBe(301);
 			expect(response.headers.get("Location")).toBe("/changelog/rss.xml");
 		});
+
+		it("redirects /workers/index.html.md to /workers/index.md", async () => {
+			const request = new Request("http://fakehost/workers/index.html.md");
+			const response = await SELF.fetch(request, { redirect: "manual" });
+
+			expect(response.status).toBe(301);
+			expect(response.headers.get("Location")).toBe("/workers/index.md");
+		});
 	});
 
 	describe("json endpoints", () => {
@@ -247,4 +255,40 @@ describe("Cloudflare Docs", () => {
 			expect(text).toContain('from "~/components"');
 		});
 	});
+
+	describe("index.md handling", () => {
+		it("style-guide fixture", async () => {
+			const request = new Request(
+				"http://fakehost/style-guide/fixtures/markdown/index.md",
+			);
+			const response = await SELF.fetch(request);
+
+			expect(response.status).toBe(200);
+
+			const text = await response.text();
+			expect(text).toMatchInlineSnapshot(`
+				"The HTML generated by this file is used as a test fixture for our Markdown generation.
+
+				* mdx
+
+				  \`\`\`mdx
+				  test
+				  \`\`\`
+
+				* md
+
+				  \`\`\`md
+				  test
+				  \`\`\`
+				"
+			`);
+		});
+
+		it("responds with 404.html at `/non-existent/index.md`", async () => {
+			const request = new Request("http://fakehost/non-existent/index.md");
+			const response = await SELF.fetch(request);
+			expect(response.status).toBe(404);
+			expect(await response.text()).toContain("Page not found.");
+		});
+	});
 });

wrangler.toml

@@ -6,7 +6,7 @@ compatibility_flags = ["nodejs_compat"]
 main = "./worker/index.ts"
 
 workers_dev = true
-route = { pattern = "developers.cloudflare.com/*", zone_name = "developers.cloudflare.com"}
+route = { pattern = "developers.cloudflare.com/*", zone_name = "developers.cloudflare.com" }
 
 rules = [
 	{ type = "Text", globs = ["**/__redirects"], fallthrough = true },
@@ -16,4 +16,4 @@ rules = [
 directory = "./dist"
 binding = "ASSETS"
 not_found_handling = "404-page"
-run_worker_first = true
+run_worker_first = true