[Docs Site] Generate index.md for all index.html files (#20988)

* [Docs Site] Generate index.md for all index.html files

* remove redundant head config

* charset

Author: KianNH

astro.config.ts

@@ -90,15 +90,6 @@ export default defineConfig({
 				src: "./src/assets/logo.svg",
 			},
 			favicon: "/favicon.png",
-			head: ["image", "og:image", "twitter:image"].map((name) => {
-				return {
-					tag: "meta",
-					attrs: {
-						name,
-						content: "https://developers.cloudflare.com/cf-twitter-card.png",
-					},
-				};
-			}),
 			social: {
 				github: "https://github.com/cloudflare/cloudflare-docs",
 				"x.com": "https://x.com/cloudflare",

public/__redirects

@@ -1752,6 +1752,7 @@
 /cloudflare-one/tutorials/zsh-env-var/ /cloudflare-one/tutorials/cli/ 301
 
 ### DYNAMIC REDIRECTS ###
+/*/index.html.md /:splat/index.md 301
 /api-next/* /api/:splat 301
 /changelog-next/* /changelog/:splat 301
 /browser-rendering/quick-actions-rest-api/* /browser-rendering/rest-api/:splat 301

public/_headers

@@ -3,3 +3,9 @@
 
 /_astro/*
 	Cache-Control: public, max-age=604800, immutable
+
+/*/llms-full.txt:
+  Content-Type: text/markdown; charset=utf-8
+
+/*/index.md:
+  Content-Type: text/markdown; charset=utf-8

src/components/overrides/Head.astro

@@ -150,6 +150,16 @@ const ogImageUrl = new URL(ogImagePath, Astro.url.origin).toString();
 	});
 });
 
+head.push({
+	tag: "link",
+	attrs: {
+		rel: "alternate",
+		type: "text/markdown",
+		href: Astro.url.pathname + "index.md",
+	},
+	content: "",
+});
+
 metaTags.map((attrs) => {
 	head.push({
 		tag: "meta",

src/content/docs/style-guide/fixtures/index.mdx

@@ -0,0 +1,8 @@
+---
+title: Fixtures
+noindex: true
+sidebar:
+  hidden: true
+---
+
+This folder stores test fixtures to be used in CI.

src/content/docs/style-guide/fixtures/markdown.mdx

@@ -0,0 +1,23 @@
+---
+title: Markdown
+noindex: true
+sidebar:
+  hidden: true
+---
+
+import { Tabs, TabItem } from "~/components";
+
+The HTML generated by this file is used as a test fixture for our Markdown generation.
+
+<Tabs>
+	<TabItem label="mdx">
+		```mdx
+		test
+		```
+	</TabItem>
+	<TabItem label="md">
+		```md
+		test
+		```
+	</TabItem>
+</Tabs>

src/pages/[...entry]/index.html.md.ts

@@ -8,6 +8,14 @@ const workspace = defineWorkspace([
 		test: {
 			name: "Workers",
 			include: ["**/*.worker.test.ts"],
+			deps: {
+				optimizer: {
+					ssr: {
+						enabled: true,
+						include: ["node-html-parser"],
+					},
+				},
+			},
 			poolOptions: {
 				workers: {
 					wrangler: { configPath: "./wrangler.toml" },

src/pages/cloudflare-one/[...entry]/index.md.ts

@@ -2,6 +2,16 @@ import { WorkerEntrypoint } from "cloudflare:workers";
 import { generateRedirectsEvaluator } from "redirects-in-workers";
 import redirectsFileContents from "../dist/__redirects";
 
+import { parse } from "node-html-parser";
+import { process } from "../src/util/rehype";
+
+import rehypeParse from "rehype-parse";
+import rehypeBaseUrl from "../src/plugins/rehype/base-url";
+import rehypeFilterElements from "../src/plugins/rehype/filter-elements";
+import remarkGfm from "remark-gfm";
+import rehypeRemark from "rehype-remark";
+import remarkStringify from "remark-stringify";
+
 const redirectsEvaluator = generateRedirectsEvaluator(redirectsFileContents, {
 	maxLineLength: 10_000, // Usually 2_000
 	maxStaticRules: 10_000, // Usually 2_000
@@ -10,6 +20,45 @@ const redirectsEvaluator = generateRedirectsEvaluator(redirectsFileContents, {
 
 export default class extends WorkerEntrypoint<Env> {
 	override async fetch(request: Request) {
+		if (request.url.endsWith("/index.md")) {
+			const res = await this.env.ASSETS.fetch(
+				request.url.replace("index.md", ""),
+				request,
+			);
+
+			if (res.status === 404) {
+				return res;
+			}
+
+			if (
+				res.status === 200 &&
+				res.headers.get("content-type")?.startsWith("text/html")
+			) {
+				const html = await res.text();
+
+				const content = parse(html).querySelector(".sl-markdown-content");
+
+				if (!content) {
+					return new Response("Not Found", { status: 404 });
+				}
+
+				const markdown = await process(content.toString(), [
+					rehypeParse,
+					rehypeBaseUrl,
+					rehypeFilterElements,
+					remarkGfm,
+					rehypeRemark,
+					remarkStringify,
+				]);
+
+				return new Response(markdown, {
+					headers: {
+						"content-type": "text/markdown; charset=utf-8",
+					},
+				});
+			}
+		}
+
 		try {
 			try {
 				const redirect = await redirectsEvaluator(request, this.env.ASSETS);