[Docs Site] Add frontmatter to index.md output (#21980)

* [Docs Site] Add frontmatter to index.md output

* rebase + update test

* fix title and desc

* hardcode lastUpdated

Author: KianNH

src/components/overrides/Head.astro

@@ -7,6 +7,7 @@ import { getEntry } from "astro:content";
 import { getOgImage } from "~/util/og";
 import type { CollectionEntry } from "astro:content";
 
+const DEFAULT_TITLE_DELIMITER = "|";
 const NOINDEX_PRODUCTS = ["email-security", "style-guide", "security"];
 
 const currentSection = Astro.url.pathname.split("/")[1].replaceAll(".", "");
@@ -31,7 +32,9 @@ if (currentSection) {
 			let title: string;
 
 			if (titleIdx !== -1) {
-				const existingTitle = head[titleIdx].content;
+				const existingTitle = head[titleIdx].content?.split(
+					` ${DEFAULT_TITLE_DELIMITER} `,
+				)[0];
 				title = `${existingTitle} · ${product.data.meta.title}`;
 
 				head[titleIdx] = {
@@ -80,6 +83,26 @@ if (shouldNoIndex) {
 	});
 }
 
+if (
+	frontmatter.description &&
+	head.findIndex(
+		({ tag, attrs }) => tag === "meta" && attrs?.name === "description",
+	) === -1
+) {
+	const existingOpenGraphTag = head.findIndex(
+		({ tag, attrs }) => tag === "meta" && attrs?.property === "og:description",
+	);
+
+	if (existingOpenGraphTag !== -1) {
+		head[existingOpenGraphTag].attrs!.content = frontmatter.description;
+	}
+
+	metaTags.push({
+		name: "description",
+		content: frontmatter.description as string,
+	});
+}
+
 if (frontmatter.pcx_content_type) {
 	["pcx_content_type", "algolia_content_type"].map((name) => {
 		metaTags.push({

src/content/docs/style-guide/fixtures/markdown.mdx

@@ -1,6 +1,7 @@
 ---
 title: Markdown
 noindex: true
+lastUpdated: 2025-01-01T00:00:00Z
 sidebar:
   hidden: true
 ---

src/middleware/index.ts

@@ -18,7 +18,7 @@ export const onRequest = defineMiddleware(async (context, next) => {
 			const htmlUrl = new URL(pathname.replace("index.md", ""), context.url);
 			const html = await (await fetch(htmlUrl)).text();
 
-			const markdown = await htmlToMarkdown(html);
+			const markdown = await htmlToMarkdown(html, context.url.toString());
 
 			if (!markdown) {
 				return new Response("Not Found", { status: 404 });

src/util/markdown.ts

@@ -10,8 +10,10 @@ import remarkStringify from "remark-stringify";
 
 export async function htmlToMarkdown(
 	html: string,
+	url: string,
 ): Promise<string | undefined> {
-	const content = parse(html).querySelector(".sl-markdown-content");
+	const dom = parse(html);
+	const content = dom.querySelector(".sl-markdown-content");
 
 	if (!content) {
 		return;
@@ -26,5 +28,24 @@ export async function htmlToMarkdown(
 		remarkStringify,
 	]);
 
-	return markdown;
+	const title = dom.querySelector("title")?.textContent;
+	const description = dom.querySelector("meta[name='description']")?.attributes
+		.content;
+	const lastUpdated = dom.querySelector(".meta time")?.attributes.datetime;
+
+	const withFrontmatter = [
+		"---",
+		`title: ${title}`,
+		description ? `description: ${description}` : [],
+		lastUpdated ? `lastUpdated: ${lastUpdated}` : [],
+		`source_url:`,
+		`  html: ${url}`,
+		`  md: ${url.replace("index.md", "")}`,
+		"---\n",
+		markdown,
+	]
+		.flat()
+		.join("\n");
+
+	return withFrontmatter;
 }

worker/index.ts

@@ -13,10 +13,8 @@ const redirectsEvaluator = generateRedirectsEvaluator(redirectsFileContents, {
 export default class extends WorkerEntrypoint<Env> {
 	override async fetch(request: Request) {
 		if (request.url.endsWith("/index.md")) {
-			const res = await this.env.ASSETS.fetch(
-				request.url.replace("index.md", ""),
-				request,
-			);
+			const htmlUrl = request.url.replace("index.md", "");
+			const res = await this.env.ASSETS.fetch(htmlUrl, request);
 
 			if (res.status === 404) {
 				return res;
@@ -28,7 +26,7 @@ export default class extends WorkerEntrypoint<Env> {
 			) {
 				const html = await res.text();
 
-				const markdown = await htmlToMarkdown(html);
+				const markdown = await htmlToMarkdown(html, request.url);
 
 				if (!markdown) {
 					return new Response("Not Found", { status: 404 });

worker/index.worker.test.ts

@@ -267,7 +267,16 @@ describe("Cloudflare Docs", () => {
 
 			const text = await response.text();
 			expect(text).toMatchInlineSnapshot(`
-				"The HTML generated by this file is used as a test fixture for our Markdown generation.
+				"---
+				title: Markdown · Cloudflare Style Guide
+				description: The HTML generated by this file is used as a test fixture for our Markdown generation.
+				lastUpdated: 2025-01-01T00:00:00.000Z
+				source_url:
+				  html: http://fakehost/style-guide/fixtures/markdown/index.md
+				  md: http://fakehost/style-guide/fixtures/markdown/
+				---
+
+				The HTML generated by this file is used as a test fixture for our Markdown generation.
 
 				* mdx