rebase + update test

Author: KianNH

src/middleware/index.ts

@@ -18,7 +18,7 @@ export const onRequest = defineMiddleware(async (context, next) => {
 			const htmlUrl = new URL(pathname.replace("index.md", ""), context.url);
 			const html = await (await fetch(htmlUrl)).text();
 
-			const markdown = await htmlToMarkdown(html);
+			const markdown = await htmlToMarkdown(html, context.url.toString());
 
 			if (!markdown) {
 				return new Response("Not Found", { status: 404 });

src/util/markdown.ts

@@ -10,8 +10,10 @@ import remarkStringify from "remark-stringify";
 
 export async function htmlToMarkdown(
 	html: string,
+	url: string,
 ): Promise<string | undefined> {
-	const content = parse(html).querySelector(".sl-markdown-content");
+	const dom = parse(html);
+	const content = dom.querySelector(".sl-markdown-content");
 
 	if (!content) {
 		return;
@@ -26,5 +28,24 @@ export async function htmlToMarkdown(
 		remarkStringify,
 	]);
 
-	return markdown;
+	const title = dom.querySelector("title")?.textContent;
+	const description = dom.querySelector("meta[name='description']")?.attributes
+		.content;
+	const lastUpdated = dom.querySelector(".meta time")?.attributes.datetime;
+
+	const withFrontmatter = [
+		"---",
+		`title: ${title}`,
+		description ? `description: ${description}` : [],
+		lastUpdated ? `lastUpdated: ${lastUpdated}` : [],
+		`source_url:`,
+		`  html: ${url}`,
+		`  md: ${url.replace("index.md", "")}`,
+		"---\n",
+		markdown,
+	]
+		.flat()
+		.join("\n");
+
+	return withFrontmatter;
 }

worker/index.ts

@@ -26,7 +26,7 @@ export default class extends WorkerEntrypoint<Env> {
 			) {
 				const html = await res.text();
 
-				const markdown = await htmlToMarkdown(html);
+				const markdown = await htmlToMarkdown(html, request.url);
 
 				if (!markdown) {
 					return new Response("Not Found", { status: 404 });

worker/index.worker.test.ts

@@ -267,7 +267,15 @@ describe("Cloudflare Docs", () => {
 
 			const text = await response.text();
 			expect(text).toMatchInlineSnapshot(`
-				"The HTML generated by this file is used as a test fixture for our Markdown generation.
+				"---
+				title: Markdown · Cloudflare Style Guide
+				description: The HTML generated by this file is used as a test fixture for our Markdown generation.
+				source_url:
+				  html: http://fakehost/style-guide/fixtures/markdown/index.md
+				  md: http://fakehost/style-guide/fixtures/markdown/
+				---
+
+				The HTML generated by this file is used as a test fixture for our Markdown generation.
 
 				* mdx