[Docs Site] Generate index.md for all index.html files

Author: KianNH

.github/workflows/generate-index-md.yml

@@ -0,0 +1,22 @@
+name: Generate index.md files
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "*/15 * * * *"
+
+jobs:
+  generate:
+    name: Generate index.md files
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: "npm"
+      - run: npm ci
+      - run: npm run build
+        env:
+          NODE_OPTIONS: "--max-old-space-size=4192"
+      - run: npx tsx bin/generate-index-md.ts

bin/generate-index-md.ts

@@ -0,0 +1,60 @@
+import {
+	readFileSync,
+	writeFileSync,
+	appendFileSync,
+	mkdirSync,
+} from "node:fs";
+
+import { glob } from "tinyglobby";
+import { parse } from "node-html-parser";
+
+import { process } from "../src/util/rehype";
+import rehypeParse from "rehype-parse";
+import rehypeBaseUrl from "../src/plugins/rehype/base-url";
+import rehypeFilterElements from "../src/plugins/rehype/filter-elements";
+import remarkGfm from "remark-gfm";
+import rehypeRemark from "rehype-remark";
+import remarkStringify from "remark-stringify";
+
+const files = await glob("dist/**/*.html", {
+	ignore: [
+		"dist/index.html",
+		"dist/404.html",
+		"dist/magic-wan/legal/3rdparty/index.html",
+	],
+});
+
+for (const file of files) {
+	const text = readFileSync(file, "utf-8");
+	const html = parse(text);
+
+	const title = html.querySelector("#top")?.innerText;
+	const content = html.querySelector(".sl-markdown-content")?.innerHTML;
+
+	if (!content) {
+		continue;
+		throw new Error(`Couldn't find .sl-markdown-content selector in ${file}`);
+	}
+
+	let markdown = await process(content, [
+		rehypeParse,
+		rehypeBaseUrl,
+		rehypeFilterElements,
+		remarkGfm,
+		rehypeRemark,
+		remarkStringify,
+	]);
+
+	markdown = `# ${title}\n\n${markdown}`;
+
+	const segments = file.split("/").slice(1, -1);
+	const product = segments.at(0);
+	const folder = segments.join("/");
+
+	mkdirSync("distmd/" + folder, { recursive: true });
+
+	appendFileSync("distmd/llms-full.txt", markdown);
+	appendFileSync(`distmd/${product}/llms-full.txt`, markdown);
+
+	writeFileSync("distmd/" + file.slice(5).replace(".html", ".md"), markdown);
+}

package-lock.json

@@ -112,6 +112,7 @@
 		"strip-markdown": "6.0.0",
 		"svgo": "3.3.2",
 		"tailwindcss": "3.4.17",
+		"tinyglobby": "0.2.12",
 		"tippy.js": "6.3.7",
 		"ts-blank-space": "0.6.1",
 		"tsx": "4.19.3",

package.json

@@ -4,9 +4,10 @@ type: overview
 pcx_content_type: overview
 sidebar:
   order: 1
-head:
-  - tag: title
-    content: Overview
+  group:
+    label: foo
+    badge:
+      text: Beta
 ---
 
 import {