Merge branch 'production' into marcio/pcx17803-mconn-kvm

Author: marciocloudflare

.github/workflows/publish-production.yml

@@ -41,18 +41,36 @@ jobs:
           cd distmd && zip -r markdown.zip .
           npx wrangler r2 object put vendored-markdown/markdown.zip --file=markdown.zip --remote
           rm markdown.zip
+          cd ..
+
+          cd distllms
+          for file in $(find . -type f); do
+            npx wrangler r2 object put vendored-markdown/$file --file=$file --remote
+          done
+      - name: Install rclone
+        run: sudo -v ; curl https://rclone.org/install.sh | sudo bash
       - name: Upload vendored Markdown files to ZT DevDocs bucket
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.ZT_DEVDOCS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.ZT_DEVDOCS_SECRET_ACCESS_KEY }}
         run: |
-          sudo -v ; curl https://rclone.org/install.sh | sudo bash
           rclone sync \
             --s3-env-auth \
             --s3-provider="Cloudflare" \
             --s3-endpoint="https://e76c849bd111ee7d3006b6625713991e.r2.cloudflarestorage.com" \
             distmd \
             :s3:/zt-dashboard-dev-docs
+      - name: Upload vendored Markdown files to AutoRAG DevDocs bucket
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AUTORAG_DEVDOCS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AUTORAG_DEVDOCS_SECRET_ACCESS_KEY }}
+        run: |
+          rclone sync \
+            --s3-env-auth \
+            --s3-provider="Cloudflare" \
+            --s3-endpoint="https://cf9267a5bf4f6b3cceca48d554d81679.r2.cloudflarestorage.com" \
+            distmd \
+            :s3:/developer-docs-full
       - uses: actions/cache/save@v4
         if: always()
         with:

.gitignore

@@ -1,6 +1,7 @@
 # build output
 dist/
 distmd/
+distllms/
 # generated types
 .astro/
 

astro.config.ts

@@ -137,6 +137,8 @@ export default defineConfig({
 									"/api/**",
 									"/changelog/**",
 									"/http/resources/**",
+									"/llms.txt",
+									"/llms-full.txt",
 									"{props.*}",
 									"/",
 									"/glossary/",
@@ -146,7 +148,12 @@ export default defineConfig({
 									"/ruleset-engine/rules-language/fields/reference/**",
 									"/workers/examples/?languages=*",
 									"/workers/examples/?tags=*",
+									"/workers/llms-full.txt",
 									"/workers-ai/models/**",
+									"**index.md",
+									"/markdown.zip",
+									"/style-guide/index.md",
+									"/style-guide/fixtures/markdown/index.md",
 								],
 							}),
 						]

bin/generate-index-md.ts

@@ -1,12 +1,23 @@
-import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import {
+	readFileSync,
+	writeFileSync,
+	mkdirSync,
+	appendFileSync,
+} from "node:fs";
 
 import glob from "fast-glob";
 import { parse } from "node-html-parser";
 import { htmlToMarkdown } from "~/util/markdown";
 
+import YAML from "yaml";
+
 const files = await glob("dist/**/*.html");
 
 for (const file of files) {
+	if (file === "dist/index.html" || file === "dist/404.html") {
+		continue;
+	}
+
 	const html = readFileSync(file, "utf-8");
 	const dom = parse(html);
 
@@ -24,9 +35,42 @@ for (const file of files) {
 		continue;
 	}
 
+	const product = file.split("/")[1];
 	const path = file.replace("dist/", "distmd/").replace(".html", ".md");
 
 	mkdirSync(path.split("/").slice(0, -1).join("/"), { recursive: true });
-
 	writeFileSync(path, markdown);
+
+	const llmsFullContent = ["<page>", markdown, "</page>\n\n"].join("\n");
+
+	mkdirSync(`distllms/${product}`, { recursive: true });
+	appendFileSync("distllms/llms-full.txt", llmsFullContent);
+	appendFileSync(`distllms/${product}/llms-full.txt`, llmsFullContent);
+
+	try {
+		const path = await glob(`src/content/products/${product}.*`).then((arr) =>
+			arr.at(0),
+		);
+
+		if (!path) {
+			continue;
+		}
+
+		const yaml = YAML.parse(readFileSync(path, "utf-8"));
+		const group = yaml.product?.group?.replaceAll(" ", "-").toLowerCase();
+
+		if (!group) {
+			continue;
+		}
+
+		mkdirSync(`distllms/${group}`, { recursive: true });
+		appendFileSync(`distllms/${group}/llms-full.txt`, llmsFullContent);
+	} catch (error) {
+		if (error instanceof Error) {
+			console.error(
+				`Failed to find a product group for ${product}:`,
+				error.message,
+			);
+		}
+	}
 }