Merge branch 'production' into patricia/pcx17148-seq-custom-rules

Author: patriciasantaana

.github/CODEOWNERS

@@ -150,7 +150,6 @@
 
 # Docs team areas
 
-/src/content/docs/docs-guide/ @dcpena @cloudflare/pcx-technical-writing
 /src/content/docs/style-guide/ @dcpena @cloudflare/pcx-technical-writing
 
 # Google tag gateway

.github/workflows/publish-production.yml

@@ -33,26 +33,49 @@ jobs:
         name: Deploy to Cloudflare Workers
         env:
           CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
-      - name: Build vendored Markdown archive
+      - name: Install rclone
+        run: sudo -v ; curl https://rclone.org/install.sh | sudo bash
+      - name: Build vendored Markdown
+        run: npx tsx bin/generate-index-md.ts
+      - name: Upload vendored Markdown archives to Vendored Markdown bucket
         env:
-          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.VENDORED_DEVDOCS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.VENDORED_DEVDOCS_SECRET_ACCESS_KEY }}
         run: |
-          npx tsx bin/generate-index-md.ts
           cd distmd && zip -r markdown.zip .
-          npx wrangler r2 object put vendored-markdown/markdown.zip --file=markdown.zip --remote
+          rclone copy \
+            --config ../bin/rclone.conf \
+            ./markdown.zip \
+            devdocs:vendored-markdown
           rm markdown.zip
+          cd ..
+
+          cd distllms
+          for file in $(find . -type f); do
+            rclone copy \
+              --config ../bin/rclone.conf \
+              $file \
+              devdocs:vendored-markdown
+          done
+          cd ..
       - name: Upload vendored Markdown files to ZT DevDocs bucket
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.ZT_DEVDOCS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.ZT_DEVDOCS_SECRET_ACCESS_KEY }}
         run: |
-          sudo -v ; curl https://rclone.org/install.sh | sudo bash
           rclone sync \
-            --s3-env-auth \
-            --s3-provider="Cloudflare" \
-            --s3-endpoint="https://e76c849bd111ee7d3006b6625713991e.r2.cloudflarestorage.com" \
+            --config bin/rclone.conf \
+            distmd \
+            zt:zt-dashboard-dev-docs
+      - name: Upload vendored Markdown files to AutoRAG DevDocs bucket
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AUTORAG_DEVDOCS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AUTORAG_DEVDOCS_SECRET_ACCESS_KEY }}
+        run: |
+          rclone sync \
+            --config bin/rclone.conf \
             distmd \
-            :s3:/zt-dashboard-dev-docs
+            autorag:developer-docs-full
       - uses: actions/cache/save@v4
         if: always()
         with:

.gitignore

@@ -1,6 +1,7 @@
 # build output
 dist/
 distmd/
+distllms/
 # generated types
 .astro/
 

astro.config.ts

@@ -137,6 +137,8 @@ export default defineConfig({
 									"/api/**",
 									"/changelog/**",
 									"/http/resources/**",
+									"/llms.txt",
+									"/llms-full.txt",
 									"{props.*}",
 									"/",
 									"/glossary/",
@@ -146,7 +148,12 @@ export default defineConfig({
 									"/ruleset-engine/rules-language/fields/reference/**",
 									"/workers/examples/?languages=*",
 									"/workers/examples/?tags=*",
+									"/workers/llms-full.txt",
 									"/workers-ai/models/**",
+									"**index.md",
+									"/markdown.zip",
+									"/style-guide/index.md",
+									"/style-guide/fixtures/markdown/index.md",
 								],
 							}),
 						]

bin/.htmltest.yml

@@ -2,7 +2,9 @@ DirectoryPath: "dist"
 EnforceHTTPS: false
 IgnoreURLs:
   - "/api/*"
-  - "#_top"
+  - "/llms-full.txt$"
+IgnoreDir:
+  - "changelog"
 CheckLinks: true
 CheckImages: false
 CheckScripts: false

bin/generate-index-md.ts

@@ -1,12 +1,23 @@
-import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import {
+	readFileSync,
+	writeFileSync,
+	mkdirSync,
+	appendFileSync,
+} from "node:fs";
 
 import glob from "fast-glob";
 import { parse } from "node-html-parser";
 import { htmlToMarkdown } from "~/util/markdown";
 
+import YAML from "yaml";
+
 const files = await glob("dist/**/*.html");
 
 for (const file of files) {
+	if (file === "dist/index.html" || file === "dist/404.html") {
+		continue;
+	}
+
 	const html = readFileSync(file, "utf-8");
 	const dom = parse(html);
 
@@ -24,9 +35,42 @@ for (const file of files) {
 		continue;
 	}
 
+	const product = file.split("/")[1];
 	const path = file.replace("dist/", "distmd/").replace(".html", ".md");
 
 	mkdirSync(path.split("/").slice(0, -1).join("/"), { recursive: true });
-
 	writeFileSync(path, markdown);
+
+	const llmsFullContent = ["<page>", markdown, "</page>\n\n"].join("\n");
+
+	mkdirSync(`distllms/${product}`, { recursive: true });
+	appendFileSync("distllms/llms-full.txt", llmsFullContent);
+	appendFileSync(`distllms/${product}/llms-full.txt`, llmsFullContent);
+
+	try {
+		const path = await glob(`src/content/products/${product}.*`).then((arr) =>
+			arr.at(0),
+		);
+
+		if (!path) {
+			continue;
+		}
+
+		const yaml = YAML.parse(readFileSync(path, "utf-8"));
+		const group = yaml.product?.group?.replaceAll(" ", "-").toLowerCase();
+
+		if (!group) {
+			continue;
+		}
+
+		mkdirSync(`distllms/${group}`, { recursive: true });
+		appendFileSync(`distllms/${group}/llms-full.txt`, llmsFullContent);
+	} catch (error) {
+		if (error instanceof Error) {
+			console.error(
+				`Failed to find a product group for ${product}:`,
+				error.message,
+			);
+		}
+	}
 }

bin/rclone.conf

@@ -0,0 +1,20 @@
+[devdocs]
+type = s3
+provider = Cloudflare
+env_auth = true
+endpoint = https://b54f07a6c269ecca2fa60f1ae4920c99.r2.cloudflarestorage.com
+acl = private
+
+[zt]
+type = s3
+provider = Cloudflare
+env_auth = true
+endpoint = https://e76c849bd111ee7d3006b6625713991e.r2.cloudflarestorage.com
+acl = private
+
+[autorag]
+type = s3
+provider = Cloudflare
+env_auth = true
+endpoint = https://cf9267a5bf4f6b3cceca48d554d81679.r2.cloudflarestorage.com
+acl = private