update script + run over tutorial files

Author: kodster28

bin/generate-descriptions.ts

@@ -4,8 +4,8 @@
  * This script generates descriptions for MDX files in the docs directory
  * that don't have a description field in their frontmatter.
  *
- * It uses the rendered HTML from the dist directory to generate descriptions
- * using the generateDescription function from src/util/props.ts.
+ * It uses the rendered markdown from the distmd directory to generate descriptions
+ * by sending the content to a localhost:8788 application.
  *
  * Usage:
  * npm run generate-descriptions [-- --pcx-content-type <type>]
@@ -17,57 +17,57 @@
 import fs from "fs/promises";
 import path from "path";
 import globby from "fast-glob";
-import { parse as parseHTML } from "node-html-parser";
-import { generateDescription } from "../src/util/props";
 import matter from "gray-matter";
 
 const DOCS_DIR = path.join(process.cwd(), "src/content/docs");
-const DIST_DIR = path.join(process.cwd(), "dist");
+const DISTMD_DIR = path.join(process.cwd(), "distmd");
 
-// Maximum length for descriptions
-const MAX_DESCRIPTION_LENGTH = 160;
+// Localhost application URL
+const LOCALHOST_URL = "http://localhost:8787";
 
 /**
- * Extracts the first paragraph from HTML content
+ * Sends text content to localhost application and receives description back
  */
-function extractFirstParagraph(html: string): string | undefined {
-	const dom = parseHTML(html);
-	const paragraph = dom.querySelector("p");
-
-	if (paragraph) {
-		return paragraph.textContent.trim();
-	}
-
-	return undefined;
-}
-
-/**
- * Truncates a description to a reasonable length
- */
-function truncateDescription(description: string): string {
-	if (description.length <= MAX_DESCRIPTION_LENGTH) {
-		return description;
-	}
+async function generateDescriptionFromAPI(
+	content: string,
+): Promise<string | undefined> {
+	try {
+		const response = await fetch(LOCALHOST_URL, {
+			method: "POST",
+			headers: {
+				"Content-Type": "text/plain",
+			},
+			body: content,
+		});
 
-	// Truncate at the last space before MAX_DESCRIPTION_LENGTH
-	const truncated = description.substring(0, MAX_DESCRIPTION_LENGTH);
-	const lastSpace = truncated.lastIndexOf(" ");
+		if (!response.ok) {
+			throw new Error(`HTTP error! status: ${response.status}`);
+		}
 
-	if (lastSpace > 0) {
-		return truncated.substring(0, lastSpace) + "...";
+		const description = await response.text();
+		// Remove surrounding quotes if they exist
+		const trimmed = description.trim();
+		if ((trimmed.startsWith('"') && trimmed.endsWith('"')) || 
+		    (trimmed.startsWith("'") && trimmed.endsWith("'"))) {
+			return trimmed.slice(1, -1);
+		}
+		return trimmed;
+	} catch (error) {
+		console.error("Error calling localhost API:", error);
+		return undefined;
 	}
-
-	return truncated + "...";
 }
 
 /**
- * Gets the rendered HTML path for a docs file
+ * Gets the rendered markdown path for a docs file
  */
 function getRenderedPath(docPath: string): string {
-	// Convert /src/content/docs/product/path/file.mdx to /dist/product/path/file/index.html
+	// Convert /src/content/docs/product/path/file.mdx to /distmd/product/path/file/index.md
 	const relativePath = path.relative(DOCS_DIR, docPath);
 	const pathWithoutExt = relativePath.replace(/\.mdx$/, "");
-	return path.join(DIST_DIR, pathWithoutExt, "index.html");
+	const filename = path.basename(pathWithoutExt);
+	const dirPath = path.dirname(pathWithoutExt);
+	return path.join(DISTMD_DIR, dirPath, filename, "index.md");
 }
 
 /**
@@ -224,43 +224,33 @@ async function main() {
 					continue;
 				}
 
-				// Get the rendered HTML path
+				// Get the rendered markdown path
 				const renderedPath = getRenderedPath(mdxFile);
 
-				// Check if rendered HTML exists
+				// Check if rendered markdown exists
 				try {
 					await fs.access(renderedPath);
 				} catch (error) {
-					console.log(error);
 					console.warn(
-						`⚠️ Rendered HTML not found for ${path.relative(process.cwd(), mdxFile)}`,
+						`⚠️ Rendered markdown not found for ${path.relative(process.cwd(), mdxFile)}`,
 					);
 					errorCount++;
 					continue;
 				}
 
-				// Read rendered HTML
-				const html = await fs.readFile(renderedPath, "utf-8");
-
-				// Extract main content from HTML
-				const dom = parseHTML(html);
-				const mainContent = dom.querySelector("main")?.innerHTML || "";
+				// Read rendered markdown content
+				const markdownContent = await fs.readFile(renderedPath, "utf-8");
 
-				if (!mainContent) {
+				if (!markdownContent.trim()) {
 					console.warn(
-						`⚠️ No main content found in rendered HTML for ${path.relative(process.cwd(), mdxFile)}`,
+						`⚠️ Empty markdown content found for ${path.relative(process.cwd(), mdxFile)}`,
 					);
 					errorCount++;
 					continue;
 				}
 
-				// Generate description
-				let description = await generateDescription({ html: mainContent });
-
-				// If no description was generated, try extracting the first paragraph
-				if (!description) {
-					description = extractFirstParagraph(mainContent);
-				}
+				// Generate description using localhost API
+				const description = await generateDescriptionFromAPI(markdownContent);
 
 				// Skip if no description could be generated
 				if (!description) {
@@ -271,9 +261,6 @@ async function main() {
 					continue;
 				}
 
-				// Truncate description if needed
-				description = truncateDescription(description);
-
 				// Update frontmatter
 				const wasUpdated = await updateFrontmatter(mdxFile, description);
 				if (wasUpdated) {

src/content/docs/1.1.1.1/additional-options/dns-in-google-sheets.mdx

@@ -2,7 +2,8 @@
 pcx_content_type: tutorial
 title: DNS in Google Sheets
 slug: 1.1.1.1/additional-options/dns-in-google-sheets
-
+description: >-
+  Cloudflare 1.1.1 works directly inside Google Sheets. To get started, create a Google Function with the following code.
 ---
 
 import { Details } from "~/components"

src/content/docs/1.1.1.1/additional-options/dns-over-discord.mdx

@@ -2,7 +2,8 @@
 pcx_content_type: tutorial
 title: DNS over Discord
 slug: 1.1.1.1/additional-options/dns-over-discord
-
+description: >-
+  1.1. 1.1 works from a Discord server. Invite the bot to your Discord server to start using DNS over Discord. Or, add it to your account to use it anywhere in Discord.
 ---
 
 import { Details } from "~/components"

src/content/docs/1.1.1.1/additional-options/dns-over-tor.mdx

@@ -2,6 +2,8 @@
 pcx_content_type: tutorial
 title: DNS over Tor
 slug: 1.1.1.1/additional-options/dns-over-tor
+description: >-
+  If you do not want to disclose your IP address to the resolver, you can use our Tor onion service. Resolving DNS queries through the Tor network guarantees a significantly higher level of anonymity. The hidden resolver is set up to listen on TCP ports 53 and 8 for HTTPS.
 ---
 
 :::caution

src/content/docs/ai-gateway/integrations/aig-workers-ai-binding.mdx

@@ -2,6 +2,8 @@
 title: Workers AI
 pcx_content_type: tutorial
 reviewed: 2024-10-17
+description: >-
+  This guide will walk you through setting up and deploying a Workers AI project. You will use Workers, an AI Gateway binding, and a large language model (LLM) to deploy your first AI-powered application on the Cloudflare global network.
 ---
 
 import { Render, PackageManagers, WranglerConfig } from "~/components";

src/content/docs/ai-gateway/integrations/worker-binding-methods.mdx

@@ -4,6 +4,8 @@ pcx_content_type: tutorial
 tags:
   - Bindings
 reviewed: 2025-04-01
+description: >-
+  This guide provides an overview of how to use the latest Cloudflare Workers AI Gateway binding methods. You will learn how to set up an AI. Gateway binding, access new methods, and integrate them into your Workers.
 ---
 
 import { Render, PackageManagers } from "~/components";

src/content/docs/ai-gateway/tutorials/create-first-aig-workers.mdx

@@ -3,6 +3,8 @@ pcx_content_type: tutorial
 difficulty: Beginner
 reviewed: 2024-08-01
 title: Create your first AI Gateway using Workers AI
+description: >-
+  This tutorial guides you through creating your first AI Gateway using Workers AI on the Cloudflare dashboard. The intended audience is beginners who are new to AI Gateway and Workers AI. An AI Gateway enables the user to efficiently manage and secure AI requests.
 ---
 
 import { Render } from "~/components";

src/content/docs/analytics/analytics-integrations/datadog.mdx

@@ -3,6 +3,8 @@ pcx_content_type: tutorial
 title: Datadog
 sidebar:
   order: 98
+description: >-
+  This tutorial explains how to analyze Cloudflare metrics using the [Cloudflare Integration tile for Datadog]
 ---
 
 This tutorial explains how to analyze Cloudflare metrics using the [Cloudflare Integration tile for Datadog](https://docs.datadoghq.com/integrations/cloudflare/).

src/content/docs/analytics/analytics-integrations/graylog.mdx

@@ -3,7 +3,8 @@ pcx_content_type: tutorial
 title: Graylog
 sidebar:
   order: 102
-
+description: >-
+  This tutorial explains how to analyze [Cloudflare Logs] using Graylog. The Graylog integration is available on GitHub.
 ---
 
 This tutorial explains how to analyze [Cloudflare Logs](https://www.cloudflare.com/products/cloudflare-logs/) using [Graylog](https://github.com/Graylog2/graylog-s3-lambda/blob/master/content-packs/cloudflare/cloudflare-logpush-content-pack.json).

src/content/docs/analytics/analytics-integrations/new-relic.mdx

@@ -3,6 +3,8 @@ pcx_content_type: tutorial
 title: New Relic
 sidebar:
   order: 103
+description: >-
+  This tutorial explains how to analyze Cloudflare metrics using the New Relic One Cloudflar Quickstart.
 ---
 
 This tutorial explains how to analyze Cloudflare metrics using the [New Relic One Cloudflare Quickstart](https://newrelic.com/instant-observability/cloudflare/fc2bb0ac-6622-43c6-8c1f-6a4c26ab5434).