[Docs Site] New schema value for Support AI (cloudflare#22794)

* [Docs Site] New schema value for Support AI

* Test value

* Update framing of frontmatter + logic to render in index.md

* Add whole product option like noindex

* Add tags to .md as well

Author: kodster28

src/components/overrides/Head.astro

@@ -9,6 +9,7 @@ import type { CollectionEntry } from "astro:content";
 
 const DEFAULT_TITLE_DELIMITER = "|";
 const NOINDEX_PRODUCTS = ["email-security", "style-guide", "security"];
+const CHATBOT_DEPRIORITIZE_PRODUCTS = ["firewall"];
 
 const currentSection = Astro.url.pathname.split("/")[1].replaceAll(".", "");
 
@@ -22,6 +23,10 @@ const shouldNoIndex =
 	frontmatter.noindex ||
 	frontmatter.external_link;
 
+const shouldChatbotDeprioritize =
+	CHATBOT_DEPRIORITIZE_PRODUCTS.includes(currentSection) ||
+	frontmatter.chatbot_deprioritize;
+
 if (currentSection) {
 	const product = await getEntry("products", currentSection);
 
@@ -83,6 +88,16 @@ if (shouldNoIndex) {
 	});
 }
 
+if (shouldChatbotDeprioritize) {
+	head.push({
+		tag: "meta",
+		attrs: {
+			name: "pcx_chatbot_deprioritize",
+			content: true,
+		},
+	});
+}
+
 if (
 	frontmatter.description &&
 	head.findIndex(

src/schemas/base.ts

@@ -97,7 +97,13 @@ export const baseSchema = ({ image }: SchemaContext) =>
 			.boolean()
 			.optional()
 			.describe(
-				"If true, this property adds a `noindex` declaration to the page, which will tell internal / external search crawlers to ignore this page. Helpful for pages that are historically accurate, but no longer recommended, such as [Workers Sites](/workers/configuration/sites/).",
+				"If true, this property adds a `noindex` declaration to the page, which will tell internal / external search crawlers to ignore this page. Helpful for pages that are historically accurate, but no longer recommended, such as [Workers Sites](/workers/configuration/sites/). Companion to the `chatbot_deprioritize` property.",
+			),
+		chatbot_deprioritize: z
+			.boolean()
+			.optional()
+			.describe(
+				"If true, this property will de-prioritize this page in the responses surfaced by Support AI. Helpful for pages that are historically accurate, but no longer recommended, such as [Workers Sites](/workers/configuration/sites/). Companion to the `noindex` property.",
 			),
 		sidebar,
 		hideChildren: z

src/util/markdown.ts

@@ -32,12 +32,18 @@ export async function htmlToMarkdown(
 	const description = dom.querySelector("meta[name='description']")?.attributes
 		.content;
 	const lastUpdated = dom.querySelector(".meta time")?.attributes.datetime;
+	const chatbotDeprioritize = dom.querySelector(
+		"meta[name='pcx_chatbot_deprioritize']",
+	)?.attributes.content;
+	const tags = dom.querySelector("meta[name='pcx_tags']")?.attributes.content;
 
 	const withFrontmatter = [
 		"---",
 		`title: ${title}`,
 		description ? `description: ${description}` : [],
 		lastUpdated ? `lastUpdated: ${lastUpdated}` : [],
+		chatbotDeprioritize ? `chatbotDeprioritize: ${chatbotDeprioritize}` : [],
+		tags ? `tags: ${tags}` : [],
 		`source_url:`,
 		`  html: ${url.replace("index.md", "")}`,
 		`  md: ${url}`,