From 2b5ddb883aaf48fb805f9215adc8c22cb1c26257 Mon Sep 17 00:00:00 2001 From: kodster28 Date: Fri, 30 May 2025 12:19:24 -0500 Subject: [PATCH 1/5] [Docs Site] New schema value for Support AI --- src/schemas/base.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/schemas/base.ts b/src/schemas/base.ts index ad3a669224addb..a6fa31c49d95a5 100644 --- a/src/schemas/base.ts +++ b/src/schemas/base.ts @@ -97,7 +97,13 @@ export const baseSchema = ({ image }: SchemaContext) => .boolean() .optional() .describe( - "If true, this property adds a `noindex` declaration to the page, which will tell internal / external search crawlers to ignore this page. Helpful for pages that are historically accurate, but no longer recommended, such as [Workers Sites](/workers/configuration/sites/).", + "If true, this property adds a `noindex` declaration to the page, which will tell internal / external search crawlers to ignore this page. Helpful for pages that are historically accurate, but no longer recommended, such as [Workers Sites](/workers/configuration/sites/). Companion to the `chatbot_relevant` property.", + ), + chatbot_relevant: z + .boolean() + .optional() + .describe( + "If false, this property will de-prioritize this page in the responses surfaced by Support AI. Helpful for pages that are historically accurate, but no longer recommended, such as [Workers Sites](/workers/configuration/sites/). Companion to the `noindex` property.", ), sidebar, hideChildren: z From 15d5d288f45bd8557b98b6e4cd224a7765dd27ab Mon Sep 17 00:00:00 2001 From: kodster28 Date: Fri, 30 May 2025 12:24:38 -0500 Subject: [PATCH 2/5] Test value --- src/content/docs/firewall/index.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/content/docs/firewall/index.mdx b/src/content/docs/firewall/index.mdx index d796b55b77b9c9..efa9ffacf50def 100644 --- a/src/content/docs/firewall/index.mdx +++ b/src/content/docs/firewall/index.mdx @@ -8,6 +8,7 @@ sidebar: head: - tag: title content: Cloudflare Firewall Rules (deprecated) +chatbot_relevant: false --- import { FeatureTable, Render } from "~/components"; From d3f8d40e995c466a64900e23fe64e44de4a554f2 Mon Sep 17 00:00:00 2001 From: kodster28 Date: Fri, 30 May 2025 12:45:45 -0500 Subject: [PATCH 3/5] Update framing of frontmatter + logic to render in index.md --- src/components/overrides/Head.astro | 10 ++++++++++ src/content/docs/firewall/index.mdx | 2 +- src/schemas/base.ts | 6 +++--- src/util/markdown.ts | 4 ++++ 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/components/overrides/Head.astro b/src/components/overrides/Head.astro index 175a8e4f25d156..808229e12186bb 100644 --- a/src/components/overrides/Head.astro +++ b/src/components/overrides/Head.astro @@ -83,6 +83,16 @@ if (shouldNoIndex) { }); } +if (frontmatter.chatbot_deprioritize) { + head.push({ + tag: "meta", + attrs: { + name: "pcx_chatbot_deprioritize", + content: frontmatter.chatbot_deprioritize, + }, + }); +} + if ( frontmatter.description && head.findIndex( diff --git a/src/content/docs/firewall/index.mdx b/src/content/docs/firewall/index.mdx index efa9ffacf50def..9a282d9cd333c8 100644 --- a/src/content/docs/firewall/index.mdx +++ b/src/content/docs/firewall/index.mdx @@ -8,7 +8,7 @@ sidebar: head: - tag: title content: Cloudflare Firewall Rules (deprecated) -chatbot_relevant: false +chatbot_deprioritize: true --- import { FeatureTable, Render } from "~/components"; diff --git a/src/schemas/base.ts b/src/schemas/base.ts index a6fa31c49d95a5..3b285b840fbc2e 100644 --- a/src/schemas/base.ts +++ b/src/schemas/base.ts @@ -97,13 +97,13 @@ export const baseSchema = ({ image }: SchemaContext) => .boolean() .optional() .describe( - "If true, this property adds a `noindex` declaration to the page, which will tell internal / external search crawlers to ignore this page. Helpful for pages that are historically accurate, but no longer recommended, such as [Workers Sites](/workers/configuration/sites/). Companion to the `chatbot_relevant` property.", + "If true, this property adds a `noindex` declaration to the page, which will tell internal / external search crawlers to ignore this page. Helpful for pages that are historically accurate, but no longer recommended, such as [Workers Sites](/workers/configuration/sites/). Companion to the `chatbot_deprioritize` property.", ), - chatbot_relevant: z + chatbot_deprioritize: z .boolean() .optional() .describe( - "If false, this property will de-prioritize this page in the responses surfaced by Support AI. Helpful for pages that are historically accurate, but no longer recommended, such as [Workers Sites](/workers/configuration/sites/). Companion to the `noindex` property.", + "If true, this property will de-prioritize this page in the responses surfaced by Support AI. Helpful for pages that are historically accurate, but no longer recommended, such as [Workers Sites](/workers/configuration/sites/). Companion to the `noindex` property.", ), sidebar, hideChildren: z diff --git a/src/util/markdown.ts b/src/util/markdown.ts index 2605c2e9944b8d..60f241ff1e4ec3 100644 --- a/src/util/markdown.ts +++ b/src/util/markdown.ts @@ -32,12 +32,16 @@ export async function htmlToMarkdown( const description = dom.querySelector("meta[name='description']")?.attributes .content; const lastUpdated = dom.querySelector(".meta time")?.attributes.datetime; + const chatbotDeprioritize = dom.querySelector( + "meta[name='pcx_chatbot_deprioritize']", + )?.attributes.content; const withFrontmatter = [ "---", `title: ${title}`, description ? `description: ${description}` : [], lastUpdated ? `lastUpdated: ${lastUpdated}` : [], + chatbotDeprioritize ? `chatbotDeprioritize: ${chatbotDeprioritize}` : [], `source_url:`, ` html: ${url.replace("index.md", "")}`, ` md: ${url}`, From 81cdeb49b105bd8dc0eeec38f177017507c2de16 Mon Sep 17 00:00:00 2001 From: kodster28 Date: Fri, 30 May 2025 13:00:54 -0500 Subject: [PATCH 4/5] Add whole product option like noindex --- src/components/overrides/Head.astro | 9 +++++++-- src/content/docs/firewall/index.mdx | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/components/overrides/Head.astro b/src/components/overrides/Head.astro index 808229e12186bb..cf92615d13a2fa 100644 --- a/src/components/overrides/Head.astro +++ b/src/components/overrides/Head.astro @@ -9,6 +9,7 @@ import type { CollectionEntry } from "astro:content"; const DEFAULT_TITLE_DELIMITER = "|"; const NOINDEX_PRODUCTS = ["email-security", "style-guide", "security"]; +const CHATBOT_DEPRIORITIZE_PRODUCTS = ["firewall"]; const currentSection = Astro.url.pathname.split("/")[1].replaceAll(".", ""); @@ -22,6 +23,10 @@ const shouldNoIndex = frontmatter.noindex || frontmatter.external_link; +const shouldChatbotDeprioritize = + CHATBOT_DEPRIORITIZE_PRODUCTS.includes(currentSection) || + frontmatter.chatbot_deprioritize; + if (currentSection) { const product = await getEntry("products", currentSection); @@ -83,12 +88,12 @@ if (shouldNoIndex) { }); } -if (frontmatter.chatbot_deprioritize) { +if (shouldChatbotDeprioritize) { head.push({ tag: "meta", attrs: { name: "pcx_chatbot_deprioritize", - content: frontmatter.chatbot_deprioritize, + content: true, }, }); } diff --git a/src/content/docs/firewall/index.mdx b/src/content/docs/firewall/index.mdx index 9a282d9cd333c8..d796b55b77b9c9 100644 --- a/src/content/docs/firewall/index.mdx +++ b/src/content/docs/firewall/index.mdx @@ -8,7 +8,6 @@ sidebar: head: - tag: title content: Cloudflare Firewall Rules (deprecated) -chatbot_deprioritize: true --- import { FeatureTable, Render } from "~/components"; From 13114dd6f0b3fc64e4ac937eff2ac460d8cd47c6 Mon Sep 17 00:00:00 2001 From: kodster28 Date: Mon, 2 Jun 2025 11:12:26 -0500 Subject: [PATCH 5/5] Add tags to .md as well --- src/util/markdown.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/util/markdown.ts b/src/util/markdown.ts index 60f241ff1e4ec3..fb0c2e1ae02338 100644 --- a/src/util/markdown.ts +++ b/src/util/markdown.ts @@ -35,6 +35,7 @@ export async function htmlToMarkdown( const chatbotDeprioritize = dom.querySelector( "meta[name='pcx_chatbot_deprioritize']", )?.attributes.content; + const tags = dom.querySelector("meta[name='pcx_tags']")?.attributes.content; const withFrontmatter = [ "---", @@ -42,6 +43,7 @@ export async function htmlToMarkdown( description ? `description: ${description}` : [], lastUpdated ? `lastUpdated: ${lastUpdated}` : [], chatbotDeprioritize ? `chatbotDeprioritize: ${chatbotDeprioritize}` : [], + tags ? `tags: ${tags}` : [], `source_url:`, ` html: ${url.replace("index.md", "")}`, ` md: ${url}`,