Making pages visible

Oxyjun · Oxyjun · commit 3076bb81717c · 2025-07-01T11:37:37.000+01:00
diff --git a/public/__redirects b/public/__redirects
@@ -160,6 +160,7 @@
 
 # ai audit
 /ai-audit/features/detect-ai-crawlers/ /ai-audit/features/analyze-ai-crawlers/ 301
+/ai-audit/features/enforce-robots-txt/ /ai-audit/features/analyze-ai-crawlers/ 301
 
 # agents
 /agents/capabilities/mcp-server/ /agents/model-context-protocol/ 301
diff --git a/src/content/docs/ai-audit/configuration/ai-audit-with-bots.mdx b/src/content/docs/ai-audit/configuration/ai-audit-with-bots.mdx
@@ -3,7 +3,6 @@ title: AI Audit with Cloudflare Bots
 pcx_content_type: concept
 sidebar:
   order: 5
-  hidden: true
 ---
 
 import { GlossaryTooltip, Example } from "~/components";
diff --git a/src/content/docs/ai-audit/configuration/ai-audit-with-waf.mdx b/src/content/docs/ai-audit/configuration/ai-audit-with-waf.mdx
@@ -3,7 +3,6 @@ title: AI Audit with Cloudflare WAF
 pcx_content_type: concept
 sidebar:
   order: 2
-  hidden: true
 
 ---
 
diff --git a/src/content/docs/ai-audit/configuration/index.mdx b/src/content/docs/ai-audit/configuration/index.mdx
@@ -2,7 +2,6 @@
 title: Configuration
 pcx_content_type: navigation
 sidebar:
-  hidden: true
   group:
     hideIndex: true
   order: 8
diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/index.mdx b/src/content/docs/ai-audit/features/pay-per-crawl/index.mdx
@@ -2,7 +2,6 @@
 title: Pay Per Crawl
 pcx_content_type: navigation
 sidebar:
-  hidden: true
   group:
     hideIndex: true
   order: 20
diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/connect-to-stripe.mdx b/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/connect-to-stripe.mdx
@@ -3,7 +3,6 @@ title: Connect Stripe
 pcx_content_type: concept
 sidebar:
   order: 4
-  hidden: true
 ---
 
 import { Steps } from "~/components";
diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/crawl-pages.mdx b/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/crawl-pages.mdx
@@ -3,7 +3,6 @@ title: Crawl pages
 pcx_content_type: concept
 sidebar:
   order: 15
-  hidden: true
 ---
 
 import { Steps } from "~/components";
diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/index.mdx b/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/index.mdx
@@ -2,7 +2,6 @@
 title: Use pay per crawl as an AI owner
 pcx_content_type: navigation
 sidebar:
-  hidden: true
   group:
     hideIndex: true
   order: 5
diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/set-up-cloudflare-account.mdx b/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/set-up-cloudflare-account.mdx
@@ -3,7 +3,6 @@ title: Set up your account
 pcx_content_type: concept
 sidebar:
   order: 2
-  hidden: true
 ---
 
 import { Steps } from "~/components";
diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/verify-ai-crawler.mdx b/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/verify-ai-crawler.mdx
@@ -3,7 +3,6 @@ title: Verify your AI crawler
 pcx_content_type: concept
 sidebar:
   order: 10
-  hidden: true
 ---
 
 import { Steps } from "~/components";
diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/connect-to-stripe.mdx b/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/connect-to-stripe.mdx
@@ -3,7 +3,6 @@ title: Connect Stripe
 pcx_content_type: concept
 sidebar:
   order: 4
-  hidden: true
 ---
 
 import { Steps } from "~/components";
diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/index.mdx b/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/index.mdx
@@ -2,7 +2,6 @@
 title: Use pay per crawl as a site owner
 pcx_content_type: navigation
 sidebar:
-  hidden: true
   group:
     hideIndex: true
   order: 5
diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/monitor-metrics.mdx b/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/monitor-metrics.mdx
@@ -3,7 +3,6 @@ title: Monitor metrics
 pcx_content_type: concept
 sidebar:
   order: 17
-  hidden: true
 ---
 
 import { Steps } from "~/components";
diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/select-crawlers-to-charge.mdx b/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/select-crawlers-to-charge.mdx
@@ -3,7 +3,6 @@ title: Select AI crawlers to charge
 pcx_content_type: concept
 sidebar:
   order: 13
-  hidden: true
 ---
 
 import { Steps } from "~/components";
diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-a-pay-per-crawl-price.mdx b/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-a-pay-per-crawl-price.mdx
@@ -3,7 +3,6 @@ title: Set a pay per crawl price
 pcx_content_type: concept
 sidebar:
   order: 10
-  hidden: true
 ---
 
 import { Steps } from "~/components";
diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-up-cloudflare-account.mdx b/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-up-cloudflare-account.mdx
@@ -3,7 +3,6 @@ title: Set up your account
 pcx_content_type: concept
 sidebar:
   order: 2
-  hidden: true
 ---
 
 import { Steps } from "~/components";
diff --git a/src/content/docs/ai-audit/reference/faq.mdx b/src/content/docs/ai-audit/reference/faq.mdx
@@ -3,7 +3,6 @@ title: FAQs
 pcx_content_type: concept
 sidebar:
   order: 2
-  hidden: true
 ---
 
 import { Steps } from "~/components";
diff --git a/src/content/glossary/ai-audit.yaml b/src/content/glossary/ai-audit.yaml
@@ -3,6 +3,12 @@ productName: AI Audit
 entries:
   - term: robots.txt
     general_definition: |-
-      A text file which lists pages in your website that are off-limits for bots. Well-behaved bots respect this file, but some bots may violate it. You can [enforce robots.txt](/ai-audit/features/enforce-robots-txt/) with Cloudflare WAF custom rules.
+      A text file which lists pages in your website that are off-limits for bots. Well-behaved bots respect this file, but some bots may violate it. You can enforce `robots.txt` with Cloudflare WAF custom rules.
 
+  - term: Merchant of Record
+    general_definition: |-
+      The entity who facilitates "buying and selling". For pay per crawl, Cloudflare is the merchant of record.
 
+  - term: AI crawler
+    general_definition: |-
+      A bot which scrapes content from websites in support of an AI model, including by scraping content for indexing, retrieval augmented generation, or training.
