diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b95e846a5cb35c8..5dae0e9aef0dc37 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -95,7 +95,7 @@ /src/content/docs/d1/ @elithrar @rozenmd @vy-ton @joshthoward @oxyjun @harshil1712 @cloudflare/pcx-technical-writing /src/content/release-notes/d1.yaml @elithrar @rozenmd @vy-ton @joshthoward @oxyjun @cloudflare/pcx-technical-writing /src/content/partials/d1/ @elithrar @rozenmd @vy-ton @joshthoward @oxyjun @harshil1712 @cloudflare/pcx-technical-writing -/src/content/docs/ai-audit/ @oxyjun @kodster28 @cloudflare/pcx-technical-writing +/src/content/docs/ai-crawl-control/ @oxyjun @kodster28 @cloudflare/pcx-technical-writing /src/content/docs/durable-objects/ @elithrar @vy-ton @joshthoward @oxyjun @harshil1712 @mikenomitch @cloudflare/pcx-technical-writing @cloudflare/workers-runtime-1 /src/content/release-notes/durable-objects.yaml @elithrar @rozenmd @vy-ton @joshthoward @oxyjun @cloudflare/pcx-technical-writing /src/content/docs/email-routing/ @cloudflare/pcx-technical-writing diff --git a/public/__redirects b/public/__redirects index 713410196300cf5..8304dc3e76227d6 100644 --- a/public/__redirects +++ b/public/__redirects @@ -149,9 +149,9 @@ /ai/ /use-cases/ai/ 301 # ai-audit -/ai-audit/features/detect-ai-crawlers/ /ai-audit/features/analyze-ai-crawlers/ 301 -/ai-audit/features/enforce-robots-txt/ /ai-audit/features/analyze-ai-crawlers/ 301 -/ai-audit/features/analyze-ai-crawlers/ /ai-audit/features/analyze-ai-traffic/ 301 +/ai-audit/features/detect-ai-crawlers/ /ai-crawl-control/features/analyze-ai-crawlers/ 301 +/ai-audit/features/enforce-robots-txt/ /ai-crawl-control/features/analyze-ai-crawlers/ 301 +/ai-audit/features/analyze-ai-crawlers/ /ai-crawl-control/features/analyze-ai-traffic/ 301 # AI Gateway /ai-gateway/get-started/configuring-settings/ /ai-gateway/get-started/ 301 @@ -2270,6 +2270,9 @@ /logs/get-started/enable-destinations/* /logs/logpush/logpush-job/enable-destinations/:splat 301 /logs/reference/log-fields/* /logs/logpush/logpush-job/datasets/:splat 301 +# AI Crawl Control +/ai-audit/* /ai-crawl-control/:splat 301 + # Cloudflare One / Zero Trust /cloudflare-one/connections/connect-networks/install-and-setup/tunnel-guide/local/as-a-service/* /cloudflare-one/connections/connect-networks/configure-tunnels/local-management/as-a-service/:splat 301 /cloudflare-one/connections/connect-apps/install-and-setup/deployment-guides/* /cloudflare-one/connections/connect-networks/deployment-guides/:splat 301 diff --git a/src/assets/images/ai-audit/ai-audit-pay-per-crawl-diagram.png b/src/assets/images/ai-crawl-control/ai-crawl-control-pay-per-crawl-diagram.png similarity index 100% rename from src/assets/images/ai-audit/ai-audit-pay-per-crawl-diagram.png rename to src/assets/images/ai-crawl-control/ai-crawl-control-pay-per-crawl-diagram.png diff --git a/src/assets/images/changelog/ai-crawl-control/ai-crawl-control-block-response.png b/src/assets/images/changelog/ai-crawl-control/ai-crawl-control-block-response.png new file mode 100644 index 000000000000000..1ee7c3908e0dac1 Binary files /dev/null and b/src/assets/images/changelog/ai-crawl-control/ai-crawl-control-block-response.png differ diff --git a/src/assets/images/changelog/ai-audit/ai-audit-overview.png b/src/assets/images/changelog/ai-crawl-control/ai-crawl-control-overview.png similarity index 100% rename from src/assets/images/changelog/ai-audit/ai-audit-overview.png rename to src/assets/images/changelog/ai-crawl-control/ai-crawl-control-overview.png diff --git a/src/assets/images/changelog/ai-crawl-control/ai-crawl-control-table.png b/src/assets/images/changelog/ai-crawl-control/ai-crawl-control-table.png new file mode 100644 index 000000000000000..880c86f694024a5 Binary files /dev/null and b/src/assets/images/changelog/ai-crawl-control/ai-crawl-control-table.png differ diff --git a/src/assets/images/changelog/ai-audit/analyze-metrics.png b/src/assets/images/changelog/ai-crawl-control/analyze-metrics.png similarity index 100% rename from src/assets/images/changelog/ai-audit/analyze-metrics.png rename to src/assets/images/changelog/ai-crawl-control/analyze-metrics.png diff --git a/src/assets/images/changelog/ai-audit/manage-ai-crawlers.png b/src/assets/images/changelog/ai-crawl-control/manage-ai-crawlers.png similarity index 100% rename from src/assets/images/changelog/ai-audit/manage-ai-crawlers.png rename to src/assets/images/changelog/ai-crawl-control/manage-ai-crawlers.png diff --git a/src/assets/images/changelog/ai-audit/pay-per-crawl.png b/src/assets/images/changelog/ai-crawl-control/pay-per-crawl.png similarity index 100% rename from src/assets/images/changelog/ai-audit/pay-per-crawl.png rename to src/assets/images/changelog/ai-crawl-control/pay-per-crawl.png diff --git a/src/content/changelog/ai-audit/2025-07-01-pay-per-crawl.mdx b/src/content/changelog/ai-audit/2025-07-01-pay-per-crawl.mdx deleted file mode 100644 index 1e0c3c77eba921f..000000000000000 --- a/src/content/changelog/ai-audit/2025-07-01-pay-per-crawl.mdx +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: Introducing Pay Per Crawl (private beta) -description: A new feature which allows site owners to charge crawlers for access to their content -date: 2025-07-01 ---- - -We are introducing a new feature of [AI Audit](/ai-audit/) — Pay Per Crawl. [Pay Per Crawl](/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl/) enables site owners to require payment from AI crawlers every time the crawlers access their content, thereby fostering a fairer Internet by enabling site owners to control and monetize how their content gets used by AI. - -![Pay per crawl](~/assets/images/changelog/ai-audit/pay-per-crawl.png) - -**For Site Owners:** - -- Set pricing and select which crawlers to charge for content access -- Manage payments via Stripe -- Monitor analytics on successful content deliveries - -**For AI Crawler Owners:** - -- Use HTTP headers to request and accept pricing -- Receive clear confirmations on charges for accessed content - -Learn more in the [Pay Per Crawl documentation](/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl/). diff --git a/src/content/changelog/ai-audit/2025-07-01-refresh.mdx b/src/content/changelog/ai-audit/2025-07-01-refresh.mdx deleted file mode 100644 index e20f66f4121ddaa..000000000000000 --- a/src/content/changelog/ai-audit/2025-07-01-refresh.mdx +++ /dev/null @@ -1,19 +0,0 @@ ---- -title: AI Audit refresh -description: More intuitive AI crawler management -date: 2025-07-01 ---- - -We redesigned the AI Audit dashboard to provide more intuitive and granular control over AI crawlers. - -- From the new **AI Crawlers** tab: block specific AI crawlers. -- From the new **Metrics** tab: view AI Audit metrics. - -![Block AI crawlers](~/assets/images/changelog/ai-audit/manage-ai-crawlers.png) - -![Analyze AI crawler activity](~/assets/images/changelog/ai-audit/analyze-metrics.png) - -To get started, explore: - -- [Manage AI crawlers](/ai-audit/features/manage-ai-crawlers/). -- [Analyze AI traffic](/ai-audit/features/analyze-ai-traffic/). diff --git a/src/content/changelog/ai-audit/2024-09-23-ai-audit-launch.mdx b/src/content/changelog/ai-crawl-control/2024-09-23-ai-audit-launch.mdx similarity index 58% rename from src/content/changelog/ai-audit/2024-09-23-ai-audit-launch.mdx rename to src/content/changelog/ai-crawl-control/2024-09-23-ai-audit-launch.mdx index 262320717928a5d..c5dcb70bcb8f4c8 100644 --- a/src/content/changelog/ai-audit/2024-09-23-ai-audit-launch.mdx +++ b/src/content/changelog/ai-crawl-control/2024-09-23-ai-audit-launch.mdx @@ -1,10 +1,10 @@ --- -title: AI Audit +title: AI Crawl Control description: AI Audit is available to all customers date: 2024-09-23 --- -Every site on Cloudflare now has access to [**AI Audit**](/ai-audit/), which summarizes the crawling behavior of popular and known AI services. +Every site on Cloudflare now has access to [**AI Audit**](/ai-crawl-control/), which summarizes the crawling behavior of popular and known AI services. You can use this data to: @@ -12,6 +12,6 @@ You can use this data to: - Block specific AI bots accessing your site. - Use Cloudflare to enforce your `robots.txt` policy via an automatic WAF rule. -![View AI bot activity with AI Audit](~/assets/images/changelog/ai-audit/ai-audit-overview.png) +![View AI bot activity with AI Audit](~/assets/images/changelog/ai-crawl-control/ai-crawl-control-overview.png) -To get started, explore [AI audit](/ai-audit/). +To get started, explore [AI audit](/ai-crawl-control/). diff --git a/src/content/changelog/ai-crawl-control/2025-07-01-pay-per-crawl.mdx b/src/content/changelog/ai-crawl-control/2025-07-01-pay-per-crawl.mdx new file mode 100644 index 000000000000000..eabb0c56026378e --- /dev/null +++ b/src/content/changelog/ai-crawl-control/2025-07-01-pay-per-crawl.mdx @@ -0,0 +1,22 @@ +--- +title: Introducing Pay Per Crawl (private beta) +description: A new feature which allows site owners to charge crawlers for access to their content +date: 2025-07-01 +--- + +We are introducing a new feature of [AI Crawl Control](/ai-crawl-control/) — Pay Per Crawl. [Pay Per Crawl](/ai-crawl-control/features/pay-per-crawl/what-is-pay-per-crawl/) enables site owners to require payment from AI crawlers every time the crawlers access their content, thereby fostering a fairer Internet by enabling site owners to control and monetize how their content gets used by AI. + +![Pay per crawl](~/assets/images/changelog/ai-crawl-control/pay-per-crawl.png) + +**For Site Owners:** + +- Set pricing and select which crawlers to charge for content access +- Manage payments via Stripe +- Monitor analytics on successful content deliveries + +**For AI Crawler Owners:** + +- Use HTTP headers to request and accept pricing +- Receive clear confirmations on charges for accessed content + +Learn more in the [Pay Per Crawl documentation](/ai-crawl-control/features/pay-per-crawl/what-is-pay-per-crawl/). diff --git a/src/content/changelog/ai-crawl-control/2025-07-01-refresh.mdx b/src/content/changelog/ai-crawl-control/2025-07-01-refresh.mdx new file mode 100644 index 000000000000000..e5e05d7ac46a30e --- /dev/null +++ b/src/content/changelog/ai-crawl-control/2025-07-01-refresh.mdx @@ -0,0 +1,19 @@ +--- +title: AI Crawl Control refresh +description: More intuitive AI crawler management +date: 2025-07-01 +--- + +We redesigned the AI Crawl Control dashboard to provide more intuitive and granular control over AI crawlers. + +- From the new **AI Crawlers** tab: block specific AI crawlers. +- From the new **Metrics** tab: view AI Crawl Control metrics. + +![Block AI crawlers](~/assets/images/changelog/ai-crawl-control/manage-ai-crawlers.png) + +![Analyze AI crawler activity](~/assets/images/changelog/ai-crawl-control/analyze-metrics.png) + +To get started, explore: + +- [Manage AI crawlers](/ai-crawl-control/features/manage-ai-crawlers/). +- [Analyze AI traffic](/ai-crawl-control/features/analyze-ai-traffic/). diff --git a/src/content/changelog/ai-crawl-control/2025-08-27-ai-crawl-control-launch.mdx b/src/content/changelog/ai-crawl-control/2025-08-27-ai-crawl-control-launch.mdx new file mode 100644 index 000000000000000..1d9e2f134aed90c --- /dev/null +++ b/src/content/changelog/ai-crawl-control/2025-08-27-ai-crawl-control-launch.mdx @@ -0,0 +1,39 @@ +--- +title: Enhanced crawler insights and custom 402 responses +description: Get detailed crawler analytics with trend charts and configure custom payment-required responses for blocked AI crawlers. +date: 2025-08-27 +--- + +We improved AI crawler management with detailed analytics and introduced custom HTTP 402 responses for blocked crawlers. AI Audit has been renamed to AI Crawl Control and is now generally available. + +**Enhanced Crawlers tab:** + +- View total allowed and blocked requests for each AI crawler +- Trend charts show crawler activity over your selected time range per crawler + +![Updated AI Crawl Control table showing request counts and trend charts](~/assets/images/changelog/ai-crawl-control/ai-crawl-control-table.png) + +**Custom block responses (paid plans):** +You can now return HTTP 402 "Payment Required" responses when blocking AI crawlers, enabling direct communication with crawler operators about licensing terms. + +For users on paid plans, when blocking AI crawlers you can configure: + +- **Response code:** Choose between 403 Forbidden or 402 Payment Required +- **Response body:** Add a custom message with your licensing contact information + +![AI Crawl Control block response configuration interface](~/assets/images/changelog/ai-crawl-control/ai-crawl-control-block-response.png) + +Example 402 response: + +```http +HTTP 402 Payment Required +Date: Mon, 24 Aug 2025 12:56:49 GMT +Content-type: application/json +Server: cloudflare +Cf-Ray: 967e8da599d0c3fa-EWR +Cf-Team: 2902f6db750000c3fa1e2ef400000001 + +{ + "message": "Please contact the site owner for access." +} +``` diff --git a/src/content/docs/ai-audit/changelog.mdx b/src/content/docs/ai-audit/changelog.mdx deleted file mode 100644 index 5cda654b0cb2fe5..000000000000000 --- a/src/content/docs/ai-audit/changelog.mdx +++ /dev/null @@ -1,12 +0,0 @@ ---- -pcx_content_type: changelog -title: Changelog -sidebar: - order: 100 ---- - -import { ProductChangelog } from "~/components"; - -{/* */} - - diff --git a/src/content/docs/ai-audit/features/analyze-ai-traffic.mdx b/src/content/docs/ai-audit/features/analyze-ai-traffic.mdx deleted file mode 100644 index 0568754482d6edd..000000000000000 --- a/src/content/docs/ai-audit/features/analyze-ai-traffic.mdx +++ /dev/null @@ -1,75 +0,0 @@ ---- -title: Analyze AI traffic -pcx_content_type: concept -tags: - - AI -sidebar: - order: 2 ---- - -import { Steps, Tabs, TabItem } from "~/components"; - -AI Audit metrics provide you with insight on how AI crawlers are interacting with your website ([Cloudflare zone](/fundamentals/concepts/accounts-and-zones/#zones)). - -To view AI Audit metrics: - -1. Log in to the [Cloudflare dashboard](https://dash.cloudflare.com/), and select your account and domain. -2. Go to **AI Audit**. - -You can find meaningful information across both **AI Crawlers** and **Metrics** tabs. - -## View AI crawler activity - -The **AI Crawlers** tab provides you with the following information: - -- Complete list of detected AI crawlers and their operators -- Number of requests made by each AI crawler -- Number of `robots.txt` violations for each crawler - -## View AI Audit metrics - -The **Metrics** tab provides you with the following metrics to help you understand how AI crawlers are interacting with your website. - -| Metric | Description | -| ------------------------------------- | ----------------------------------------------------------------------------- | -| Total requests | The total number of requests to crawl your website, from all AI crawlers | -| Blocked requests | The number of requests you have blocked (by blocking AI crawlers) | -| Allowed requests | The number of requests you have allowed (by allowing AI crawlers) | -| Hosts | The owner of the AI crawler | -| Overall popular paths | The most popular pages crawled by AI crawlers, from all AI crawlers | -| Most active AI crawlers by operators | The AI crawler owners with the highest number of requests to access your site | -| Request by AI crawlers | A graph which displays the number of crawl requests from each AI crawler | -| Most popular paths by AI crawlers | The most popular pages crawled by AI crawlers, for each AI crawler | -| Referrals | A graph which displays the number of referrals from each AI operator | -| Referers | The list of referers who directed traffic to your site | - -:::note[Requests in AI Audit metrics] -The number of requests in AI Audit metrics are specifically requests which were met with HTTP code 200 (the request was successfully served, with actual content). - -AI Audit metrics filter all other HTTP codes. -::: - -## Filter date range - -You can use the date filter to choose the period of time you wish to analyze. - - - - -Filter options: - -- Past 24 hours - - - -Filter options: - -- Past 24 hours -- Past 7 days -- Past 14 days -- Past month - - - - -The values of the AI Audit metrics will update according to your filter. diff --git a/src/content/docs/ai-audit/features/manage-ai-crawlers.mdx b/src/content/docs/ai-audit/features/manage-ai-crawlers.mdx deleted file mode 100644 index 5094141f50bab6f..000000000000000 --- a/src/content/docs/ai-audit/features/manage-ai-crawlers.mdx +++ /dev/null @@ -1,127 +0,0 @@ ---- -title: Manage AI crawlers -pcx_content_type: concept -sidebar: - order: 5 ---- - -import { Steps, GlossaryTooltip, Tabs, TabItem, Example } from "~/components"; - -AI Audit enables you to take specific action for each AI crawler. - -To manage AI crawlers: - -1. Log in to the [Cloudflare dashboard](https://dash.cloudflare.com/), and select your account and domain. -2. Go to **AI Audit**. -3. Go to the **AI Crawlers** tab. - -## Review AI crawler activity - -The **AI Crawlers** tab displays a table of AI crawlers that are requesting access to your content, and how they interact with your pages. The table provides the following information. - - - -| Column | Details | -| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| AI Crawlers | The name of the AI crawler. | -| Operator | The name of the entity who owns the AI crawler. Note that an operator may have multiple AI crawlers. | -| Category | The category of the AI crawler. Refer to [Verified bot categories](/bots/concepts/bot/verified-bots/categories/). | -| Block | Toggle for blocking specific AI crawlers. | - - -| Column | Details | -| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| AI Crawlers | The name of the AI crawler. | -| Operator | The name of the entity who owns the AI crawler. Note that an operator may have multiple AI crawlers. | -| Category | The category of the AI crawler. Refer to [Verified bot categories](/bots/concepts/bot/verified-bots/categories/). | -| Requests | The number of times the AI crawler has requested to crawl your content. | -| Robots.txt violations | The number of times the AI crawler has violated your `robots.txt` file. | -| Action | The action you wish to take for the AI crawler. Refer to [Take action for each AI crawler](/ai-audit/features/manage-ai-crawlers/#take-action-for-each-ai-crawler). | - - - - -:::note[Quality of AI crawler detection] -On the free plan, AI Audit identifies AI crawlers based on their [user agent strings](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/User-Agent). This enables AI Audit to detect easy-to-detect (well-known) AI crawlers. - -Upgrade your plan to enable a more thorough detection using Cloudflare's [Bot Management detection ID](/bots/reference/bot-management-variables/#ruleset-engine-fields) field. -::: - -### Filter AI crawler data - -You can use filters to narrow the scope of your result: - -- **Name:** Search the name of the AI crawler. -- **Operator:** Filter by the AI crawler operator. -- **Category:** Filter by the category of the AI crawler (for example, AI crawler, AI assistant, or archiver). - -The values of the table will update according to your filter. - -## Take action for each AI crawler - - - - -:::note[Pay per crawl closed beta] -Pay per crawl is currently in closed beta. - -To find out how to join the beta program, reach out to us at [Pay per crawl signup](http://www.cloudflare.com/paypercrawl-signup/), or contact your account executive if you are an existing Enterprise customer. - -To learn more about pay per crawl, refer to Cloudflare blog: [Introducing pay per crawl: enabling content owners to charge AI crawlers for access](https://blog.cloudflare.com/introducing-pay-per-crawl/). -::: - -For each AI crawler, you can take one of three actions: allow, charge, or block. - - - -- **Summary:** You can allow an AI crawler to scrape your content. -- **When to use:** Allow AI crawlers that offer services which provide value through citations, referrals, or existing agreements. -- **Implementation:** From the **Actions** column, select **Allow**. -Note that you can still choose to [Enforce `robots.txt`](/ai-audit/features/manage-ai-crawlers/#take-action-for-each-ai-crawler). - -For more details on how this rule interacts with other Cloudflare settings, refer to [How it works](/bots/concepts/bot/#how-it-works). - - - - - -- **Summary:** You can block an AI crawler to completely stop the AI crawler from scraping your webpage. -- **When to use:** Block AI crawlers when their behavior do not align with your content strategy, or violate your policies. -- **Implementation:** From the **Actions** column, select **Block**. - - - - -- **Summary:** You can charge the owner of the AI crawler for each crawl request. -- **When to use:** Charge AI crawlers when your content has training value, and you want to explore monetization options -- **Implementation:** From the **Actions** column, select **Charge**. - -For more information, refer to [What is Pay Per Crawl?](/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl/). - - - - - - -For each AI crawler, you can choose to allow or block access. - - - -- **Summary:** You can allow an AI crawler to scrape your content. -- **When to use:** Allow AI crawlers that offer services which provide value through citations, referrals, or existing agreements. -- **Implementation:** From the **Actions** column, select **Allow**. -Note that you can still choose to [Enforce `robots.txt`](/ai-audit/features/manage-ai-crawlers/#take-action-for-each-ai-crawler). - - - -- **Summary:** You can block an AI crawler to completely stop the AI crawler from scraping your webpage. -- **When to use:** Block AI crawlers when their behavior do not align with your content strategy, or violate your policies. -- **Implementation:** From the **Actions** column, select **Block**. - - - - - -## Related resources - -- Use [pay per crawl](/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl/) to charge AI crawlers every time they access your content. diff --git a/src/content/docs/ai-crawl-control/changelog.mdx b/src/content/docs/ai-crawl-control/changelog.mdx new file mode 100644 index 000000000000000..4211dbfb0d86d52 --- /dev/null +++ b/src/content/docs/ai-crawl-control/changelog.mdx @@ -0,0 +1,12 @@ +--- +pcx_content_type: changelog +title: Changelog +sidebar: + order: 100 +--- + +import { ProductChangelog } from "~/components"; + +{/* */} + + diff --git a/src/content/docs/ai-audit/configuration/ai-audit-with-bots.mdx b/src/content/docs/ai-crawl-control/configuration/ai-crawl-control-with-bots.mdx similarity index 53% rename from src/content/docs/ai-audit/configuration/ai-audit-with-bots.mdx rename to src/content/docs/ai-crawl-control/configuration/ai-crawl-control-with-bots.mdx index cea31e83922993d..d02e3c10681edfe 100644 --- a/src/content/docs/ai-audit/configuration/ai-audit-with-bots.mdx +++ b/src/content/docs/ai-crawl-control/configuration/ai-crawl-control-with-bots.mdx @@ -1,5 +1,5 @@ --- -title: AI Audit with Cloudflare Bots +title: AI Crawl Control with Cloudflare Bots pcx_content_type: concept sidebar: order: 5 @@ -7,18 +7,18 @@ sidebar: import { GlossaryTooltip, Example } from "~/components"; -AI Audit works alongside other Cloudflare products, such as Cloudflare [bot solutions](/bots/). Bot solutions identifies traffic matching patterns of known bots, and can challenge or block the bots as you wish. +AI Crawl Control works alongside other Cloudflare products, such as Cloudflare [bot solutions](/bots/). Bot solutions identifies traffic matching patterns of known bots, and can challenge or block the bots as you wish. ## Order of precedence -- AI Audit's AI crawler blocking uses [WAF custom rules](/waf/custom-rules/), which take place before Cloudflare bot solutions. -- AI Audit's pay per crawl takes place after Cloudflare bot solutions. +- AI Crawl Control's AI crawler blocking uses [WAF custom rules](/waf/custom-rules/), which take place before Cloudflare bot solutions. +- AI Crawl Control's pay per crawl takes place after Cloudflare bot solutions. ```mermaid graph LR -A[Traffic] --> B[WAF custom rules
AI Audit: Crawler blocks] +A[Traffic] --> B[WAF custom rules
AI Crawl Control: Crawler blocks] B --> C[Cloudflare
Bot Solutions] -C --> D[AI Audit:
Pay Per Crawl] +C --> D[AI Crawl Control:
Pay Per Crawl] classDef highlight fill:#F6821F,color:white ``` @@ -32,7 +32,7 @@ Consider the following examples. You may have both of the following enabled: -- A selection of AI crawlers to be charged through AI Audit's pay per crawl +- A selection of AI crawlers to be charged through AI Crawl Control's pay per crawl - Bot configuration option to [Block AI Bots](/bots/get-started/bot-fight-mode/#block-ai-bots). Since pay per crawl happens after bot solutions, you need to first turn off **Block AI Bots** to ensure pay per crawl works as intended. \ No newline at end of file diff --git a/src/content/docs/ai-audit/configuration/ai-audit-with-waf.mdx b/src/content/docs/ai-crawl-control/configuration/ai-crawl-control-with-waf.mdx similarity index 51% rename from src/content/docs/ai-audit/configuration/ai-audit-with-waf.mdx rename to src/content/docs/ai-crawl-control/configuration/ai-crawl-control-with-waf.mdx index 620bafa18a8b535..c35a05f4b78f030 100644 --- a/src/content/docs/ai-audit/configuration/ai-audit-with-waf.mdx +++ b/src/content/docs/ai-crawl-control/configuration/ai-crawl-control-with-waf.mdx @@ -1,5 +1,5 @@ --- -title: AI Audit with Cloudflare WAF +title: AI Crawl Control with Cloudflare WAF pcx_content_type: concept sidebar: order: 2 @@ -8,32 +8,32 @@ sidebar: import { GlossaryTooltip, Example, Steps, Tabs, TabItem } from "~/components"; -AI Audit works alongside other Cloudflare products, such as Cloudflare [Web Application Firewall (WAF)](/waf/). WAF checks incoming web and API requests, and filters undesired traffic based on rules. [WAF custom rules](/waf/custom-rules/) allow you to perform certain actions such as enforcing `robots.txt`. +AI Crawl Control works alongside other Cloudflare products, such as Cloudflare [Web Application Firewall (WAF)](/waf/). WAF checks incoming web and API requests, and filters undesired traffic based on rules. [WAF custom rules](/waf/custom-rules/) allow you to perform certain actions such as enforcing `robots.txt`. ## Order of precedence -- AI Audit uses WAF custom rules to block the selection of AI crawlers the site owner has decided to block. -- AI Audit's pay per crawl feature takes place after WAF. +- AI Crawl Control uses WAF custom rules to block the selection of AI crawlers the site owner has decided to block. +- AI Crawl Control's pay per crawl feature takes place after WAF. ```mermaid graph LR -A[Traffic] --> B[WAF custom rules
AI Audit: Crawler blocks] +A[Traffic] --> B[WAF custom rules
AI Crawl Control: Crawler blocks] B --> C[Cloudflare
Bot Solutions] -C --> D[AI Audit:
Pay Per Crawl] +C --> D[AI Crawl Control:
Pay Per Crawl] classDef highlight fill:#F6821F,color:white ``` -For this reason, if you plan on using AI Audit to manage AI crawlers, you may wish to modify your existing WAF custom rules such that it does not affect AI crawlers. This will allow you to manage AI crawlers only from AI Audit, thereby streamlining your workflow. +For this reason, if you plan on using AI Crawl Control to manage AI crawlers, you may wish to modify your existing WAF custom rules such that it does not affect AI crawlers. This will allow you to manage AI crawlers only from AI Crawl Control, thereby streamlining your workflow. -:::note[How AI Audit uses WAF custom rules] -When you block AI crawlers via AI Audit (either all or some), you are using **one** WAF custom rule to block those AI crawlers. +:::note[How AI Crawl Control uses WAF custom rules] +When you block AI crawlers via AI Crawl Control (either all or some), you are using **one** WAF custom rule to block those AI crawlers. -If you choose to allow all AI crawlers, AI Audit does not utilize any WAF custom rules. +If you choose to allow all AI crawlers, AI Crawl Control does not utilize any WAF custom rules. Depending on the type of account you have, you may have a limited number of WAF custom rules. ::: -## Examples of using WAF vs AI Audit +## Examples of using WAF vs AI Crawl Control Consider the following examples. @@ -42,24 +42,24 @@ Consider the following examples. You may have both of the following features enabled: - [WAF custom rule to block traffic from specific countries](/waf/custom-rules/use-cases/block-traffic-from-specific-countries/) -- AI Audit's [pay per crawl](/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl/) to charge AI crawlers when they request access to your content +- AI Crawl Control's [pay per crawl](/ai-crawl-control/features/pay-per-crawl/what-is-pay-per-crawl/) to charge AI crawlers when they request access to your content -Since WAF custom rules are enforced before pay per crawl, traffic (including AI crawlers) from your blocked countries will continue to be blocked, even if they provide the [required headers](/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/crawl-pages/#1-include-required-headers) for pay per crawl. +Since WAF custom rules are enforced before pay per crawl, traffic (including AI crawlers) from your blocked countries will continue to be blocked, even if they provide the [required headers](/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/crawl-pages/#1-include-required-headers) for pay per crawl. ### Allowed search engine bots via WAF custom rule vs pay per crawl You may have both of the following features enabled: - [WAF custom rule to allow search engine bots](/waf/custom-rules/use-cases/allow-traffic-from-verified-bots/) -- AI Audit's [pay per crawl](/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl/) to charge all AI crawlers when they request access to your content (including search engine bots). +- AI Crawl Control's [pay per crawl](/ai-crawl-control/features/pay-per-crawl/what-is-pay-per-crawl/) to charge all AI crawlers when they request access to your content (including search engine bots). Since custom rules are enforced before pay per crawl: - Only search engine bots will be able to access your site (enforced by custom rule). -- The search engine bots will then be charged for access to your content (enforced by AI Audit's pay per crawl). +- The search engine bots will then be charged for access to your content (enforced by AI Crawl Control's pay per crawl). :::note -This example only serves to highlight the order of precedence between WAF and AI Audit. +This example only serves to highlight the order of precedence between WAF and AI Crawl Control. Practically, it may be beneficial to allow well-behaved search engine bots to access your content to ensure your content is indexed. ::: @@ -69,7 +69,7 @@ Practically, it may be beneficial to allow well-behaved search engine bots to ac You may have both of the following features enabled: - A WAF custom rule which blocks all bots. -- AI Audit selection which allows certain AI crawlers. +- AI Crawl Control selection which allows certain AI crawlers. In this scenario, you have two custom rules, each directing a different logic for handling AI crawlers. To resolve this issue: @@ -78,8 +78,8 @@ In this scenario, you have two custom rules, each directing a different logic fo 1. Log in to the [Cloudflare dashboard](https://dash.cloudflare.com/), and select your account and domain. 2. Go to **Security** > **WAF** > **Custom rules** tab. -3. Identify your WAF custom rule and the AI Audit rule. -4. Drag the rule you wish to prioritize to the top, or modify your WAF custom rule to ensure it does not conflict with your AI Audit configurations. +3. Identify your WAF custom rule and the AI Crawl Control rule. +4. Drag the rule you wish to prioritize to the top, or modify your WAF custom rule to ensure it does not conflict with your AI Crawl Control configurations. @@ -87,8 +87,8 @@ In this scenario, you have two custom rules, each directing a different logic fo 1. Log in to the [Cloudflare dashboard](https://dash.cloudflare.com/), and select your account and domain. 2. Go to **Security** > **Security rules** and filter by **Custom rules**. -3. Identify your custom rule and the AI Audit rule. -4. Drag the rule you wish to prioritize to the top, or modify your custom rule to ensure it does not conflict with your AI Audit configurations. +3. Identify your custom rule and the AI Crawl Control rule. +4. Drag the rule you wish to prioritize to the top, or modify your custom rule to ensure it does not conflict with your AI Crawl Control configurations. diff --git a/src/content/docs/ai-audit/configuration/index.mdx b/src/content/docs/ai-crawl-control/configuration/index.mdx similarity index 100% rename from src/content/docs/ai-audit/configuration/index.mdx rename to src/content/docs/ai-crawl-control/configuration/index.mdx diff --git a/src/content/docs/ai-crawl-control/features/analyze-ai-traffic.mdx b/src/content/docs/ai-crawl-control/features/analyze-ai-traffic.mdx new file mode 100644 index 000000000000000..e6a87aa85aab2a4 --- /dev/null +++ b/src/content/docs/ai-crawl-control/features/analyze-ai-traffic.mdx @@ -0,0 +1,75 @@ +--- +title: Analyze AI traffic +pcx_content_type: concept +tags: + - AI +sidebar: + order: 2 +--- + +import { Steps, Tabs, TabItem } from "~/components"; + +AI Crawl Control metrics provide you with insight on how AI crawlers are interacting with your website ([Cloudflare zone](/fundamentals/concepts/accounts-and-zones/#zones)). + +To view AI Crawl Control metrics: + +1. Log in to the [Cloudflare dashboard](https://dash.cloudflare.com/), and select your account and domain. +2. Go to **AI Crawl Control**. + +You can find meaningful information across both **AI Crawlers** and **Metrics** tabs. + +## View AI crawler activity + +The **AI Crawlers** tab provides you with the following information: + +- Complete list of detected AI crawlers and their operators +- Number of requests made by each AI crawler +- Number of `robots.txt` violations for each crawler + +## View AI Crawl Control metrics + +The **Metrics** tab provides you with the following metrics to help you understand how AI crawlers are interacting with your website. + +| Metric | Description | +| ------------------------------------ | ----------------------------------------------------------------------------- | +| Total requests | The total number of requests to crawl your website, from all AI crawlers | +| Blocked requests | The number of crawler requests you have blocked, from any rule | +| Allowed requests | The number of crawler requests you have allowed | +| Hosts | The owner of the AI crawler | +| Overall popular paths | The most popular pages crawled by AI crawlers, from all AI crawlers | +| Most active AI crawlers by operators | The AI crawler owners with the highest number of requests to access your site | +| Request by AI crawlers | A graph which displays the number of crawl requests from each AI crawler | +| Most popular paths by AI crawlers | The most popular pages crawled by AI crawlers, for each AI crawler | +| Referrals | A graph which displays the number of referrals from each AI operator | +| Referers | The list of referers who directed traffic to your site | + +:::note[Requests in AI Crawl Control metrics] +The number of requests in AI Crawl Control metrics are specifically requests which were met with HTTP code 200 (the request was successfully served, with actual content). + +AI Crawl Control metrics filter all other HTTP codes. +::: + +## Filter date range + +You can use the date filter to choose the period of time you wish to analyze. + + + + +Filter options: + +- Past 24 hours + + + +Filter options: + +- Past 24 hours +- Past 7 days +- Past 14 days +- Past month + + + + +The values of the AI Crawl Control metrics will update according to your filter. diff --git a/src/content/docs/ai-audit/features/index.mdx b/src/content/docs/ai-crawl-control/features/index.mdx similarity index 100% rename from src/content/docs/ai-audit/features/index.mdx rename to src/content/docs/ai-crawl-control/features/index.mdx diff --git a/src/content/docs/ai-crawl-control/features/manage-ai-crawlers.mdx b/src/content/docs/ai-crawl-control/features/manage-ai-crawlers.mdx new file mode 100644 index 000000000000000..aa5d6f7bdc54608 --- /dev/null +++ b/src/content/docs/ai-crawl-control/features/manage-ai-crawlers.mdx @@ -0,0 +1,171 @@ +--- +title: Manage AI crawlers +pcx_content_type: concept +sidebar: + order: 5 +--- + +import { Steps, GlossaryTooltip, Tabs, TabItem, Example, Plan } from "~/components"; + +AI Crawl Control enables you to take specific action for each AI crawler. + +To manage AI crawlers: + +1. Log in to the [Cloudflare dashboard](https://dash.cloudflare.com/), and select your account and domain. +2. Go to **AI Crawl Control**. +3. Go to the **AI Crawlers** tab. + +## Review AI crawler activity + +The **Crawlers** tab displays a table of AI crawlers that are requesting access to your content, and how they interact with your pages. The table provides the following information. + +| Column | Details | +| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Crawler | The name of the AI crawler and the operator that owns it. | +| Category | The category of the AI crawler. Refer to [Verified bot categories](/bots/concepts/bot/verified-bots/categories/). | +| Requests | Total allowed and blocked requests with trend chart. Blocked requests may come from any configured rule, not just the actions shown here. | +| Robots.txt violations | The number of times the AI crawler has violated your `robots.txt` file. | +| Action | The action you wish to take for the AI crawler. Refer to [Take action for each AI crawler](/ai-crawl-control/features/manage-ai-crawlers/#take-action-for-each-ai-crawler). | + +:::note[Quality of AI crawler detection] +On the free plan, AI Crawl Control identifies AI crawlers based on their [user agent strings](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/User-Agent). This enables AI Crawl Control to detect easy-to-detect (well-known) AI crawlers. + +Upgrade your plan to enable a more thorough detection using Cloudflare's [Bot Management detection ID](/bots/reference/bot-management-variables/#ruleset-engine-fields) field. +::: + +### Filter AI crawler data + +You can use filters to narrow the scope of your result: + +- **Name:** Search the name of the AI crawler. +- **Operator:** Filter by the AI crawler operator. +- **Category:** Filter by the category of the AI crawler (for example, AI crawler, AI assistant, or archiver). + +The values of the table will update according to your filter. + +## Take action for each AI crawler + + + + +:::note[Pay per crawl closed beta] +Pay per crawl is currently in closed beta. + +To find out how to join the beta program, reach out to us at [Pay per crawl signup](http://www.cloudflare.com/paypercrawl-signup/), or contact your account executive if you are an existing Enterprise customer. + +To learn more about pay per crawl, refer to Cloudflare blog: [Introducing pay per crawl: enabling content owners to charge AI crawlers for access](https://blog.cloudflare.com/introducing-pay-per-crawl/). +::: + +For each AI crawler, you can take one of three actions: allow, charge, or block. + + + +- **Summary:** You can allow an AI crawler to scrape your content. +- **When to use:** Allow AI crawlers that offer services which provide value through citations, referrals, or existing agreements. +- **Implementation:** From the **Actions** column, select **Allow**. +Note that you can still choose to [Enforce `robots.txt`](/ai-crawl-control/features/manage-ai-crawlers/#take-action-for-each-ai-crawler). + +For more details on how this rule interacts with other Cloudflare settings, refer to [How it works](/bots/concepts/bot/#how-it-works). + + + + + +- **Summary:** You can block an AI crawler to completely stop the AI crawler from scraping your webpage. +- **When to use:** Block AI crawlers when their behavior do not align with your content strategy, or violate your policies. +- **Implementation:** From the **Actions** column, select **Block**. + +Note that you can configure the response that gets returned when blocking an AI crawler. Refer to [Configure block response](/ai-crawl-control/features/manage-ai-crawlers/#configure-block-response). + + + + + +- **Summary:** You can charge the owner of the AI crawler for each crawl request. +- **When to use:** Charge AI crawlers when your content has training value, and you want to explore monetization options +- **Implementation:** From the **Actions** column, select **Charge**. + +For more information, refer to [What is Pay Per Crawl?](/ai-crawl-control/features/pay-per-crawl/what-is-pay-per-crawl/). + + + + + + +For each AI crawler, you can choose to allow or block access. + + + +- **Summary:** You can allow an AI crawler to scrape your content. +- **When to use:** Allow AI crawlers that offer services which provide value through citations, referrals, or existing agreements. +- **Implementation:** From the **Actions** column, select **Allow**. + +Note that you can still choose to [Enforce `robots.txt`](/ai-crawl-control/features/manage-ai-crawlers/#take-action-for-each-ai-crawler). + + + + +- **Summary:** You can block an AI crawler to completely stop the AI crawler from scraping your webpage. +- **When to use:** Block AI crawlers when their behavior do not align with your content strategy, or violate your policies. +- **Implementation:** From the **Actions** column, select **Block**. + +Note that you can configure the response that gets returned when blocking an AI crawler. Refer to [Configure block response](/ai-crawl-control/features/manage-ai-crawlers/#configure-block-response). + + + + + + +## Configure block response + + + +When blocking an AI crawler, you can configure the details of the response that gets returned to the AI crawler. Specifically, you can configure: + +- The response code +- The response body + +This provides you with a channel to open dialogue with the AI crawler owner, and to inform the AI crawler how to properly license their content, thereby creating a direct path from crawling attempt to commercial agreement. + +To edit these values: + + +1. Log in to the [Cloudflare dashboard](https://dash.cloudflare.com/), and select your account and domain. +2. Go to **AI Crawl Control**. +3. Go to the **Settings** tab. +4. Under **Block response**, select **Edit**. +5. Once you have edited the values, select **Save**. + + +:::note +You must have opted to block at least one AI crawler to configure a custom block response. +::: + +### Edit the response code + +You can choose which HTTP response code to return when blocking an AI crawler. + +Use the dropdown menu to select the desired response code. You can choose from: + + - `403 Forbidden`: Use this option if you wish to indicate that you do not want the AI crawler to access your content. + - `402 Payment Required`: Use this option if you wish to indicate that the AI crawler must pay to access your content. + +:::note +Behind the scenes, AI Crawl Control uses [Cloudflare WAF](/waf/) to return custom block responses. + +If you have manually configured a WAF rule to return a response code other than `403` or `402`, AI Crawl Control will not be able to enforce the response code you have selected, and the dropdown will appear blank. Ensure you have selected either `403` or `402`. + +Refer to [Configure a custom response for blocked requests](/waf/custom-rules/create-dashboard/#configure-a-custom-response-for-blocked-requests) for more information. +::: + +### Edit the response body + +You can write a custom message (HTTP response body) to return when blocking an AI crawler. + +In the **Response body** text field, enter the response you wish to display for the AI crawler in plain text. + +{/* Need guidance on what sort of information they should include */} + +## Related resources + +- Use [pay per crawl](/ai-crawl-control/features/pay-per-crawl/what-is-pay-per-crawl/) to charge AI crawlers every time they access your content. diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/index.mdx b/src/content/docs/ai-crawl-control/features/pay-per-crawl/index.mdx similarity index 91% rename from src/content/docs/ai-audit/features/pay-per-crawl/index.mdx rename to src/content/docs/ai-crawl-control/features/pay-per-crawl/index.mdx index 81d4a3b313acdfa..5287cecafcd6a4e 100644 --- a/src/content/docs/ai-audit/features/pay-per-crawl/index.mdx +++ b/src/content/docs/ai-crawl-control/features/pay-per-crawl/index.mdx @@ -4,6 +4,7 @@ pcx_content_type: navigation sidebar: group: hideIndex: true + badge: Beta order: 20 --- diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/connect-to-stripe.mdx b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/connect-to-stripe.mdx similarity index 90% rename from src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/connect-to-stripe.mdx rename to src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/connect-to-stripe.mdx index 94f30d136e20d04..56bda5c5c2ec123 100644 --- a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/connect-to-stripe.mdx +++ b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/connect-to-stripe.mdx @@ -40,6 +40,6 @@ Cloudflare is not responsible for configuring spending limits. Ensure you have c ## Billing -Charges are recorded upon successful delivery of content that is requested with valid [crawler price headers](/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/crawl-pages/#22-include-payment-headers). +Charges are recorded upon successful delivery of content that is requested with valid [crawler price headers](/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/crawl-pages/#22-include-payment-headers). Invoices are created and managed via Stripe. Crawlers are responsible for setting and enforcing their own spending limits. \ No newline at end of file diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/crawl-pages.mdx b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/crawl-pages.mdx similarity index 95% rename from src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/crawl-pages.mdx rename to src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/crawl-pages.mdx index ef4f3729857c258..b9495377cc90695 100644 --- a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/crawl-pages.mdx +++ b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/crawl-pages.mdx @@ -15,7 +15,7 @@ C --> D[Crawl pages]:::highlight classDef highlight fill:#F6821F,color:white ``` -Once your AI crawler complies with Web Bot Auth, you can begin to crawl webpages. For more information on how pay per crawl works, refer to [What is Pay Per Crawl?](/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl/) +Once your AI crawler complies with Web Bot Auth, you can begin to crawl webpages. For more information on how pay per crawl works, refer to [What is Pay Per Crawl?](/ai-crawl-control/features/pay-per-crawl/what-is-pay-per-crawl/) ## 1. Identify payment requirements @@ -85,4 +85,4 @@ Cloudflare strongly recommends tracking and saving these values to keep an accur You may wish to refer to the following resources. -- [AI Audit FAQs](/ai-audit/reference/faq/). \ No newline at end of file +- [AI Crawl Control FAQs](/ai-crawl-control/reference/faq/). \ No newline at end of file diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/index.mdx b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/index.mdx similarity index 100% rename from src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/index.mdx rename to src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/index.mdx diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/set-up-cloudflare-account.mdx b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/set-up-cloudflare-account.mdx similarity index 100% rename from src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/set-up-cloudflare-account.mdx rename to src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/set-up-cloudflare-account.mdx diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/verify-ai-crawler.mdx b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/verify-ai-crawler.mdx similarity index 100% rename from src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/verify-ai-crawler.mdx rename to src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/verify-ai-crawler.mdx diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/connect-to-stripe.mdx b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/connect-to-stripe.mdx similarity index 97% rename from src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/connect-to-stripe.mdx rename to src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/connect-to-stripe.mdx index 6bc49a773cda530..749282fe09bdffb 100644 --- a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/connect-to-stripe.mdx +++ b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/connect-to-stripe.mdx @@ -23,7 +23,7 @@ Connect your Cloudflare account to Stripe to process payments. Pay per crawl use {/* prettier-ignore */} 1. Log in to the [Cloudflare dashboard](https://dash.cloudflare.com/), and select your account and domain. -2. Go to **AI Audit**. +2. Go to **AI Crawl Control**. 3. Go to the **Settings** tab. 4. Under **Pay Per Crawl**, select **Connect**. 5. Select **Continue to Stripe**. diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/index.mdx b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/index.mdx similarity index 100% rename from src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/index.mdx rename to src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/index.mdx diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/monitor-metrics.mdx b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/monitor-metrics.mdx similarity index 77% rename from src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/monitor-metrics.mdx rename to src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/monitor-metrics.mdx index af25a1a28b7437e..9c11fcb1b07e54e 100644 --- a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/monitor-metrics.mdx +++ b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/monitor-metrics.mdx @@ -21,7 +21,7 @@ Once you have confirmed the selection of AI crawlers to charge, monitor key metr {/* prettier-ignore */} 1. Go to the **Metrics** tab. -2. [View AI Audit metrics](/ai-audit/features/analyze-ai-traffic/#view-ai-audit-metrics). Understanding these metrics can help you better manage AI crawlers accessing your content. +2. [View AI Crawl Control metrics](/ai-crawl-control/features/analyze-ai-traffic/#view-ai-audit-metrics). Understanding these metrics can help you better manage AI crawlers accessing your content. ## Other considerations @@ -34,4 +34,4 @@ You may also wish to do the following: You may wish to refer to the following resources. -- [AI Audit FAQs](/ai-audit/reference/faq/). \ No newline at end of file +- [AI Crawl Control FAQs](/ai-crawl-control/reference/faq/). \ No newline at end of file diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/select-crawlers-to-charge.mdx b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/select-crawlers-to-charge.mdx similarity index 67% rename from src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/select-crawlers-to-charge.mdx rename to src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/select-crawlers-to-charge.mdx index 5e807cd22175286..d83e248b6d25012 100644 --- a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/select-crawlers-to-charge.mdx +++ b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/select-crawlers-to-charge.mdx @@ -16,7 +16,7 @@ D --> E[Monitor
metrics] classDef highlight fill:#F6821F,color:white ``` -Once you have set a pay per crawl price, AI Audit automatically sets the default action for all applicable AI crawlers as **Charge**. However, you can control which action to take for each AI crawler. +Once you have set a pay per crawl price, AI Crawl Control automatically sets the default action for all applicable AI crawlers as **Charge**. However, you can control which action to take for each AI crawler. {/* prettier-ignore */} @@ -26,4 +26,4 @@ Once you have set a pay per crawl price, AI Audit automatically sets the default AI crawlers for which you select **Charge** will be prompted to pay your configured pay per crawl price each time they attempt to scrape content from a zone. -Refer to [Manage AI crawlers](/ai-audit/features/manage-ai-crawlers/) for more information on how to manage AI crawlers. +Refer to [Manage AI crawlers](/ai-crawl-control/features/manage-ai-crawlers/) for more information on how to manage AI crawlers. diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-a-pay-per-crawl-price.mdx b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-a-pay-per-crawl-price.mdx similarity index 82% rename from src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-a-pay-per-crawl-price.mdx rename to src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-a-pay-per-crawl-price.mdx index d4609add6fce72b..b0fc6bf86b82aca 100644 --- a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-a-pay-per-crawl-price.mdx +++ b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-a-pay-per-crawl-price.mdx @@ -20,7 +20,7 @@ Once you have connected your Stripe account, you can set your pay per crawl pric {/* prettier-ignore */} 1. From the [Cloudflare dashboard](https://dash.cloudflare.com/), select your account and domain. -2. Go to **AI Audit**. +2. Go to **AI Crawl Control**. 3. Go to the **Settings** tab. 4. From **Pay Per Crawl**, select the **Edit** icon under **Pay Per Crawl price**. 5. Enter the pay per crawl price you wish to set. Note that smallest unit of payment is 0.01 USD. @@ -28,5 +28,5 @@ Once you have connected your Stripe account, you can set your pay per crawl pric :::note -When you configure a pay per crawl price, AI Audit automatically sets the default action for all applicable AI crawlers as **Charge**. +When you configure a pay per crawl price, AI Crawl Control automatically sets the default action for all applicable AI crawlers as **Charge**. ::: \ No newline at end of file diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-up-cloudflare-account.mdx b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-up-cloudflare-account.mdx similarity index 96% rename from src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-up-cloudflare-account.mdx rename to src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-up-cloudflare-account.mdx index 1b1f58e124ee522..35d9a2e39b83c09 100644 --- a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-up-cloudflare-account.mdx +++ b/src/content/docs/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-up-cloudflare-account.mdx @@ -26,7 +26,7 @@ Sign up for a [Cloudflare account](https://dash.cloudflare.com/sign-up). [Connect your domain](/fundamentals/manage-domains/add-site/) to Cloudflare. Domains you add to Cloudflare become [zones](/fundamentals/concepts/accounts-and-zones/#zones). -AI Audit scopes pay per crawl to your zones. +AI Crawl Control scopes pay per crawl to your zones. ## 3. Proxy traffic through Cloudflare diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl.mdx b/src/content/docs/ai-crawl-control/features/pay-per-crawl/what-is-pay-per-crawl.mdx similarity index 68% rename from src/content/docs/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl.mdx rename to src/content/docs/ai-crawl-control/features/pay-per-crawl/what-is-pay-per-crawl.mdx index 3b817eca9e3df22..82b4ea73ed01413 100644 --- a/src/content/docs/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl.mdx +++ b/src/content/docs/ai-crawl-control/features/pay-per-crawl/what-is-pay-per-crawl.mdx @@ -21,7 +21,7 @@ The resulting landscape leaves content owners with limited options for managing ## What is Pay Per Crawl? -Pay per crawl is a feature of AI Audit that enables site owners to control and monetize AI crawler access to content by setting a price per zone. +Pay per crawl is a feature of AI Crawl Control that enables site owners to control and monetize AI crawler access to content by setting a price per zone. Each time an AI crawler requests content, they either present payment intent via request headers for successful `HTTP 200` access, or receive an `HTTP 402 Payment Required` response with pricing. Cloudflare acts as the Merchant of Record for pay per crawl and also provides the underlying technical infrastructure. @@ -31,16 +31,16 @@ If you block an AI crawler from a zone via either of Cloudflare's [WAF](/waf/) o Ultimately, pay per crawl enables: -- Site owners to take control of their content, and charge a fee every time an AI crawler accesses a page in their [Cloudflare zone](/fundamentals/concepts/accounts-and-zones/#zones). For more details, refer to [use pay per crawl as a site owner](/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/). -- AI crawler owners to pay to access content on sites protected by pay per crawl. For more details, refer to [use pay per crawl as an AI owner](/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/). +- Site owners to take control of their content, and charge a fee every time an AI crawler accesses a page in their [Cloudflare zone](/fundamentals/concepts/accounts-and-zones/#zones). For more details, refer to [use pay per crawl as a site owner](/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/). +- AI crawler owners to pay to access content on sites protected by pay per crawl. For more details, refer to [use pay per crawl as an AI owner](/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/). -![Pay per crawl components](~/assets/images/ai-audit/ai-audit-pay-per-crawl-diagram.png) +![Pay per crawl components](~/assets/images/ai-crawl-control/ai-crawl-control-pay-per-crawl-diagram.png) ## Additional resources Refer to the following resources. -- [Use pay per crawl as a site owner](/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-up-cloudflare-account/). -- [Use pay per crawl as an AI owner](/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/set-up-cloudflare-account/). -- [AI Audit with Cloudflare WAF](/ai-audit/configuration/ai-audit-with-waf/). -- [AI Audit with Cloudflare Bots](/ai-audit/configuration/ai-audit-with-bots/). +- [Use pay per crawl as a site owner](/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-up-cloudflare-account/). +- [Use pay per crawl as an AI owner](/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/set-up-cloudflare-account/). +- [AI Crawl Control with Cloudflare WAF](/ai-crawl-control/configuration/ai-crawl-control-with-waf/). +- [AI Crawl Control with Cloudflare Bots](/ai-crawl-control/configuration/ai-crawl-control-with-bots/). diff --git a/src/content/docs/ai-audit/get-started.mdx b/src/content/docs/ai-crawl-control/get-started.mdx similarity index 72% rename from src/content/docs/ai-audit/get-started.mdx rename to src/content/docs/ai-crawl-control/get-started.mdx index f49c456f1d36e07..9f0a016263b4f78 100644 --- a/src/content/docs/ai-audit/get-started.mdx +++ b/src/content/docs/ai-crawl-control/get-started.mdx @@ -7,8 +7,8 @@ sidebar: badge: beta head: - tag: title - content: Get started with Cloudflare AI Audit -description: Learn how to set up AI Audit. + content: Get started with Cloudflare AI Crawl Control +description: Learn how to set up AI Crawl Control. --- import { Details, Render, Steps, Tabs, TabItem, GlossaryTooltip } from "~/components"; @@ -16,7 +16,7 @@ import { Details, Render, Steps, Tabs, TabItem, GlossaryTooltip } from "~/compon This guide instructs you on how to: - View AI crawlers that are interacting with pages in your domain (a [Cloudflare zone](/fundamentals/concepts/accounts-and-zones/#zones)). -- Use AI Audit to block individual crawlers from accessing your content. +- Use AI Crawl Control to block individual crawlers from accessing your content. ## Prerequisites @@ -33,14 +33,14 @@ To only block specific AI crawlers: {/* prettier-ignore */} -1. Go to **AI Audit**. +1. Go to **AI Crawl Control**. 2. From the **AI Crawlers** tab, go to the list of AI crawlers. 3. Review the AI crawlers that are accessing your domain. 4. From the **Block** column, select the option **Block**. :::note[Quality of AI crawler detection] -On the free plan, AI Audit identifies AI crawlers based on their [user agent strings](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/User-Agent). This enables AI Audit to detect easy-to-detect (well-known) AI crawlers. +On the free plan, AI Crawl Control identifies AI crawlers based on their [user agent strings](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/User-Agent). This enables AI Crawl Control to detect easy-to-detect (well-known) AI crawlers. Upgrade your plan to enable a more thorough detection using Cloudflare's [Bot Management detection ID](/bots/reference/bot-management-variables/#ruleset-engine-fields) field. ::: @@ -51,7 +51,7 @@ To only block specific AI crawlers: {/* prettier-ignore */} -1. Go to **AI Audit**. +1. Go to **AI Crawl Control**. 2. From the **AI Crawlers** tab, go to the list of AI crawlers. 3. Review key information such as: - Bot operator @@ -63,7 +63,7 @@ To only block specific AI crawlers: -For more information, refer to [Manage AI crawlers](/ai-audit/features/manage-ai-crawlers/). +For more information, refer to [Manage AI crawlers](/ai-crawl-control/features/manage-ai-crawlers/). You can also create more complex rules when taking action on AI crawlers, using [Cloudflare WAF](/waf/). For more information on creating more specific rules, refer to [Create a custom rule in the dashboard](/waf/custom-rules/create-dashboard/). @@ -75,7 +75,7 @@ You can also create more complex rules when taking action on AI crawlers, using Review the AI crawlers detected on your site. -1. Go to **AI Audit**. +1. Go to **AI Crawl Control**. 2. From the **Metrics** tab, view key metrics on AI crawlers. @@ -85,13 +85,13 @@ Note that on free plans, the **Metrics** tab only displays metrics for the past ## Plan comparison - + ## Next steps -- [Manage AI crawlers](/ai-audit/features/manage-ai-crawlers/) with granular allow/block controls. -- [Analyze AI traffic](/ai-audit/features/analyze-ai-traffic/) to understand crawler patterns and content popularity. -- [Explore pay per crawl](/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl/) to test content monetization options (private beta). +- [Manage AI crawlers](/ai-crawl-control/features/manage-ai-crawlers/) with granular allow/block controls. +- [Analyze AI traffic](/ai-crawl-control/features/analyze-ai-traffic/) to understand crawler patterns and content popularity. +- [Explore pay per crawl](/ai-crawl-control/features/pay-per-crawl/what-is-pay-per-crawl/) to test content monetization options (private beta). ## Related resources diff --git a/src/content/docs/ai-audit/index.mdx b/src/content/docs/ai-crawl-control/index.mdx similarity index 78% rename from src/content/docs/ai-audit/index.mdx rename to src/content/docs/ai-crawl-control/index.mdx index 15ea064f03747f5..1ca3b9890f8c031 100644 --- a/src/content/docs/ai-audit/index.mdx +++ b/src/content/docs/ai-crawl-control/index.mdx @@ -1,16 +1,14 @@ --- -title: AI Audit +title: AI Crawl Control pcx_content_type: overview tags: - AI sidebar: order: 1 - badge: Beta head: - tag: title content: Overview description: Monitor and control how AI services access your website content. - --- import { Description, Feature, FeatureTable, Plan, LinkButton, RelatedProduct, Card } from "~/components"; @@ -23,21 +21,17 @@ Monitor and control how AI services access your website content. -AI companies are increasingly using web content to train their models and power AI applications. AI Audit gives you visibility into which AI services are accessing your content, and provides tools to manage access according to your preferences. +AI companies use web content to train their models and power AI applications. AI Crawl Control (formerly AI Audit) gives you visibility into which AI services are accessing your content, and provides tools to manage access according to your preferences. -With AI Audit, you can: +With AI Crawl Control, you can: - **See which AI services access your content** - Monitor the dashboard to see crawler activity and request patterns - **Control access with granular policies** - Set allow or block rules for individual crawlers - **Monitor robots.txt compliance** - Track which crawlers follow your directives and create enforcement rules -- **Explore monetization options** - Set up pay per crawl pricing for content access [(private beta)](/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl/) +- **Explore monetization options** - Set up pay per crawl pricing for content access [(private beta)](/ai-crawl-control/features/pay-per-crawl/what-is-pay-per-crawl/) - **Deploy with zero configuration** - Works automatically on all Cloudflare plans -:::note[Beta phase] -AI Audit is currently only available as a beta product. -::: - -Get started +Get started --- @@ -45,7 +39,7 @@ AI Audit is currently only available as a beta product. Control how AI crawlers interact with your domain. @@ -53,7 +47,7 @@ AI Audit is currently only available as a beta product. Gain insight into how AI crawlers are interacting with your pages. @@ -61,7 +55,7 @@ AI Audit is currently only available as a beta product. Allow AI crawlers to access content by paying per crawl. diff --git a/src/content/docs/ai-audit/reference/faq.mdx b/src/content/docs/ai-crawl-control/reference/faq.mdx similarity index 100% rename from src/content/docs/ai-audit/reference/faq.mdx rename to src/content/docs/ai-crawl-control/reference/faq.mdx diff --git a/src/content/docs/ai-audit/reference/glossary.mdx b/src/content/docs/ai-crawl-control/reference/glossary.mdx similarity index 62% rename from src/content/docs/ai-audit/reference/glossary.mdx rename to src/content/docs/ai-crawl-control/reference/glossary.mdx index 0060ea697e269c4..41f9c21f96f0747 100644 --- a/src/content/docs/ai-audit/reference/glossary.mdx +++ b/src/content/docs/ai-crawl-control/reference/glossary.mdx @@ -8,6 +8,6 @@ sidebar: import { Glossary } from "~/components" -Review the definitions for terms used across Cloudflare's AI Audit documentation. +Review the definitions for terms used across Cloudflare's AI Crawl Control documentation. diff --git a/src/content/docs/ai-audit/reference/index.mdx b/src/content/docs/ai-crawl-control/reference/index.mdx similarity index 100% rename from src/content/docs/ai-audit/reference/index.mdx rename to src/content/docs/ai-crawl-control/reference/index.mdx diff --git a/src/content/docs/bots/additional-configurations/block-ai-bots.mdx b/src/content/docs/bots/additional-configurations/block-ai-bots.mdx index b0896149b590228..567ae2a0f3026eb 100644 --- a/src/content/docs/bots/additional-configurations/block-ai-bots.mdx +++ b/src/content/docs/bots/additional-configurations/block-ai-bots.mdx @@ -30,4 +30,4 @@ To block [AI bots](/bots/concepts/bot/#ai-bots): 6. Select **Save** to save your configuration. -To block individual AI crawlers (rather than blocking all crawlers), use [AI Audit](/ai-audit/). \ No newline at end of file +To block individual AI crawlers (rather than blocking all crawlers), use [AI Crawl Control](/ai-crawl-control/). \ No newline at end of file diff --git a/src/content/docs/bots/additional-configurations/managed-robots-txt.mdx b/src/content/docs/bots/additional-configurations/managed-robots-txt.mdx index b9d52c8fe134233..f54bb3c9808f28a 100644 --- a/src/content/docs/bots/additional-configurations/managed-robots-txt.mdx +++ b/src/content/docs/bots/additional-configurations/managed-robots-txt.mdx @@ -13,7 +13,7 @@ Protect your website or application from AI crawlers by implementing a `robots.t AI bots are expected to follow the `robots.txt` directives. :::note -Respecting `robots.txt` is voluntary. If you want to prevent crawling, use AI Audit's [manage AI crawlers](/ai-audit/features/manage-ai-crawlers/) feature. +Respecting `robots.txt` is voluntary. If you want to prevent crawling, use AI Crawl Control's [manage AI crawlers](/ai-crawl-control/features/manage-ai-crawlers/) feature. ::: ## Compatibility with existing `robots.txt` files diff --git a/src/content/docs/fundamentals/api/how-to/create-via-api.mdx b/src/content/docs/fundamentals/api/how-to/create-via-api.mdx index 4a49e8cbac2eb78..1dff560331a022c 100644 --- a/src/content/docs/fundamentals/api/how-to/create-via-api.mdx +++ b/src/content/docs/fundamentals/api/how-to/create-via-api.mdx @@ -106,20 +106,24 @@ To fetch all available permission groups and their IDs, use the [List permission ```json output { - "result": [ - { - "id": "19637fbb73d242c0a92845d8db0b95b1", - "name": "AI Audit Read", - "description": "Grants access to reading AI Audit", - "scopes": ["com.cloudflare.api.account.zone"] - }, - { - "id": "1ba6ab4cacdb454b913bbb93e1b8cb8c", - "name": "AI Audit Write", - "description": "Grants access to reading and editing AI Audit", - "scopes": ["com.cloudflare.api.account.zone"] - } - // (...) + "result": [ + { + "id": "19637fbb73d242c0a92845d8db0b95b1", + "name": "AI Crawl Control Read", + "description": "Grants access to reading AI Crawl Control", + "scopes": [ + "com.cloudflare.api.account.zone" + ] + }, + { + "id": "1ba6ab4cacdb454b913bbb93e1b8cb8c", + "name": "AI Crawl Control Write", + "description": "Grants access to reading and editing AI Crawl Control", + "scopes": [ + "com.cloudflare.api.account.zone" + ] + }, + // (...) ] } ``` diff --git a/src/content/docs/style-guide/how-we-docs/metadata.mdx b/src/content/docs/style-guide/how-we-docs/metadata.mdx index 275bdcfaf7865ac..ecf5cf80c046b2c 100644 --- a/src/content/docs/style-guide/how-we-docs/metadata.mdx +++ b/src/content/docs/style-guide/how-we-docs/metadata.mdx @@ -82,12 +82,12 @@ However, the `last_modified` value is pulled automatically from the git history We choose to render all of these values as specific `meta` properties for each page. -For example, these are the `meta` properties and values on the [AI Audit - Get Started page](/ai-audit/get-started/). +For example, these are the `meta` properties and values on the [AI Crawl Control - Get Started page](/ai-crawl-control/get-started/). {/* prettier-ignore */} -```html title="Get Started | AI Audit" +```html title="Get Started | AI Crawl Control" - + ``` diff --git a/src/content/glossary/ai-audit.yaml b/src/content/glossary/ai-audit.yaml index 5c9f7cc25ca3581..3d45354a33278ec 100644 --- a/src/content/glossary/ai-audit.yaml +++ b/src/content/glossary/ai-audit.yaml @@ -1,5 +1,5 @@ --- -productName: AI Audit +productName: AI Crawl Control entries: - term: robots.txt general_definition: |- diff --git a/src/content/partials/ai-audit/plan-comparison.mdx b/src/content/partials/ai-crawl-control/plan-comparison.mdx similarity index 93% rename from src/content/partials/ai-audit/plan-comparison.mdx rename to src/content/partials/ai-crawl-control/plan-comparison.mdx index f9bd61ef47ad546..f56f51d865fe7c5 100644 --- a/src/content/partials/ai-audit/plan-comparison.mdx +++ b/src/content/partials/ai-crawl-control/plan-comparison.mdx @@ -7,4 +7,4 @@ | ----------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | | AI crawler detection via [user agent strings](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/User-Agent) | Advanced AI crawler detection via [Bot Management detection ID](/bots/reference/bot-management-variables/#ruleset-engine-fields) | | Maximum 24-hour analytics window | Configurable analytics timeframes | -| Allow/block controls | Allow/block controls, and the ability to charge AI crawlers using [pay per crawl](/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl/) | \ No newline at end of file +| Allow/block controls | Allow/block controls, and the ability to charge AI crawlers using [pay per crawl](/ai-crawl-control/features/pay-per-crawl/what-is-pay-per-crawl/) | \ No newline at end of file diff --git a/src/content/products/ai-audit.yaml b/src/content/products/ai-crawl-control.yaml similarity index 60% rename from src/content/products/ai-audit.yaml rename to src/content/products/ai-crawl-control.yaml index 81b91f74f6a7c10..cf73152d8cc0089 100644 --- a/src/content/products/ai-audit.yaml +++ b/src/content/products/ai-crawl-control.yaml @@ -1,11 +1,11 @@ -name: AI Audit +name: AI Crawl Control product: - title: AI Audit - url: /ai-audit/ + title: AI Crawl Control + url: /ai-crawl-control/ group: Core platform additional_groups: [AI] tags: [AI] meta: - title: Cloudflare AI Audit docs + title: Cloudflare AI Crawl Control docs description: Analyze and control third-party AI crawlers in your website author: "@cloudflare" diff --git a/src/icons/ai-audit.svg b/src/icons/ai-crawl-control.svg similarity index 100% rename from src/icons/ai-audit.svg rename to src/icons/ai-crawl-control.svg