From 60400f9de6bbb3f515973673b92d70d6fdbd6c65 Mon Sep 17 00:00:00 2001 From: Jun Lee Date: Tue, 1 Jul 2025 14:32:56 +0100 Subject: [PATCH] Misc docs improvements --- .../set-up-cloudflare-account.mdx | 7 +++++++ .../set-up-cloudflare-account.mdx | 8 ++++++++ .../features/pay-per-crawl/what-is-pay-per-crawl.mdx | 6 ++++-- .../bots/additional-configurations/managed-robots-txt.mdx | 4 ++++ 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/set-up-cloudflare-account.mdx b/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/set-up-cloudflare-account.mdx index 6fb2323d9d1b29..ad895fc488316f 100644 --- a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/set-up-cloudflare-account.mdx +++ b/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/set-up-cloudflare-account.mdx @@ -19,3 +19,10 @@ To begin using pay per crawl, set up your Cloudflare account. Sign up for a [Cloudflare account](https://dash.cloudflare.com/sign-up). +:::note[Pay per crawl closed beta] +Pay per crawl is currently in closed beta. + +To find out how to join the beta program, reach out to us at [Pay per crawl signup](http://www.cloudflare.com/paypercrawl-signup/), or contact your account executive if you are an existing Enterprise customer. + +To learn more about pay per crawl, refer to Cloudflare blog: [Introducing pay per crawl: enabling content owners to charge AI crawlers for access](https://blog.cloudflare.com/introducing-pay-per-crawl/). +::: \ No newline at end of file diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-up-cloudflare-account.mdx b/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-up-cloudflare-account.mdx index f696dd381b6ecf..1b1f58e124ee52 100644 --- a/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-up-cloudflare-account.mdx +++ b/src/content/docs/ai-audit/features/pay-per-crawl/use-pay-per-crawl-as-site-owner/set-up-cloudflare-account.mdx @@ -31,3 +31,11 @@ AI Audit scopes pay per crawl to your zones. ## 3. Proxy traffic through Cloudflare Make sure your domain is [proxying traffic through Cloudflare](/fundamentals/concepts/how-cloudflare-works/#cloudflare-as-a-reverse-proxy). + +:::note[Pay per crawl closed beta] +Pay per crawl is currently in closed beta. + +To find out how to join the beta program, reach out to us at [Pay per crawl signup](http://www.cloudflare.com/paypercrawl-signup/), or contact your account executive if you are an existing Enterprise customer. + +To learn more about pay per crawl, refer to Cloudflare blog: [Introducing pay per crawl: enabling content owners to charge AI crawlers for access](https://blog.cloudflare.com/introducing-pay-per-crawl/). +::: \ No newline at end of file diff --git a/src/content/docs/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl.mdx b/src/content/docs/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl.mdx index f3a8a7e2e8a81a..32c34a18e7b41c 100644 --- a/src/content/docs/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl.mdx +++ b/src/content/docs/ai-audit/features/pay-per-crawl/what-is-pay-per-crawl.mdx @@ -7,10 +7,12 @@ sidebar: import { Steps, GlossaryTooltip } from "~/components"; -:::note[Closed beta] +:::note[Pay per crawl closed beta] Pay per crawl is currently in closed beta. -To find out how to join the beta program and to learn more about pay per crawl, refer to Cloudflare blog: [Introducing pay per crawl: enabling content owners to charge AI crawlers for access](https://blog.cloudflare.com/introducing-pay-per-crawl/). +To find out how to join the beta program, reach out to us at [Pay per crawl signup](http://www.cloudflare.com/paypercrawl-signup/), or contact your account executive if you are an existing Enterprise customer. + +To learn more about pay per crawl, refer to Cloudflare blog: [Introducing pay per crawl: enabling content owners to charge AI crawlers for access](https://blog.cloudflare.com/introducing-pay-per-crawl/). ::: AI crawlers often consume vast amounts of web content. Some provide mutual benefit to content owners by indexing content for search engines, but others engage in activities such as content scraping without permission. diff --git a/src/content/docs/bots/additional-configurations/managed-robots-txt.mdx b/src/content/docs/bots/additional-configurations/managed-robots-txt.mdx index bbc9db3944dd91..ab47554a74b5f5 100644 --- a/src/content/docs/bots/additional-configurations/managed-robots-txt.mdx +++ b/src/content/docs/bots/additional-configurations/managed-robots-txt.mdx @@ -14,6 +14,10 @@ Cloudflare's managed `robots.txt` explicitly disallows known bots engaged in scr AI bots are expected to follow the `robots.txt` directives. +:::note +Respecting `robots.txt` is voluntary. If you want to prevent crawling, use AI Audit's [manage AI crawlers](/ai-audit/features/manage-ai-crawlers/) feature. +::: + ## Compatibility with existing `robots.txt` files Cloudflare will independently check whether your website has an existing `robots.txt` file and update the behavior of this feature based on your website.