diff --git a/src/content/docs/bots/additional-configurations/managed-robots-txt.mdx b/src/content/docs/bots/additional-configurations/managed-robots-txt.mdx index ec0f90331dd979d..35f3b13730bddcb 100644 --- a/src/content/docs/bots/additional-configurations/managed-robots-txt.mdx +++ b/src/content/docs/bots/additional-configurations/managed-robots-txt.mdx @@ -1,9 +1,9 @@ --- pcx_content_type: reference -title: Instruct AI crawlers with managed robots.txt +title: robots.txt setting sidebar: order: 10 - label: Managed robots.txt + label: robots.txt setting --- import { Render, Tabs, TabItem, Steps, DashButton } from "~/components"; @@ -41,7 +41,7 @@ With the managed `robots.txt` enabled, Cloudflare will prepend our managed conte ```txt title="Feature enabled" # As a condition of accessing this website, you agree to abide by the -# following content-signals: +# following content signals: # (a) If a content-signal = yes, you may collect content for the # corresponding use. @@ -61,14 +61,14 @@ With the managed `robots.txt` enabled, Cloudflare will prepend our managed conte # content for generative AI search answers). # ai-train: training or fine-tuning AI models. -# ANY RESTRICTIONS EXPRESSED VIA CONTENT-SIGNALS ARE EXPRESS RESERVATIONS OF +# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF # RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT # AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET. # BEGIN Cloudflare Managed content User-Agent: * -Content-signal: search=yes,ai-train=no +Content-signal: search=yes, ai-train=no Allow: / User-agent: Amazonbot @@ -127,8 +127,8 @@ To implement a `robots.txt` file on your domain: 2. Filter by **Bot traffic**. - 3. Go to **Instruct AI bot traffic with robots.txt**. - 4. Turn **Instruct AI bot traffic with robots.txt** on. + 3. Go to **robots.txt setting**. + 4. Turn **robots.txt setting** on. @@ -141,7 +141,7 @@ This file only outlines the Content Signals framework. It does not express your ```txt title="Content Signals Policy" # As a condition of accessing this website, you agree to abide by the -# following content-signals: +# following content signals: # (a) If a content-signal = yes, you may collect content for the # corresponding use. @@ -161,16 +161,16 @@ This file only outlines the Content Signals framework. It does not express your # content for generative AI search answers). # ai-train: training or fine-tuning AI models. -# ANY RESTRICTIONS EXPRESSED VIA CONTENT-SIGNALS ARE EXPRESS RESERVATIONS OF +# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF # RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT # AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET. ``` -Cloudflare's Content Signals Policy is included by default in the `robots.txt` file when you turn on **Instruct AI bot traffic with robots.txt**. +Cloudflare's Content Signals Policy is included by default in the `robots.txt` file when you turn on **robots.txt setting**. If you would like to opt out of displaying the policy in your `robots.txt` file, you can uncheck **Display Content Signals Policy** under **Control AI Crawlers** in your zone's overview. - + Alternatively, you can use [Security Settings](#implementation).