diff --git a/public/robots.txt b/public/robots.txt index 2520e82139fece..fb27101216b00a 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,5 +1,35 @@ +# As a condition of accessing this website, you agree to +# abide by the following content signals: + +# (a) If a content-signal = yes, you may collect content +# for the corresponding use. +# (b) If a content-signal = no, you may not collect content +# for the corresponding use. +# (c) If the website operator does not include a content +# signal for a corresponding use, the website operator +# neither grants nor restricts permission via content signal +# with respect to the corresponding use. + +# The content signals and their meanings are: + +# search: building a search index and providing search +# results (e.g., returning hyperlinks and short excerpts +# from your website's contents). Search does not include +# providing AI-generated search summaries. +# ai-input: inputting content into one or more AI models +# (e.g., retrieval augmented generation, grounding, or other +# real-time taking of content for generative AI search +# answers). +# ai-train: training or fine-tuning AI models. + +# ANY RESTRICTIONS EXPRESSED VIA CONTENT-SIGNALS ARE EXPRESS +# RESERVATIONS OF RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN +# UNION DIRECTIVE 2019/790 ON COPYRIGHT AND RELATED RIGHTS +# IN THE DIGITAL SINGLE MARKET. + User-agent: * +Allow: / Disallow: /client-ip-geolocation Disallow: /plans/ Disallow: /constellation @@ -7,5 +37,6 @@ Disallow: /cdn-cgi/ Disallow: /email-security/ Content-Usage: tdm=y +Content-Signal: ai-train=yes, search=yes, ai-input=yes -Sitemap: https://developers.cloudflare.com/sitemap-index.xml +Sitemap: https://developers.cloudflare.com/sitemap-index.xml \ No newline at end of file