Add comment to Sitemap explaining how to customize it (#12164)

Author: ericholscher

readthedocs/proxito/tests/test_full.py

@@ -864,7 +864,7 @@ def test_default_robots_txt(self, storage_exists):
             Sitemap: https://project.readthedocs.io/sitemap.xml
             """
         ).lstrip()
-        self.assertEqual(response.content.decode(), expected)
+        self.assertContains(response, expected)
 
     @mock.patch.object(BuildMediaFileSystemStorageTest, "exists")
     def test_default_robots_txt_disallow_hidden_versions(self, storage_exists):
@@ -918,7 +918,7 @@ def test_default_robots_txt_disallow_hidden_versions(self, storage_exists):
             Sitemap: https://project.readthedocs.io/sitemap.xml
             """
         ).lstrip()
-        self.assertEqual(response.content.decode(), expected)
+        self.assertContains(response, expected)
 
     @mock.patch.object(BuildMediaFileSystemStorageTest, "exists")
     def test_default_robots_txt_private_version(self, storage_exists):

readthedocs/templates/robots.txt

@@ -1,3 +1,11 @@
+# This robots.txt file is autogenerated by Read the Docs.
+# It controls the crawling and indexing of your documentation by search engines.
+#
+# You can learn more about robots.txt, including how to customize it, in our documentation:
+#
+# * Our documentation on Robots.txt: https://docs.readthedocs.com/platform/stable/reference/robots.html
+# * Our guide about SEO techniques: https://docs.readthedocs.com/platform/stable/guides/technical-docs-seo-guide.html
+
 User-agent: *
 {% for path in hidden_paths %}
 Disallow: {{ path }} # Hidden version

readthedocs/templates/sitemap.xml

@@ -1,3 +1,13 @@
+<!--
+This sitemap is autogenerated by Read the Docs.
+It contains all public versions,
+sorted by version number.
+
+You can learn more about sitemaps, including how to customize them, in our documentation:
+
+* Our documentation on Sitemaps: https://docs.readthedocs.com/platform/stable/reference/sitemaps.html
+* Our guide about SEO techniques: https://docs.readthedocs.com/platform/stable/guides/technical-docs-seo-guide.html
+-->
 <?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
         xmlns:xhtml="http://www.w3.org/1999/xhtml">