Add more pages to robots.txt (#17798)

Author: di

tests/functional/test_basic.py

@@ -37,6 +37,9 @@ def test_robots_txt(app_config, domain, indexable):
             "Disallow: /pypi/*/*/json\n"
             "Disallow: /pypi*?\n"
             "Disallow: /search*\n"
+            "Disallow: /_/\n"
+            "Disallow: /integrity/\n"
+            "Disallow: /admin/\n"
         )
     else:
         assert body == (

warehouse/templates/robots.txt

@@ -11,4 +11,7 @@ Disallow: /pypi/*/json
 Disallow: /pypi/*/*/json
 Disallow: /pypi*?
 Disallow: /search*
+Disallow: /_/
+Disallow: /integrity/
+Disallow: /admin/
 {% endif %}