Add a robots.txt for forklift (#17796)

* Add a robots.txt for forklift

* Update tests

Author: di

tests/unit/forklift/test_init.py

@@ -72,6 +72,13 @@ def test_includeme(forklift_domain, monkeypatch):
                 route_kw={"domain": forklift_domain},
                 view_kw={"has_translations": True},
             ),
+            pretend.call(
+                "forklift.robots.txt",
+                "/robots.txt",
+                "forklift.robots.txt",
+                route_kw={"domain": forklift_domain},
+                view_kw={"has_translations": False},
+            ),
             pretend.call(
                 "forklift.legacy.invalid_request",
                 "/legacy/",

warehouse/forklift/__init__.py

@@ -71,6 +71,14 @@ def includeme(config):
             view_kw={"has_translations": True},
         )
 
+        config.add_template_view(
+            "forklift.robots.txt",
+            "/robots.txt",
+            "forklift.robots.txt",
+            route_kw={"domain": forklift},
+            view_kw={"has_translations": False},
+        )
+
         # Any call to /legacy/ not handled by another route (e.g. no :action
         # URL parameter, or an invalid :action URL parameter) falls through to
         # this catch-all route.

warehouse/templates/forklift.robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /