Proxito: add X-Robots-Tag header (#12349)

Closes #12245

Author: humitos

dockerfiles/nginx/proxito.conf.template

@@ -75,6 +75,8 @@ server {
         add_header X-RTD-Redirect $rtd_redirect always;
         set $rtd_resolver_filename $upstream_http_x_rtd_resolver_filename;
         add_header X-RTD-Resolver-Filename $rtd_resolver_filename always;
+        set $x_robots_tag $upstream_http_x_robots_tag;
+        add_header X-Robots-Tag $x_robots_tag always;
         set $cdn_cache_control $upstream_http_cdn_cache_control;
         add_header CDN-Cache-Control $cdn_cache_control always;
         set $cache_control $upstream_http_cache_control;

readthedocs/proxito/middleware.py

@@ -193,6 +193,12 @@ def add_cdn_cache_headers(self, request, response):
         else:
             cache_response(response, force=False)
 
+    def add_x_robots_tag_headers(self, request, response):
+        """Add `X-Robots-Tag: noindex` header for external versions."""
+        unresolved_domain = request.unresolved_domain
+        if unresolved_domain and unresolved_domain.is_from_external_domain:
+            response["X-Robots-Tag"] = "noindex"
+
     def _set_request_attributes(self, request, unresolved_domain):
         """
         Set attributes in the request from the unresolved domain.
@@ -367,6 +373,7 @@ def process_response(self, request, response):  # noqa
         self.add_proxito_headers(request, response)
         self.add_cache_headers(request, response)
         self.add_cdn_cache_headers(request, response)
+        self.add_x_robots_tag_headers(request, response)
         self.add_hsts_headers(request, response)
         self.add_user_headers(request, response)
         self.add_hosting_integrations_headers(request, response)

readthedocs/proxito/tests/test_headers.py

@@ -396,3 +396,27 @@ def test_cache_headers_at_browser_level_on_external_domain(self):
         )
         self.assertEqual(r.status_code, 200)
         self.assertEqual(r["Cache-Control"], "no-cache")
+
+    def test_x_robots_tag_header(self):
+        r = self.client.get(
+            "/en/latest/", secure=True, headers={"host": "project.dev.readthedocs.io"}
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertNotIn("X-Robots-Tag", r.headers)
+
+        get(
+            Version,
+            project=self.project,
+            slug="111",
+            active=True,
+            privacy_level=PUBLIC,
+            type=EXTERNAL,
+        )
+
+        r = self.client.get(
+            "/en/111/",
+            secure=True,
+            headers={"host": "project--111.dev.readthedocs.build"},
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r["X-Robots-Tag"], "noindex")