Add a deny-all robots.txt

helm-quarry/values.yaml

@@ -1,6 +1,6 @@
 web:
   repository: 'quay.io/wikimedia-quarry/quarry'
-  tag: pr-90 # web tag managed by github actions
+  tag: pr-92 # web tag managed by github actions
   resources:
     requests:
       memory: "300Mi"
@@ -11,7 +11,7 @@ web:
 
 worker:
   repository: 'quay.io/wikimedia-quarry/quarry'
-  tag: pr-90 # worker tag managed by github actions
+  tag: pr-92 # worker tag managed by github actions
   resources:
     requests:
       memory: "400Mi"

quarry/web/app.py

@@ -1,4 +1,4 @@
-from flask import current_app, Flask, render_template, g
+from flask import current_app, Flask, render_template, g, Response
 from flask_caching import Cache
 from werkzeug.middleware.proxy_fix import ProxyFix
 
@@ -72,6 +72,10 @@ def index():
             stats_count_runs=global_conn.session.query(QueryRun).count(),
         )
 
+    @app.route("/robots.txt")
+    def robots_txt():
+        return Response("User-Agent: *\nDisallow: /\n", mimetype="text/plain")
+
     return app
 
 

tests/test_app.py

@@ -35,3 +35,9 @@ def test_frontpage(self, mocker, client):
         assert response.data
         assert "5 users" in response.data.decode("utf8")
         assert f"{5 * 3 * 3 * 3} queries" in response.data.decode("utf8")
+
+    def test_robots_txt(self, client):
+        response = client.get("/robots.txt")
+        assert response.status_code == 200
+        assert response.headers["Content-Type"] == "text/plain; charset=utf-8"
+        assert "User-Agent: *" in response.data.decode("utf8")