CORS fixes and a new API

dbutenhof · portante · commit abab42d06f8d · 2020-12-15T15:52:23.000-05:00
diff --git a/lib/pbench/server/api/__init__.py b/lib/pbench/server/api/__init__.py
@@ -8,6 +8,7 @@
 
 from flask import Flask
 from flask_restful import Api
+from flask_cors import CORS
 
 from pbench.server import PbenchServerConfig
 from pbench.common.exceptions import BadConfig, ConfigFileNotSpecified
@@ -16,6 +17,7 @@
 from pbench.common.logger import get_pbench_logger
 from pbench.server.api.resources.query_apis.elasticsearch_api import Elasticsearch
 from pbench.server.api.resources.query_apis.query_controllers import QueryControllers
+from pbench.server.api.resources.query_apis.query_month_indices import QueryMonthIndices
 
 
 def register_endpoints(api, app, config):
@@ -41,12 +43,16 @@ def register_endpoints(api, app, config):
     api.add_resource(
         GraphQL, f"{base_uri}/graphql", resource_class_args=(config, app.logger),
     )
-
     api.add_resource(
         QueryControllers,
         f"{base_uri}/controllers/list",
         resource_class_args=(config, app.logger),
     )
+    api.add_resource(
+        QueryMonthIndices,
+        f"{base_uri}/controllers/months",
+        resource_class_args=(config, app.logger),
+    )
 
 
 def get_server_config():
@@ -69,6 +75,7 @@ def create_app(server_config):
 
     app = Flask("api-server")
     api = Api(app)
+    CORS(app, resources={r"/api/*": {"origins": "*"}})
 
     app.logger = get_pbench_logger(__name__, server_config)
 
diff --git a/lib/pbench/server/api/resources/graphql_api.py b/lib/pbench/server/api/resources/graphql_api.py
@@ -1,7 +1,6 @@
 import requests
 from flask_restful import Resource, abort
 from flask import request, make_response
-from flask_cors import cross_origin
 
 
 class GraphQL(Resource):
@@ -12,7 +11,6 @@ def __init__(self, config, logger):
         self.graphql_host = config.get_conf(__name__, "graphql", "host", self.logger)
         self.graphql_port = config.get_conf(__name__, "graphql", "port", self.logger)
 
-    @cross_origin()
     def post(self):
         self.graphql = f"http://{self.graphql_host}:{self.graphql_port}"
 
@@ -27,6 +25,9 @@ def post(self):
             # query GraphQL
             gql_response = requests.post(self.graphql, json=json_data)
             gql_response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            self.logger.exception("HTTP error {} from Elasticsearch post request", e)
+            abort(gql_response.status_code, message=f"HTTP error {e} from GraphQL")
         except requests.exceptions.ConnectionError:
             self.logger.exception("Connection refused during the GraphQL post request")
             abort(502, message="Network problem, could not post to GraphQL Endpoint")
diff --git a/lib/pbench/server/api/resources/query_apis/elasticsearch_api.py b/lib/pbench/server/api/resources/query_apis/elasticsearch_api.py
@@ -1,7 +1,6 @@
 import requests
 from flask_restful import Resource, abort
 from flask import request, make_response
-from flask_cors import cross_origin
 from pbench.server.api.resources.query_apis import get_es_url
 
 
@@ -12,7 +11,6 @@ def __init__(self, config, logger):
         self.logger = logger
         self.elasticsearch = get_es_url(config)
 
-    @cross_origin()
     def post(self):
         json_data = request.get_json(silent=True)
         if not json_data:
diff --git a/lib/pbench/server/api/resources/query_apis/query_controllers.py b/lib/pbench/server/api/resources/query_apis/query_controllers.py
@@ -1,4 +1,4 @@
-from flask import request
+from flask import request, jsonify
 from flask_restful import Resource, abort
 import requests
 
@@ -94,14 +94,6 @@ def post(self):
         cluster by choosing unique prefixes to generate unique index
         names.
 
-        TODO: This implementation currently returns the raw Elasticsearch
-        JSON response payload to the caller. The intent is to make this
-        more attuned to the Pbench domain by removing the Elasticsearch
-        "boilerplate" without dropping information required by a client
-        (including, in particular, the Pbench dashboard). Essentially,
-        this will extend the implementation up through the dashboard
-        "model" layer instead of simply replacing the "service" layer.
-
         Required headers include
 
         X-auth-token:   Pbench session authorization token authorized to access
@@ -115,6 +107,10 @@ def post(self):
         name, the number of runs using that controller name, and the start
         timestamp of the latest run both in binary and string form.
 
+        TODO: We probably need to return all *unowned* controllers when called
+        without a session token or user parameter. We need to define precisely
+        how this should work.
+
         NOTE: This is the format currently constructed by the Pbench
         dashboard `src/model/dashboard.js` fetchControllers method, which
         becomes part of the Redux state.
@@ -131,23 +127,17 @@ def post(self):
         """
         json_data = request.get_json(silent=True)
         if not json_data:
-            self.logger.info(
-                "QueryControllers: Invalid JSON object. Query: {}", request.url
-            )
-            abort(400, message="QueryControllers: Missing request payload")
+            self.logger.info("Invalid JSON object. Query: {}", request.url)
+            abort(400, message="Missing request payload")
 
         try:
             user = json_data["user"]
             start_arg = json_data["start"]
             end_arg = json_data["end"]
         except KeyError:
             keys = [k for k in ("user", "start", "end") if k not in json_data]
-            self.logger.info(
-                "QueryControllers: Missing required JSON keys {}", ",".join(keys)
-            )
-            abort(
-                400, message=f"QueryControllers: Missing request data: {','.join(keys)}"
-            )
+            self.logger.info("Missing required JSON keys {}", ",".join(keys))
+            abort(400, message=f"Missing request data: {','.join(keys)}")
 
         # We need to support a query for public data without requiring
         # authorization; however if an Authorization header is given,
@@ -167,7 +157,7 @@ def post(self):
                     '"Authorization" header specifies unsupported or missing '
                     ' schema; use "Authorization: Bearer <JWT-token>"'
                 )
-                abort(401, message="QueryControllers: invalid user authorization")
+                abort(401, message="invalid user authorization")
 
         try:
             start = parser.parse(start_arg)
@@ -176,7 +166,7 @@ def post(self):
             self.logger.info(
                 "Invalid start or end time string: {}, {}: {}", start_arg, end_arg, e
             )
-            abort(400, message="QueryControllers: Invalid start or end time string")
+            abort(400, message="Invalid start or end time string")
 
         # We have nothing to authorize against yet, but print the specified
         # user and session token to prove we got them (and so the variables
@@ -234,7 +224,7 @@ def post(self):
             es_response.raise_for_status()
         except requests.exceptions.HTTPError as e:
             self.logger.exception("HTTP error {} from Elasticsearch post request", e)
-            abort(500, message=f"HTTP error {e} from Elasticsearch")
+            abort(502, message="INTERNAL ERROR")
         except requests.exceptions.ConnectionError:
             self.logger.exception(
                 "Connection refused during the Elasticsearch post request"
@@ -244,22 +234,17 @@ def post(self):
             self.logger.exception(
                 "Connection timed out during the Elasticsearch post request"
             )
-            abort(
-                504, message="Connection timed out, could not post to Elasticsearch",
-            )
+            abort(504, message="Connection timed out, could not post to Elasticsearch")
         except requests.exceptions.InvalidURL:
             self.logger.exception(
                 "Invalid url {} during the Elasticsearch post request", uri
             )
-            abort(
-                500,
-                message=f"Invalid Elasticsearch url {uri}, could not complete the post request",
-            )
+            abort(500, message="INTERNAL ERROR")
         except Exception as e:
             self.logger.exception(
                 "Exception {!r} occurred during the Elasticsearch post request", e
             )
-            abort(500, message=f"Could not post to Elasticsearch: {e!r}")
+            abort(500, message="INTERNAL ERROR")
         else:
             controllers = []
             try:
@@ -276,7 +261,7 @@ def post(self):
                     controllers.append(c)
             except KeyError:
                 self.logger.exception("ES response not formatted as expected")
-                abort(500, message="Elasticsearch response is odd")
+                abort(500, message="INTERNAL ERROR")
             else:
                 # construct response object
-                return controllers
+                return jsonify(controllers)
diff --git a/lib/pbench/server/api/resources/query_apis/query_month_indices.py b/lib/pbench/server/api/resources/query_apis/query_month_indices.py
@@ -0,0 +1,95 @@
+from flask import jsonify
+from flask_restful import Resource, abort
+import requests
+
+from pbench.server.api.resources.query_apis import get_es_url, get_index_prefix
+
+
+class QueryMonthIndices(Resource):
+    """
+    Abstracted Pbench API to get date-bounded controller data.
+    """
+
+    def __init__(self, config, logger):
+        self.logger = logger
+        self.es_url = get_es_url(config)
+        self.prefix = get_index_prefix(config)
+
+    def get(self):
+        """
+        GET to detect the month suffixes existing for run data indices.
+
+        Required headers include
+
+        Content-Type:   application/json
+        Accept:         application/json
+
+        The return payload is a list of "YYYY-mm" date strings corresponding
+        to the months in which tarballs were indexed into the appropriate
+        `run-data` index. (E.g., `drb.v6.run-data.2020-11`). Note that this
+        list is in DESCENDING order, so the earliest date is last.
+
+        NOTE: No authorization or input payload is required for this API.
+
+        NOTE: This is the format currently constructed by the Pbench
+        dashboard `src/model/datastore.js` fetchMonthIndices method, which
+        becomes part of the Redux state.
+
+        [
+            "2020-12",
+            "2020-11",
+            "2020-04"
+        ]
+        """
+        self.logger.info(
+            "QueryMonthIndices GET for prefix {}", self.prefix,
+        )
+
+        uri = f"{self.es_url}/_aliases"
+        try:
+            # query Elasticsearch
+            es_response = requests.get(uri, headers={"Accept": "application/json"})
+            es_response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            self.logger.exception("HTTP error {} from Elasticsearch post request", e)
+            abort(502, message="INTERNAL ERROR")
+        except requests.exceptions.ConnectionError:
+            self.logger.exception(
+                "Connection refused during the Elasticsearch post request"
+            )
+            abort(502, message="Network problem, could not post to Elasticsearch")
+        except requests.exceptions.Timeout:
+            self.logger.exception(
+                "Connection timed out during the Elasticsearch post request"
+            )
+            abort(504, message="Connection timed out, could not post to Elasticsearch")
+        except requests.exceptions.InvalidURL:
+            self.logger.exception(
+                "Invalid url {} during the Elasticsearch post request", uri
+            )
+            abort(500, message="INTERNAL ERROR")
+        except Exception as e:
+            self.logger.exception(
+                "Exception {!r} occurred during the Elasticsearch post request", e
+            )
+            abort(500, message="INTERNAL ERROR")
+        else:
+            months = []
+            target = f"{self.prefix}.v6.run-data."
+            try:
+                es_json = es_response.json()
+                self.logger.info("looking for {} in {}", target, es_json)
+                for index in es_json.keys():
+                    if target in index:
+                        months.append(index.split(".")[-1])
+                # The dashboard converts the strings to int (removing the '-')
+                # and does a numeric sort; however since we always have a full
+                # "YYYY-mm" date, this isn't necessary.
+                months.sort(reverse=True)
+                self.logger.info("found months {!r}", months)
+            except KeyError:
+                self.logger.exception("ES response not formatted as expected")
+                abort(500, message="INTERNAL ERROR")
+            else:
+                # construct response object
+                return jsonify(months)
diff --git a/lib/pbench/test/unit/server/test_query_controller.py b/lib/pbench/test/unit/server/test_query_controller.py
@@ -67,9 +67,7 @@ def test_missing_json_object(self, client, server_config):
         """
         response = client.post(f"{server_config.rest_uri}/controllers/list")
         assert response.status_code == 400
-        assert (
-            response.json.get("message") == "QueryControllers: Missing request payload"
-        )
+        assert response.json.get("message") == "Missing request payload"
 
     @pytest.mark.parametrize(
         "keys",
@@ -95,8 +93,7 @@ def test_missing_keys(self, client, server_config, keys):
         assert response.status_code == 400
         missing = [k for k in ("user", "start", "end") if k not in keys]
         assert (
-            response.json.get("message")
-            == f"QueryControllers: Missing request data: {','.join(missing)}"
+            response.json.get("message") == f"Missing request data: {','.join(missing)}"
         )
 
     def test_bad_dates(self, client, server_config):
@@ -113,10 +110,7 @@ def test_bad_dates(self, client, server_config):
             },
         )
         assert response.status_code == 400
-        assert (
-            response.json.get("message")
-            == "QueryControllers: Invalid start or end time string"
-        )
+        assert response.json.get("message") == "Invalid start or end time string"
 
     def test_query(self, client, server_config, query_helper):
         """
@@ -182,7 +176,7 @@ def test_query(self, client, server_config, query_helper):
     @pytest.mark.parametrize(
         "exceptions",
         (
-            {"exception": requests.exceptions.HTTPError, "status": 500},
+            {"exception": requests.exceptions.HTTPError, "status": 502},
             {"exception": requests.exceptions.ConnectionError, "status": 502},
             {"exception": requests.exceptions.Timeout, "status": 504},
             {"exception": requests.exceptions.InvalidURL, "status": 500},
diff --git a/lib/pbench/test/unit/server/test_query_month_indices.py b/lib/pbench/test/unit/server/test_query_month_indices.py
