Merge pull request #28 from geo-engine/wfs-error-check

jdroenner · web-flow · commit 74104e1c29b3 · 2021-10-21T10:18:24.000+02:00
Wfs-error-check
diff --git a/geoengine/__init__.py b/geoengine/__init__.py
@@ -3,7 +3,7 @@
 from .auth import Session, get_session, initialize, reset
 from .error import GeoEngineException, InputException, UninitializedException, TypeException, \
     MethodNotCalledOnPlotException, MethodNotCalledOnRasterException, MethodNotCalledOnVectorException, \
-    SpatialReferenceMismatchException
+    SpatialReferenceMismatchException, check_response_for_error
 from .types import QueryRectangle
 from .workflow import WorkflowId, Workflow, workflow_by_id, register_workflow
 from .datasets import upload_dataframe, StoredDataset
diff --git a/geoengine/error.py b/geoengine/error.py
@@ -4,6 +4,8 @@
 
 from typing import Dict
 
+from requests import Response, HTTPError
+
 
 class GeoEngineException(Exception):
     '''
@@ -107,3 +109,28 @@ def __init__(self, spatial_reference_a: str, spatial_reference_b: str) -> None:
 
     def __str__(self) -> str:
         return f"Spatial reference mismatch {self.__spatial_reference_a} != {self.__spatial_reference_b}"
+
+
+def check_response_for_error(response: Response):
+    '''
+    Checks a `Response` for an error and raises it if there is one.
+    '''
+
+    try:
+        response.raise_for_status()
+
+        return  # no error
+    except HTTPError as http_error:
+        exception = http_error
+
+    # try to parse it as a Geo Engine error
+    try:
+        response_json = response.json()
+        if 'error' in response_json:
+            # override exception with `GeoEngineException`
+            exception = GeoEngineException(response_json)
+    except Exception:  # pylint: disable=broad-except
+        pass  # ignore errors, it seemed not to be JSON
+
+    # either raise the `GeoEngineException` or any other `HTTPError`
+    raise exception
diff --git a/geoengine/workflow.py b/geoengine/workflow.py
@@ -7,7 +7,7 @@
 
 from uuid import UUID
 from logging import debug
-from io import StringIO, BytesIO
+from io import BytesIO
 import urllib.parse
 import json
 
@@ -26,7 +26,7 @@
 from geoengine.types import InternalDatasetId, ProvenanceOutput, QueryRectangle, ResultDescriptor
 from geoengine.auth import get_session
 from geoengine.error import GeoEngineException, MethodNotCalledOnPlotException, MethodNotCalledOnRasterException, \
-    MethodNotCalledOnVectorException, SpatialReferenceMismatchException
+    MethodNotCalledOnVectorException, SpatialReferenceMismatchException, check_response_for_error
 from geoengine.datasets import StoredDataset, UploadId
 
 
@@ -165,17 +165,20 @@ def get_dataframe(self, bbox: QueryRectangle) -> gpd.GeoDataFrame:
 
         data_response = req.get(wfs_url, headers=session.auth_header)
 
-        def geo_json_with_time_to_geopandas(data_response):
+        check_response_for_error(data_response)
+
+        data = data_response.json()
+
+        def geo_json_with_time_to_geopandas(geo_json):
             '''
             GeoJson has no standard for time, so we parse the when field
             separately and attach it to the data frame as columns `start`
             and `end`.
             '''
 
-            data = gpd.read_file(StringIO(data_response.text))
+            data = gpd.GeoDataFrame.from_features(geo_json)
             data = data.set_crs(bbox.srs, allow_override=True)
 
-            geo_json = data_response.json()
             start = [f['when']['start'] for f in geo_json['features']]
             end = [f['when']['end'] for f in geo_json['features']]
 
@@ -186,7 +189,7 @@ def geo_json_with_time_to_geopandas(data_response):
 
             return data
 
-        return geo_json_with_time_to_geopandas(data_response)
+        return geo_json_with_time_to_geopandas(data)
 
     def plot_image(self, bbox: QueryRectangle, ax: plt.Axes = None, timeout=3600) -> plt.Axes:
         '''
@@ -244,6 +247,8 @@ def wms_get_map_as_image(self, bbox: QueryRectangle, colorizer_min_max: Tuple[fl
         wms_request = self.__wms_get_map_request(bbox, colorizer_min_max)
         response = req.Session().send(wms_request)
 
+        check_response_for_error(response)
+
         return Image.open(BytesIO(response.content))
 
     def __wms_get_map_request(self,
@@ -321,7 +326,11 @@ def plot_chart(self, bbox: QueryRectangle) -> VegaLite:
 
         plot_url = f'{session.server_url}/plot/{self}?bbox={spatial_bounds}&time={time}&spatialResolution={resolution}'
 
-        response = req.get(plot_url, headers=session.auth_header).json()
+        response = req.get(plot_url, headers=session.auth_header)
+
+        check_response_for_error(response)
+
+        response = response.json()
 
         vega_spec = json.loads(response['data']['vegaString'])
 
@@ -406,10 +415,11 @@ def save_as_dataset(self, bbox: QueryRectangle, name: str, description: str = ''
             url=f'{session.server_url}/datasetFromWorkflow/{self.__workflow_id}',
             json=request_body,
             headers=session.auth_header,
-        ).json()
+        )
 
-        if 'error' in response:
-            raise GeoEngineException(response)
+        check_response_for_error(response)
+
+        response = response.json()
 
         return StoredDataset(
             dataset_id=InternalDatasetId.from_response(response['dataset']),
diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,5 @@
 build
+filelock # for cartopy mpl tests
 flufl.lock # for cartopy mpl tests
 pycodestyle # formatter
 pylint # code linter
diff --git a/setup.cfg b/setup.cfg
@@ -18,7 +18,7 @@ package_dir =
 packages = find:
 python_requires = >=3.7
 install_requires =
-    cartopy
+    cartopy==0.19.0.post1
     geopandas
     matplotlib
     numpy
diff --git a/tests/test_plot.py b/tests/test_plot.py
@@ -179,6 +179,79 @@ def test_wrong_request(self):
                     [time, time]
                 ))
 
+    def test_plot_error(self):
+        with requests_mock.Mocker() as m:
+            m.post('http://mock-instance/anonymous', json={
+                "id": "c4983c3e-9b53-47ae-bda9-382223bd5081",
+                "project": None,
+                "view": None
+            })
+
+            m.post('http://mock-instance/workflow',
+                   json={
+                       "id": "5b9508a8-bd34-5a1c-acd6-75bb832d2d38"
+                   },
+                   request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'})
+
+            m.get('http://mock-instance/workflow/5b9508a8-bd34-5a1c-acd6-75bb832d2d38/metadata',
+                  json={
+                      "type": "plot"
+                  },
+                  request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'})
+
+            m.get(
+                # pylint: disable=line-too-long
+                'http://mock-instance/plot/5b9508a8-bd34-5a1c-acd6-75bb832d2d38?bbox=-180.0%2C-90.0%2C180.0%2C90.0&time=2004-04-01T12%3A00%3A00.000%2B00%3A00&spatialResolution=0.1,0.1',
+                json={
+                    "error": "Operator",
+                    "message": 'Operator: Could not open gdal dataset for file path '
+                    '"test_data/raster/modis_ndvi/MOD13A2_M_NDVI_2004-04-01.TIFF"'
+                },
+                status_code=400,
+                request_headers={
+                    'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}
+            )
+
+            ge.initialize("http://mock-instance")
+
+            workflow_definition = {
+                "type": "Plot",
+                "operator": {
+                    "type": "Histogram",
+                    "params": {
+                        "bounds": "data",
+                        "buckets": 20
+                    },
+                    "sources": {
+                        "source": {
+                            "type": "GdalSource",
+                            "params": {
+                                "dataset": {
+                                    "internal": "36574dc3-560a-4b09-9d22-d5945f2b8093"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+
+            time = datetime.strptime(
+                '2004-04-01T12:00:00.000Z', "%Y-%m-%dT%H:%M:%S.%f%z")
+
+            workflow = ge.register_workflow(workflow_definition)
+
+            with self.assertRaises(ge.GeoEngineException) as ctx:
+                workflow.plot_chart(
+                    QueryRectangle(
+                        [-180.0, -90.0, 180.0, 90.0],
+                        [time, time]
+                    )
+                )
+
+            self.assertEqual(str(ctx.exception),
+                             'Operator: Operator: Could not open gdal dataset for file path '
+                             '"test_data/raster/modis_ndvi/MOD13A2_M_NDVI_2004-04-01.TIFF"')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/test_wfs.py b/tests/test_wfs.py
@@ -292,12 +292,6 @@ def test_geopandas(self):
             expected_df = gpd.GeoDataFrame(
                 # pylint: disable=line-too-long
                 {
-                    "scalerank": [7, 7, 6, 5, 5, 5, 3],
-                    "website": ["www.ghanaports.gov.gh", "www.nationalportauthorityliberia.org", None, "www.paa-ci.org", None, None, "www.paa-ci.org"],
-                    "NDVI": [nan, 178.0, 108.0, 99.0, 159.0, 128.0, 126.0],
-                    "natlscale": [10.0, 10.0, 20.0, 30.0, 30.0, 30.0, 75.0],
-                    "featurecla": ["Port", "Port", "Port", "Port", "Port", "Port", "Port"],
-                    "name": ["Tema", "Buchanan", "Nieuw Nickerie", "Abidjan", "Kourou", "Paramaribo", "Abidjan"],
                     "geometry": [
                         Point(0.007420495, 5.631944444),
                         Point(-10.05265018, 5.858055556),
@@ -307,6 +301,12 @@ def test_geopandas(self):
                         Point(-55.13898704, 5.82),
                         Point(-4.021260306, 5.283333333),
                     ],
+                    "scalerank": [7, 7, 6, 5, 5, 5, 3],
+                    "website": ["www.ghanaports.gov.gh", "www.nationalportauthorityliberia.org", None, "www.paa-ci.org", None, None, "www.paa-ci.org"],
+                    "NDVI": [nan, 178.0, 108.0, 99.0, 159.0, 128.0, 126.0],
+                    "natlscale": [10.0, 10.0, 20.0, 30.0, 30.0, 30.0, 75.0],
+                    "featurecla": ["Port", "Port", "Port", "Port", "Port", "Port", "Port"],
+                    "name": ["Tema", "Buchanan", "Nieuw Nickerie", "Abidjan", "Kourou", "Paramaribo", "Abidjan"],
                     "start": [datetime.strptime(
                         '2014-04-01T00:00:00.000Z', "%Y-%m-%dT%H:%M:%S.%f%z") for _ in range(7)],
                     "end": [datetime.strptime(
@@ -404,6 +404,103 @@ def test_wfs_curl(self):
                 """curl -X GET -H "Authorization: Bearer e327d9c3-a4f3-4bd7-a5e1-30b26cae8064" 'http://mock-instance/wfs/956d3656-2d14-5951-96a0-f962b92371cd?service=WFS&version=2.0.0&request=GetFeature&outputFormat=application%2Fjson&typeNames=956d3656-2d14-5951-96a0-f962b92371cd&bbox=-60.0%2C5.0%2C61.0%2C6.0&time=2014-04-01T12%3A00%3A00.000%2B00%3A00&srsName=EPSG%3A4326&queryResolution=0.1%2C0.1'"""
             )
 
+    def test_wfs_error(self):
+        with requests_mock.Mocker() as m:
+            m.post('http://mock-instance/anonymous', json={
+                "id": "e327d9c3-a4f3-4bd7-a5e1-30b26cae8064",
+                "user": {
+                    "id": "328ca8d1-15d7-4f59-a989-5d5d72c98744",
+                },
+                "created": "2021-06-08T15:22:22.605891994Z",
+                "validUntil": "2021-06-08T16:22:22.605892183Z",
+                "project": None,
+                "view": None
+            })
+
+            m.post('http://mock-instance/workflow',
+                   json={
+                       "id": "956d3656-2d14-5951-96a0-f962b92371cd"
+                   },
+                   request_headers={'Authorization': 'Bearer e327d9c3-a4f3-4bd7-a5e1-30b26cae8064'})
+
+            m.get('http://mock-instance/workflow/956d3656-2d14-5951-96a0-f962b92371cd/metadata',
+                  json={
+                      "type": "vector",
+                      "dataType": "MultiPoint",
+                      "spatialReference": "EPSG:4326",
+                      "columns": {
+                          "natlscale": "float",
+                          "featurecla": "text",
+                          "scalerank": "int",
+                          "name": "text",
+                          "NDVI": "int",
+                          "website": "text"
+                      }
+                  },
+                  request_headers={'Authorization': 'Bearer e327d9c3-a4f3-4bd7-a5e1-30b26cae8064'})
+
+            m.get('http://mock-instance/wfs/956d3656-2d14-5951-96a0-f962b92371cd',
+                  json={
+                      "error": "Operator",
+                      "message": 'Operator: Could not open gdal dataset for file path '
+                      '"test_data/raster/modis_ndvi/MOD13A2_M_NDVI_2004-04-01.TIFF"'
+                  },
+                  status_code=400,
+                  request_headers={'Authorization': 'Bearer e327d9c3-a4f3-4bd7-a5e1-30b26cae8064'}
+                  )
+
+            ge.initialize("http://mock-instance")
+
+            workflow_definition = {
+                "type": "Vector",
+                "operator": {
+                    "type": "RasterVectorJoin",
+                    "params": {
+                        "names": ["NDVI"],
+                        "featureAggregation": "first",
+                        "temporalAggregation": "none"
+                    },
+                    "sources": {
+                        "vector": {
+                            "type": "OgrSource",
+                            "params": {
+                                "dataset": {
+                                    "type": "internal",
+                                    "datasetId": "a9623a5b-b6c5-404b-bc5a-313ff72e4e75"
+                                },
+                                "attributeProjection": None
+                            }
+                        },
+                        "rasters": [{
+                            "type": "GdalSource",
+                            "params": {
+                                    "dataset": {
+                                        "type": "internal",
+                                        "datasetId": "36574dc3-560a-4b09-9d22-d5945f2b8093"
+                                    }
+                            }
+                        }]
+                    }
+                }
+            }
+
+            time = datetime.strptime(
+                '2004-04-01T12:00:00.000Z', "%Y-%m-%dT%H:%M:%S.%f%z")
+
+            workflow = ge.register_workflow(workflow_definition)
+
+            with self.assertRaises(ge.GeoEngineException) as ctx:
+                workflow.get_dataframe(
+                    QueryRectangle(
+                        [-60.0, 5.0, 61.0, 6.0],
+                        [time, time]
+                    )
+                )
+
+            self.assertEqual(str(ctx.exception),
+                             'Operator: Operator: Could not open gdal dataset for file path '
+                             '"test_data/raster/modis_ndvi/MOD13A2_M_NDVI_2004-04-01.TIFF"')
+
     def test_repr(self):
         with requests_mock.Mocker() as m:
             m.post('http://mock-instance/anonymous', json={
diff --git a/tests/test_wms.py b/tests/test_wms.py

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`build`
	`2`	`+filelock # for cartopy mpl tests`
`2`	`3`	`flufl.lock # for cartopy mpl tests`
`3`	`4`	`pycodestyle # formatter`
`4`	`5`	`pylint # code linter`