Handle internal server errors and 404

Author: vshlapakov

scrapinghub/client/exceptions.py

@@ -52,6 +52,10 @@ class DuplicateJobError(ScrapinghubAPIError):
     pass
 
 
+class ServerError(ScrapinghubAPIError):
+    pass
+
+
 def wrap_http_errors(method):
     @wraps(method)
     def wrapped(*args, **kwargs):
@@ -69,6 +73,8 @@ def wrapped(*args, **kwargs):
                 raise ValueTooLarge(http_error=exc)
             elif 400 <= status_code < 500:
                 raise ScrapinghubAPIError(http_error=exc)
+            elif 500 <= status_code < 600:
+                raise ServerError(http_error=exc)
             raise
         except APIError as exc:
             msg = exc.args[0]
@@ -80,6 +86,8 @@ def wrapped(*args, **kwargs):
                 raise BadRequest(msg)
             elif exc._type == APIError.ERR_AUTH_ERROR:
                 raise Unauthorized(http_error=exc)
+            elif exc._type == APIError.ERR_SERVER_ERROR:
+                raise ServerError(http_error=exc)
             raise ScrapinghubAPIError(msg)
     return wrapped
 

scrapinghub/legacy.py

@@ -138,6 +138,11 @@ def _request(self, url, data, headers, format, raw, files=None):
         return self._decode_response(response, format, raw)
 
     def _decode_response(self, response, format, raw):
+        if response.status_code == 404:
+            raise APIError("Not found", _type=APIError.ERR_NOT_FOUND)
+        elif 500 <= response.status_code < 600:
+            raise APIError("Internal server error",
+                           _type=APIError.ERR_SERVER_ERROR)
         if raw:
             return response.raw
         elif format == 'json':
@@ -402,11 +407,12 @@ def _add_params(self, params):
 
 class APIError(Exception):
 
-    ERR_DEFAULT = 0
-    ERR_NOT_FOUND = 1
-    ERR_VALUE_ERROR = 2
-    ERR_BAD_REQUEST = 3
-    ERR_AUTH_ERROR = 4
+    ERR_DEFAULT = "err_default"
+    ERR_NOT_FOUND = "err_not_found"
+    ERR_VALUE_ERROR = "err_value_error"
+    ERR_BAD_REQUEST = "err_bad_request"
+    ERR_AUTH_ERROR = "err_auth_error"
+    ERR_SERVER_ERROR = "err_server_error"
 
     def __init__(self, message, _type=None):
         super(APIError, self).__init__(message)

tests/client/test_projects.py

@@ -2,12 +2,14 @@
 from collections import defaultdict, Iterator
 
 import pytest
+import responses
 from six.moves import range
+from requests.compat import urljoin
 
 from scrapinghub import ScrapinghubClient
 from scrapinghub.client.activity import Activity
 from scrapinghub.client.collections import Collections
-from scrapinghub.client.exceptions import DuplicateJobError
+from scrapinghub.client.exceptions import DuplicateJobError, ServerError
 from scrapinghub.client.frontiers import Frontiers
 from scrapinghub.client.jobs import Jobs, Job
 from scrapinghub.client.projects import Project, Settings
@@ -45,6 +47,15 @@ def test_projects_list(client):
     assert int(TEST_PROJECT_ID) in projects
 
 
+@responses.activate
+def test_projects_list_server_error(client):
+    url = urljoin(TEST_DASH_ENDPOINT, 'scrapyd/listprojects.json')
+    responses.add(responses.GET, url, body='some error body', status=500)
+    with pytest.raises(ServerError):
+        client.projects.list()
+    assert len(responses.calls) == 1
+
+
 def test_projects_summary(client, project):
     # add at least one running or pending job to ensure summary is returned
     project.jobs.schedule(TEST_SPIDER_NAME, meta={'state': 'running'})