Retry HTTP requests on 409 errors (#78)

* Retry http requests on 409 errors

* Break up complex conditional

Author: samanehsan

pipeline_tools/http_requests.py

@@ -159,7 +159,12 @@ def is_retryable(error):
             print('{0} {1}'.format(now, repr(error)))
 
             def is_retryable_status_code(error):
-                return isinstance(error, requests.HTTPError) and not (400 <= error.response.status_code <= 499)
+                if not isinstance(error, requests.HTTPError):
+                    return False
+                if error.response.status_code == 409:
+                    return True
+                else:
+                    return not (400 <= error.response.status_code <= 499)
 
             return is_retryable_status_code(error) or isinstance(error,
                                                                  (requests.ConnectionError, requests.ReadTimeout))
@@ -293,7 +298,7 @@ def _get_next_file_suffix(files):
 
     @staticmethod
     def check_status(response):
-        """Check that the response status code is in range 200-299.
+        """Check that the response status code is in range 200-299, or 409.
         Raises a ValueError and prints response_text if status is not in the expected range. Otherwise,
         just returns silently.
         Args:
@@ -311,7 +316,7 @@ def check_status(response):
             check_status(301, 'bar') raises error
         """
         status = response.status_code
-        matches = 200 <= status <= 299
+        matches = 200 <= status <= 299 or status == 409
         if not matches:
             message = 'HTTP status code {0} is not in expected range 2xx. Response: {1}'.format(status, response.text)
             raise requests.HTTPError(message, response=response)

pipeline_tools/tests/test_http_requests.py

@@ -40,6 +40,13 @@ def test_check_status_acceptable_codes(self):
         except requests.HTTPError as e:
             pytest.fail(str(e))
 
+        try:
+            response = requests.Response()
+            response.status_code = 409
+            HttpRequests.check_status(response)
+        except requests.HTTPError as e:
+            pytest.fail(str(e))
+
     def test_get_retries_then_raises_exception_on_500(self, requests_mock):
         def callback(request, response):
             response.status_code = 500