Make the maximum retries a provider setting

During local testing, all 429 failures succeeded upon first retry.
The issue turned out to be "namespace lock contention" instead of
"two many requests". It is reasonable to set the default maximum
value to 2 retires per upload.

Author: cslzchen

waterbutler/providers/dropbox/provider.py

@@ -61,6 +61,7 @@ class DropboxProvider(provider.BaseProvider):
     BASE_URL = pd_settings.BASE_URL
     CONTIGUOUS_UPLOAD_SIZE_LIMIT = pd_settings.CONTIGUOUS_UPLOAD_SIZE_LIMIT
     CHUNK_SIZE = pd_settings.CHUNK_SIZE
+    MAX_429_RETRIES = pd_settings.MAX_429_RETRIES
 
     def __init__(self, auth, credentials, settings):
         super().__init__(auth, credentials, settings)
@@ -311,7 +312,7 @@ async def _contiguous_upload(self,
             chunk = await stream.read()
 
         rate_limit_retry = 0
-        while rate_limit_retry < 2:
+        while rate_limit_retry < self.MAX_429_RETRIES:
             file_stream = streams.FileStreamReader(file_cache)
             resp = await self.make_request(
                 'POST',

waterbutler/providers/dropbox/settings.py

@@ -10,3 +10,5 @@
 CONTIGUOUS_UPLOAD_SIZE_LIMIT = int(config.get('CONTIGUOUS_UPLOAD_SIZE_LIMIT', 150000000))  # 150 MB
 
 CHUNK_SIZE = int(config.get('CHUNK_SIZE', 4000000))  # 4 MB
+
+MAX_429_RETRIES = int(config.get('MAX_429_RETRIES', 2))