Merge pull request #187 from cloudant/60008-optional-429

Made 429 replays optional and configurable

Author: ricellis

CHANGES.rst

@@ -3,6 +3,12 @@
 - [NEW] Added support for Cloudant Search execution.
 - [NEW] Added support for Cloudant Search index management.
 - [NEW] Added ``rewrites`` accessor property for URL rewriting.
+- [NEW] Added support for a custom ``requests.HTTPAdapter`` to be configured using an optional ``adapter`` arg e.g.
+  ``Cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME, adapter=Replay429Adapter())``.
+- [IMPROVED] Made the 429 response code backoff optional and configurable. To enable the backoff add
+  an ``adapter`` arg of a ``Replay429Adapter`` with the desired number of retries and initial backoff. To replicate
+  the 2.0.0 behaviour use: ``adapter=Replay429Adapter(retries=10, initialBackoff=0.25)``. If ``retries`` or
+  ``initialBackoff`` are not specified they will default to 3 retries and a 0.25 s initial backoff.
 
 2.0.3 (2016-06-03)
 ==================

docs/adapters.rst

@@ -0,0 +1,8 @@
+adapters
+========
+
+.. automodule:: cloudant.adapters
+    :members:
+    :undoc-members:
+    :special-members: __getitem__, __iter__
+    :show-inheritance:

docs/modules.rst

@@ -15,3 +15,4 @@ Modules
    replicator
    feed
    error
+   adapters

src/cloudant/adapters.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright © 2016 IBM Corp. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Module that contains default transport adapters for use with requests.
+"""
+from requests.adapters import HTTPAdapter
+
+from requests.packages.urllib3.util import Retry
+
+class Replay429Adapter(HTTPAdapter):
+    """
+    A requests TransportAdapter that extends the default HTTPAdapter with configuration
+    to replay requests that receive a 429 Too Many Requests response from the server.
+    The duration of the sleep between requests will be doubled for each 429 response
+    received.
+
+    Parameters can be passed in to control behavior:
+
+    :param int retries: the number of times the request can be replayed before failing.
+    :param float initialBackoff: time in seconds for the first backoff.
+    """
+    def __init__(self, retries=3, initialBackoff=0.25):
+        super(Replay429Adapter, self).__init__(max_retries=Retry(
+            # Configure the number of retries for status codes
+            total=retries,
+            # No retries for connect|read errors
+            connect=0,
+            read=0,
+            # Allow retries for all the CouchDB HTTP method types
+            method_whitelist=frozenset(['GET', 'HEAD', 'PUT', 'POST',
+                                        'DELETE', 'COPY']),
+            # Only retry for a 429 too many requests status code
+            status_forcelist=[429],
+            # Configure the start value of the doubling backoff
+            backoff_factor=initialBackoff))

src/cloudant/client.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright (c) 2015, 2016 IBM. All rights reserved.
+# Copyright (C) 2015, 2016 IBM Corp. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,8 +21,6 @@
 import posixpath
 import requests
 
-from requests.adapters import HTTPAdapter
-from requests.packages.urllib3.util import Retry
 from ._2to3 import bytes_, unicode_
 from .database import CloudantDatabase, CouchDatabase
 from .feed import Feed, InfiniteFeed
@@ -46,6 +44,7 @@ class CouchDB(dict):
         CouchDB.  Defaults to ``False``.
     :param str encoder: Optional json Encoder object used to encode
         documents for storage.  Defaults to json.JSONEncoder.
+    :param requests.HTTPAdapter adapter: Optional adapter to use for configuring requests.
     """
     _DATABASE_CLASS = CouchDatabase
 
@@ -58,6 +57,7 @@ def __init__(self, user, auth_token, admin_party=False, **kwargs):
         self._client_user_header = None
         self.admin_party = admin_party
         self.encoder = kwargs.get('encoder') or json.JSONEncoder
+        self.adapter = kwargs.get('adapter')
         self.r_session = None
 
     def connect(self):
@@ -66,21 +66,9 @@ def connect(self):
         authentication.
         """
         self.r_session = requests.Session()
-        # Configure a Transport Adapter for custom retry behaviour
-        self.r_session.mount(self.server_url, HTTPAdapter(
-            max_retries=Retry(
-                # Allow 10 retries for status
-                total=10,
-                # No retries for connect|read errors
-                connect=0,
-                read=0,
-                # Allow retries for all the CouchDB HTTP method types
-                method_whitelist=frozenset(['GET', 'HEAD', 'PUT', 'POST',
-                                            'DELETE', 'COPY']),
-                # Only retry for a 429 too many requests status code
-                status_forcelist=[429],
-                # Configure the doubling backoff to start at 0.25 s
-                backoff_factor=0.25)))
+        # If a Transport Adapter was supplied add it to the session
+        if self.adapter is not None:
+            self.r_session.mount(self.server_url, self.adapter)
         if self._client_user_header is not None:
             self.r_session.headers.update(self._client_user_header)
         if not self.admin_party:
@@ -394,6 +382,7 @@ class Cloudant(CouchDB):
         the x_cloudant_user parameter setting is ignored.
     :param str encoder: Optional json Encoder object used to encode
         documents for storage. Defaults to json.JSONEncoder.
+    :param requests.HTTPAdapter adapter: Optional adapter to use for configuring requests.
     """
     _DATABASE_CLASS = CloudantDatabase
 

tests/unit/adapter_tests.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright © 2016 IBM Corp. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from cloudant.client import CouchDB
+from cloudant.adapters import Replay429Adapter
+from requests.packages.urllib3.util import Retry
+from .unit_t_db_base import UnitTestDbBase
+
+class AdapterTests(UnitTestDbBase):
+    """
+    Unit tests for transport adapters
+    """
+
+    def test_new_Replay429Adapter(self):
+        """
+        Test that a new Replay429Adapter is accepted as a parameter for a client.
+        """
+        self.client = CouchDB(
+            self.user,
+            self.pwd,
+            url=self.url,
+            adapter=Replay429Adapter())
+
+    def test_retries_arg_Replay429Adapter(self):
+        """
+        Test constructing a new Replay429Adapter with a configured number of retries.
+        """
+        self.client = CouchDB(
+            self.user,
+            self.pwd,
+            url=self.url,
+            adapter=Replay429Adapter(retries=10))
+
+
+
+    def test_backoff_arg_Replay429Adapter(self):
+        """
+        Test constructing a new Replay429Adapter with a configured initial backoff.
+        """
+        self.client = CouchDB(
+            self.user,
+            self.pwd,
+            url=self.url,
+            adapter=Replay429Adapter(initialBackoff=0.1))
+
+    def test_args_Replay429Adapter(self):
+        """
+        Test constructing a new Replay429Adapter with configured retries and initial backoff.
+        """
+        self.client = CouchDB(
+            self.user,
+            self.pwd,
+            url=self.url,
+            adapter=Replay429Adapter(retries=10, initialBackoff=0.01))