Made 429 replays optional and configurable

Added support for adapter argument on client constructors.
Moved 429 HTTPAdapter configuration into Replay429Adapter class in
cloudant.adapters.
Added retries and initialBackoff args to Replay429Adapter.
Added tests for init and adding adapter with Replay429Adapter and args.
Updated documentation including CHANGES.rst and new adapters module.

Author: ricellis

CHANGES.rst

@@ -3,6 +3,12 @@
 - [NEW] Added support for Cloudant Search execution.
 - [NEW] Added support for Cloudant Search index management.
 - [NEW] Added ``rewrites`` accessor property for URL rewriting.
+- [NEW] Added support for a custom ``requests.HTTPAdapter`` to be configured using an optional ``adapter`` arg e.g.
+  ``Cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME, adapter=Replay429Adapter())``.
+- [IMPROVED] Made the 429 response code backoff optional and configurable. To enable the backoff add
+  an ``adapter`` arg of a ``Replay429Adapter`` with the desired number of retries and initial backoff. To replicate
+  the 2.0.0 behaviour use: ``adapter=Replay429Adapter(retries=10, initialBackoff=0.25)``. If ``retries`` or
+  ``initialBackoff`` are not specified they will default to 3 retries and a 0.25 s initial backoff.
 
 2.0.3 (2016-06-03)
 ==================

docs/adapters.rst

@@ -0,0 +1,8 @@
+adapters
+========
+
+.. automodule:: cloudant.adapters
+    :members:
+    :undoc-members:
+    :special-members: __getitem__, __iter__
+    :show-inheritance:

docs/modules.rst

@@ -15,3 +15,4 @@ Modules
    replicator
    feed
    error
+   adapters

src/cloudant/adapters.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright © 2016 IBM Corp. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Module that contains default transport adapters for use with requests.
+"""
+from requests.adapters import HTTPAdapter
+
+from requests.packages.urllib3.util import Retry
+
+class Replay429Adapter(HTTPAdapter):
+    """
+    A requests TransportAdapter that extends the default HTTPAdapter with configuration
+    to replay requests that receive a 429 Too Many Requests response from the server.
+    The duration of the sleep between requests will be doubled for each 429 response
+    received.
+
+    Parameters can be passed in to control behavior:
+
+    :param int retries: the number of times the request can be replayed before failing.
+    :param float initialBackoff: time in seconds for the first backoff.
+    """
+    def __init__(self, retries=3, initialBackoff=0.25):
+        super(Replay429Adapter, self).__init__(max_retries=Retry(
+            # Configure the number of retries for status codes
+            total=retries,
+            # No retries for connect|read errors
+            connect=0,
+            read=0,
+            # Allow retries for all the CouchDB HTTP method types
+            method_whitelist=frozenset(['GET', 'HEAD', 'PUT', 'POST',
+                                        'DELETE', 'COPY']),
+            # Only retry for a 429 too many requests status code
+            status_forcelist=[429],
+            # Configure the start value of the doubling backoff
+            backoff_factor=initialBackoff))

src/cloudant/client.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright (c) 2015, 2016 IBM. All rights reserved.
+# Copyright (C) 2015, 2016 IBM Corp. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,8 +21,6 @@
 import posixpath
 import requests
 
-from requests.adapters import HTTPAdapter
-from requests.packages.urllib3.util import Retry
 from ._2to3 import bytes_, unicode_
 from .database import CloudantDatabase, CouchDatabase
 from .feed import Feed, InfiniteFeed
@@ -46,6 +44,7 @@ class CouchDB(dict):
         CouchDB.  Defaults to ``False``.
     :param str encoder: Optional json Encoder object used to encode
         documents for storage.  Defaults to json.JSONEncoder.
+    :param requests.HTTPAdapter adapter: Optional adapter to use for configuring requests.
     """
     _DATABASE_CLASS = CouchDatabase
 
@@ -58,6 +57,7 @@ def __init__(self, user, auth_token, admin_party=False, **kwargs):
         self._client_user_header = None
         self.admin_party = admin_party
         self.encoder = kwargs.get('encoder') or json.JSONEncoder
+        self.adapter = kwargs.get('adapter')
         self.r_session = None
 
     def connect(self):
@@ -66,21 +66,9 @@ def connect(self):
         authentication.
         """
         self.r_session = requests.Session()
-        # Configure a Transport Adapter for custom retry behaviour
-        self.r_session.mount(self.server_url, HTTPAdapter(
-            max_retries=Retry(
-                # Allow 10 retries for status
-                total=10,
-                # No retries for connect|read errors
-                connect=0,
-                read=0,
-                # Allow retries for all the CouchDB HTTP method types
-                method_whitelist=frozenset(['GET', 'HEAD', 'PUT', 'POST',
-                                            'DELETE', 'COPY']),
-                # Only retry for a 429 too many requests status code
-                status_forcelist=[429],
-                # Configure the doubling backoff to start at 0.25 s
-                backoff_factor=0.25)))
+        # If a Transport Adapter was supplied add it to the session
+        if self.adapter is not None:
+            self.r_session.mount(self.server_url, self.adapter)
         if self._client_user_header is not None:
             self.r_session.headers.update(self._client_user_header)
         if not self.admin_party:
@@ -394,6 +382,7 @@ class Cloudant(CouchDB):
         the x_cloudant_user parameter setting is ignored.
     :param str encoder: Optional json Encoder object used to encode
         documents for storage. Defaults to json.JSONEncoder.
+    :param requests.HTTPAdapter adapter: Optional adapter to use for configuring requests.
     """
     _DATABASE_CLASS = CloudantDatabase
 

tests/unit/adapter_tests.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright © 2016 IBM Corp. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from cloudant.client import CouchDB
+from cloudant.adapters import Replay429Adapter
+from requests.packages.urllib3.util import Retry
+from .unit_t_db_base import UnitTestDbBase
+
+class AdapterTests(UnitTestDbBase):
+    """
+    Unit tests for transport adapters
+    """
+
+    def test_new_Replay429Adapter(self):
+        """
+        Test that a new Replay429Adapter is accepted as a parameter for a client.
+        """
+        self.client = CouchDB(
+            self.user,
+            self.pwd,
+            url=self.url,
+            adapter=Replay429Adapter())
+
+    def test_retries_arg_Replay429Adapter(self):
+        """
+        Test constructing a new Replay429Adapter with a configured number of retries.
+        """
+        self.client = CouchDB(
+            self.user,
+            self.pwd,
+            url=self.url,
+            adapter=Replay429Adapter(retries=10))
+
+
+
+    def test_backoff_arg_Replay429Adapter(self):
+        """
+        Test constructing a new Replay429Adapter with a configured initial backoff.
+        """
+        self.client = CouchDB(
+            self.user,
+            self.pwd,
+            url=self.url,
+            adapter=Replay429Adapter(initialBackoff=0.1))
+
+    def test_args_Replay429Adapter(self):
+        """
+        Test constructing a new Replay429Adapter with configured retries and initial backoff.
+        """
+        self.client = CouchDB(
+            self.user,
+            self.pwd,
+            url=self.url,
+            adapter=Replay429Adapter(retries=10, initialBackoff=0.01))