Merge pull request #412 from cloudant/409-workaround-requests

Replaced usage of requests `response.json()` method

Author: ricellis

CHANGES.md

@@ -1,3 +1,9 @@
+# Unreleased
+
+- [FIXED] Unexpected keyword argument errors when using the library with the
+ `simplejson` module present in the environment caused by `requests` preferentially
+  loading it over the system `json` module.
+
 # 2.10.0 (2018-09-19)
 
 - [NEW] Add custom JSON encoder/decoder option to `Document` constructor.

Jenkinsfile

@@ -10,12 +10,13 @@ def getEnvForSuite(suiteName) {
     case 'basic':
       envVars.add("RUN_BASIC_AUTH_TESTS=1")
       break
-    case 'cookie':
-      break
     case 'iam':
       // Setting IAM_API_KEY forces tests to run using an IAM enabled client.
       envVars.add("IAM_API_KEY=$DB_IAM_API_KEY")
       break
+    case 'cookie':
+    case 'simplejson':
+      break
     default:
       error("Unknown test suite environment ${suiteName}")
   }
@@ -36,6 +37,7 @@ def setupPythonAndTest(pythonVersion, testSuite) {
             . ./tmp/bin/activate
             pip install -r requirements.txt
             pip install -r test-requirements.txt
+            ${'simplejson'.equals(testSuite) ? 'pip install simplejson' : ''}
             pylint ./src/cloudant
             nosetests -A 'not db or (db is "cloudant" or "cloudant" in db)' -w ./tests/unit --with-xunit
           """
@@ -58,12 +60,15 @@ stage('Checkout'){
 }
 
 stage('Test'){
-  axes = [:]
-  ['2.7.12','3.5.2'].each { version ->
+  def py2 = '2.7.12'
+  def py3 = '3.5.2'
+  def axes = [:]
+  [py2, py3].each { version ->
     ['basic','cookie','iam'].each { auth ->
        axes.put("Python${version}-${auth}", {setupPythonAndTest(version, auth)})
     }
   }
+  axes.put("Python${py3}-simplejson", {setupPythonAndTest(py3, 'simplejson')})
   parallel(axes)
 }
 

src/cloudant/_client_session.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright (c) 2015, 2018 IBM Corp. All rights reserved.
+# Copyright (C) 2015, 2018 IBM Corp. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@
 from requests import RequestException, Session
 
 from ._2to3 import bytes_, unicode_, url_join
+from ._common_util import response_to_json_dict
 from .error import CloudantException
 
 
@@ -75,7 +76,7 @@ def info(self):
 
         resp = self.get(self._session_url)
         resp.raise_for_status()
-        return resp.json()
+        return response_to_json_dict(resp)
 
     def set_credentials(self, username, password):
         """
@@ -170,7 +171,7 @@ def request(self, method, url, **kwargs):
 
         is_expired = any((
             resp.status_code == 403 and
-            resp.json().get('error') == 'credentials_expired',
+            response_to_json_dict(resp).get('error') == 'credentials_expired',
             resp.status_code == 401
         ))
 
@@ -282,10 +283,10 @@ def _get_access_token(self):
                     'apikey': self._api_key
                 }
             )
-            err = resp.json().get('errorMessage', err)
+            err = response_to_json_dict(resp).get('errorMessage', err)
             resp.raise_for_status()
 
-            return resp.json()['access_token']
+            return response_to_json_dict(resp)['access_token']
 
         except KeyError:
             raise CloudantException('Invalid response from IAM token service')

src/cloudant/_common_util.py

@@ -270,7 +270,7 @@ def append_response_error_content(response, **kwargs):
     """
     if response.status_code >= 400:
         try:
-            resp_dict = response.json()
+            resp_dict = response_to_json_dict(response)
             error = resp_dict.get('error', '')
             reason = resp_dict.get('reason', '')
             # Append to the existing response's reason
@@ -279,6 +279,17 @@ def append_response_error_content(response, **kwargs):
             pass
     return response
 
+def response_to_json_dict(response, **kwargs):
+    """
+    Standard place to convert responses to JSON.
+
+    :param response: requests response object
+    :param **kwargs: arguments accepted by json.loads
+
+    :returns: dict of JSON response
+    """
+    return json.loads(response.text, **kwargs)
+
 # Classes
 
 

src/cloudant/client.py

@@ -35,6 +35,7 @@
     USER_AGENT,
     append_response_error_content,
     CloudFoundryService,
+    response_to_json_dict,
     )
 
 
@@ -256,7 +257,7 @@ def all_dbs(self):
         url = '/'.join((self.server_url, '_all_dbs'))
         resp = self.r_session.get(url)
         resp.raise_for_status()
-        return resp.json()
+        return response_to_json_dict(resp)
 
     def create_database(self, dbname, **kwargs):
         """
@@ -345,7 +346,7 @@ def metadata(self):
         """
         resp = self.r_session.get(self.server_url)
         resp.raise_for_status()
-        return resp.json()
+        return response_to_json_dict(resp)
 
     def keys(self, remote=False):
         """
@@ -622,7 +623,7 @@ def _usage_endpoint(self, endpoint, year=None, month=None):
             raise CloudantArgumentError(101, year, month)
         else:
             resp.raise_for_status()
-            return resp.json()
+            return response_to_json_dict(resp)
 
     def bill(self, year=None, month=None):
         """
@@ -687,7 +688,7 @@ def shared_databases(self):
             self.server_url, '_api', 'v2', 'user', 'shared_databases'))
         resp = self.r_session.get(endpoint)
         resp.raise_for_status()
-        data = resp.json()
+        data = response_to_json_dict(resp)
         return data.get('shared_databases', [])
 
     def generate_api_key(self):
@@ -699,7 +700,7 @@ def generate_api_key(self):
         endpoint = '/'.join((self.server_url, '_api', 'v2', 'api_keys'))
         resp = self.r_session.post(endpoint)
         resp.raise_for_status()
-        return resp.json()
+        return response_to_json_dict(resp)
 
     def cors_configuration(self):
         """
@@ -712,7 +713,7 @@ def cors_configuration(self):
         resp = self.r_session.get(endpoint)
         resp.raise_for_status()
 
-        return resp.json()
+        return response_to_json_dict(resp)
 
     def disable_cors(self):
         """
@@ -807,7 +808,7 @@ def _write_cors_configuration(self, config):
         )
         resp.raise_for_status()
 
-        return resp.json()
+        return response_to_json_dict(resp)
 
     @classmethod
     def bluemix(cls, vcap_services, instance_name=None, service_name=None, **kwargs):

src/cloudant/database.py

@@ -26,7 +26,9 @@
     SEARCH_INDEX_ARGS,
     SPECIAL_INDEX_TYPE,
     TEXT_INDEX_TYPE,
-    get_docs)
+    get_docs,
+    response_to_json_dict,
+    )
 from .document import Document
 from .design_document import DesignDocument
 from .security_document import SecurityDocument
@@ -123,7 +125,7 @@ def metadata(self):
         """
         resp = self.r_session.get(self.database_url)
         resp.raise_for_status()
-        return resp.json()
+        return response_to_json_dict(resp)
 
     def doc_count(self):
         """
@@ -192,7 +194,7 @@ def design_documents(self):
         query = "startkey=\"_design\"&endkey=\"_design0\"&include_docs=true"
         resp = self.r_session.get(url, params=query)
         resp.raise_for_status()
-        data = resp.json()
+        data = response_to_json_dict(resp)
         return data['rows']
 
     def list_design_documents(self):
@@ -206,7 +208,7 @@ def list_design_documents(self):
         query = "startkey=\"_design\"&endkey=\"_design0\""
         resp = self.r_session.get(url, params=query)
         resp.raise_for_status()
-        data = resp.json()
+        data = response_to_json_dict(resp)
         return [x.get('key') for x in data.get('rows', [])]
 
     def get_design_document(self, ddoc_id):
@@ -403,7 +405,7 @@ def all_docs(self, **kwargs):
                         '/'.join([self.database_url, '_all_docs']),
                         self.client.encoder,
                         **kwargs)
-        return resp.json()
+        return response_to_json_dict(resp)
 
     @contextlib.contextmanager
     def custom_result(self, **options):
@@ -718,7 +720,7 @@ def bulk_docs(self, docs):
             headers=headers
         )
         resp.raise_for_status()
-        return resp.json()
+        return response_to_json_dict(resp)
 
     def missing_revisions(self, doc_id, *revisions):
         """
@@ -742,7 +744,7 @@ def missing_revisions(self, doc_id, *revisions):
         )
         resp.raise_for_status()
 
-        resp_json = resp.json()
+        resp_json = response_to_json_dict(resp)
         missing_revs = resp_json['missing_revs'].get(doc_id)
         if missing_revs is None:
             missing_revs = []
@@ -771,7 +773,7 @@ def revisions_diff(self, doc_id, *revisions):
         )
         resp.raise_for_status()
 
-        return resp.json()
+        return response_to_json_dict(resp)
 
     def get_revision_limit(self):
         """
@@ -787,7 +789,7 @@ def get_revision_limit(self):
         try:
             ret = int(resp.text)
         except ValueError:
-            raise CloudantDatabaseException(400, resp.json())
+            raise CloudantDatabaseException(400, response_to_json_dict(resp))
 
         return ret
 
@@ -806,7 +808,7 @@ def set_revision_limit(self, limit):
         resp = self.r_session.put(url, data=json.dumps(limit, cls=self.client.encoder))
         resp.raise_for_status()
 
-        return resp.json()
+        return response_to_json_dict(resp)
 
     def view_cleanup(self):
         """
@@ -822,7 +824,7 @@ def view_cleanup(self):
         )
         resp.raise_for_status()
 
-        return resp.json()
+        return response_to_json_dict(resp)
 
     def get_list_function_result(self, ddoc_id, list_name, view_name, **kwargs):
         """
@@ -974,10 +976,10 @@ def get_query_indexes(self, raw_result=False):
         resp.raise_for_status()
 
         if raw_result:
-            return resp.json()
+            return response_to_json_dict(resp)
 
         indexes = []
-        for data in resp.json().get('indexes', []):
+        for data in response_to_json_dict(resp).get('indexes', []):
             if data.get('type') == JSON_INDEX_TYPE:
                 indexes.append(Index(
                     self,
@@ -1239,7 +1241,7 @@ def share_database(self, username, roles=None):
             headers={'Content-Type': 'application/json'}
         )
         resp.raise_for_status()
-        return resp.json()
+        return response_to_json_dict(resp)
 
     def unshare_database(self, username):
         """
@@ -1264,7 +1266,7 @@ def unshare_database(self, username):
             headers={'Content-Type': 'application/json'}
         )
         resp.raise_for_status()
-        return resp.json()
+        return response_to_json_dict(resp)
 
     def shards(self):
         """
@@ -1276,7 +1278,7 @@ def shards(self):
         resp = self.r_session.get(url)
         resp.raise_for_status()
 
-        return resp.json()
+        return response_to_json_dict(resp)
 
     def get_search_result(self, ddoc_id, index_name, **query_params):
         """
@@ -1399,4 +1401,4 @@ def get_search_result(self, ddoc_id, index_name, **query_params):
             data=json.dumps(query_params, cls=self.client.encoder)
         )
         resp.raise_for_status()
-        return resp.json()
+        return response_to_json_dict(resp)

src/cloudant/design_document.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright (c) 2015 IBM. All rights reserved.
+# Copyright (C) 2015, 2018 IBM. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@
 API module/class for interacting with a design document in a database.
 """
 from ._2to3 import iteritems_, STRTYPE
-from ._common_util import QUERY_LANGUAGE, codify
+from ._common_util import QUERY_LANGUAGE, codify, response_to_json_dict
 from .document import Document
 from .view import View, QueryIndexView
 from .error import CloudantArgumentError, CloudantDesignDocumentException
@@ -686,7 +686,7 @@ def info(self):
         ddoc_info = self.r_session.get(
             '/'.join([self.document_url, '_info']))
         ddoc_info.raise_for_status()
-        return ddoc_info.json()
+        return response_to_json_dict(ddoc_info)
 
     def search_info(self, search_index):
         """
@@ -698,7 +698,7 @@ def search_info(self, search_index):
         ddoc_search_info = self.r_session.get(
             '/'.join([self.document_url, '_search_info', search_index]))
         ddoc_search_info.raise_for_status()
-        return ddoc_search_info.json()
+        return response_to_json_dict(ddoc_search_info)
 
     def search_disk_size(self, search_index):
         """
@@ -710,4 +710,4 @@ def search_disk_size(self, search_index):
         ddoc_search_disk_size = self.r_session.get(
             '/'.join([self.document_url, '_search_disk_size', search_index]))
         ddoc_search_disk_size.raise_for_status()
-        return ddoc_search_disk_size.json()
+        return response_to_json_dict(ddoc_search_disk_size)