Merge pull request #29 from uc-cdis/feat/add_urls_metadata_in_list_with_params

feat(url_key partial filter): include urls_metadata in list_with_param

Author: khan08

indexclient/client.py

@@ -1,11 +1,11 @@
-import json
 try:
     from urlparse import urljoin
 except ImportError:
     from urllib.parse import urljoin
 
 import requests
 import copy
+import json
 
 UPDATABLE_ATTRS = [
     'file_name', 'urls', 'version',
@@ -58,7 +58,7 @@ def global_get(self, did, no_dist=False):
         """
         try:
             if no_dist:
-                response = self._get(did, params={'no_dist':''})
+                response = self._get(did, params={'no_dist': ''})
             else:
                 response = self._get(did)
         except requests.HTTPError as e:
@@ -126,14 +126,20 @@ def list(self, limit=float("inf"), start=None, page_size=100):
         """ Returns a generator of document objects. """
         return self.list_with_params(limit, start, page_size)
 
-    def list_with_params(self, limit=float("inf"), start=None, page_size=100, params=None):
+    def list_with_params(self, limit=float("inf"), start=None, page_size=100, params=None, negate_params=None):
         """
         Return a generator of document object corresponding to the supplied parameters, such
-        as ``{'hashes': {'md5': '...'}, 'size': '...', 'metadata': {'file_state': '...'}}``.
+        as ``{'hashes': {'md5': '...'},
+              'size': '...',
+              'metadata': {'file_state': '...'},
+              'urls_metadata': {'s3://url': {'state': '...'}
+             }``.
         """
         params_copy = copy.deepcopy(params) or {}
         if 'hashes' in params_copy:
             params_copy['hash'] = params_copy.pop('hashes')
+        if 'urls_metadata' in params_copy:
+            params_copy['urls_metadata'] = json.dumps(params_copy.pop('urls_metadata'))
         reformatted_params = dict()
         for param in ['hash', 'metadata']:
             if param in params_copy:
@@ -143,21 +149,23 @@ def list_with_params(self, limit=float("inf"), start=None, page_size=100, params
                 del params_copy[param]
         reformatted_params.update(params_copy)
         reformatted_params.update({"limit": page_size, "start": start})
+        if negate_params:
+            reformatted_params.update({"negate_params": json.dumps(negate_params)})
         yielded = 0
         while True:
             resp = self._get("index", params=reformatted_params)
             handle_error(resp)
-            json = resp.json()
-            if not json["records"]:
+            json_str = resp.json()
+            if not json_str["records"]:
                 return
-            for doc in json["records"]:
+            for doc in json_str["records"]:
                 if yielded < limit:
                     yield Document(self, None, json=doc)
                     yielded += 1
                 else:
                     return
-            if len(json['records']) == page_size:
-                reformatted_params['start'] = json['records'][-1]['did']
+            if len(json_str['records']) == page_size:
+                reformatted_params['start'] = json_str['records'][-1]['did']
             else:
                 # There's no more results
                 return

test-requirements.txt

@@ -2,7 +2,7 @@ pytest==3.0.6
 mock
 -e git+https://github.com/uc-cdis/cdisutils-test.git@0.2.3#egg=cdisutilstest
 -e git+https://github.com/uc-cdis/cdis-python-utils.git@0.1.7#egg=cdispyutils
--e git+https://github.com/uc-cdis/indexd.git@1.0.0#egg=indexd
+-e git+https://github.com/uc-cdis/indexd.git@1.0.3#egg=indexd 
 -e git+https://github.com/uc-cdis/doiclient.git@1.0.0#egg=doiclient
 -e git+https://github.com/uc-cdis/dosclient.git@1.0.0#egg=dosclient
 -e git+https://github.com/uc-cdis/cdislogging.git@0.0.2#egg=cdislogging

tests/test_client.py

@@ -60,6 +60,22 @@ def test_list_with_params(index_client):
     assert set(dids) == set(found)
 
 
+def test_list_with_params_negate(index_client):
+    doc1 = create_random_index(index_client, version='1'
+                               )
+    create_random_index(index_client, version='2')
+
+    docs = index_client.list_with_params(
+        negate_params={'version': '2'}
+    )
+
+    dids = {record.did for record in docs}
+    assert dids == {doc1.did}
+
+
+
+
+
 def test_get_latest_version(index_client):
     """
     Args: