Relax dataset filter matching

Relax the optional search filter in `Dataset.get_datasets` to allow
matches anywhere in the dataset name (instead of only at the
beginning). In addition, also search the dataset's ID for matches.

Author: simenheg

CHANGELOG.md

@@ -1,3 +1,9 @@
+## ?.?.?
+
+* The optional search filter in `Dataset.get_datasets` has been relaxed to allow
+  matches anywhere in the dataset name (instead of only at the beginning). In
+  addition it now also searches the dataset's ID.
+
 ## 2.4.0
 
 * Added support for Python 3.11.

okdata/sdk/data/dataset.py

@@ -19,19 +19,19 @@ def create_dataset(self, data=None, retries=0):
         log.info(f"Created dataset: {body['Id']}")
         return body
 
+    def _matches(self, dataset, pattern):
+        """Return true if `dataset`'s ID or name matches `pattern`."""
+        return re.search(pattern, dataset["Id"], re.IGNORECASE) or (
+            "title" in dataset and re.search(pattern, dataset["title"], re.IGNORECASE)
+        )
+
     def get_datasets(self, filter=None, retries=0):
         url = self.config.get("datasetUrl")
         log.info(f"SDK:Get datasets from: {url}")
-        result = self.get(url, retries=retries)
-        ret = result.json()
-        if filter is not None:
-            if isinstance(filter, str):
-                tmp = []
-                for el in ret:
-                    if "title" in el and re.match(filter, el["title"], re.IGNORECASE):
-                        tmp.append(el)
-                ret = tmp
-        return ret
+        datasets = self.get(url, retries=retries).json()
+        if isinstance(filter, str):
+            return [d for d in datasets if self._matches(d, filter)]
+        return datasets
 
     def get_dataset(self, datasetid, retries=0):
         datasetUrl = self.config.get("datasetUrl")

tests/data/dataset_test.py

@@ -1,11 +1,12 @@
-import re
 import json
+import re
+
 import pytest
 from requests.exceptions import HTTPError
 
-from okdata.sdk.data.dataset import Dataset
 from okdata.sdk.auth.auth import Authenticate
 from okdata.sdk.config import Config
+from okdata.sdk.data.dataset import Dataset
 from okdata.sdk.file_cache import FileCache
 
 config = Config()
@@ -30,33 +31,42 @@ def test_sdk_no_auth_headers(self):
 
         assert ds.headers() == {}
 
-    def test_getDatasets(self, requests_mock):
+    def test_get_datasets(self, requests_mock):
         ds = Dataset(config=config, auth=auth_default)
-        response = json.dumps([{"Id": "test-get-dataset"}])
+        response = json.dumps([{"Id": "test-get-datasets"}])
         matcher = re.compile("datasets")
         requests_mock.register_uri("GET", matcher, text=response, status_code=200)
-        list = ds.get_datasets()
-        assert list[0]["Id"] == "test-get-dataset"
+        res = ds.get_datasets()
+        assert [d["Id"] for d in res] == ["test-get-datasets"]
 
-    def test_getDatasets_filter_no_result(self, requests_mock):
+    def test_get_datasets_filter_no_results(self, requests_mock):
         ds = Dataset(config=config, auth=auth_default)
         response = json.dumps(
             [{"Id": "foo-bar", "title": "deichman", "publisher": "someone"}]
         )
         matcher = re.compile("datasets")
         requests_mock.register_uri("GET", matcher, text=response, status_code=200)
-        list = ds.get_datasets("eide")
-        assert len(list) == 0
+        assert ds.get_datasets("eide") == []
 
-    def test_getDatasets_filter(self, requests_mock):
+    def test_get_datasets_filter_by_id(self, requests_mock):
         ds = Dataset(config=config, auth=auth_default)
         response = json.dumps(
             [{"Id": "foo-bar", "title": "eide"}, {"Id": "foo-bar2", "title": "someone"}]
         )
         matcher = re.compile("datasets")
         requests_mock.register_uri("GET", matcher, text=response, status_code=200)
-        list = ds.get_datasets("eide")
-        assert len(list) == 1
+        res = ds.get_datasets("bar2")
+        assert [d["Id"] for d in res] == ["foo-bar2"]
+
+    def test_get_datasets_filter_by_title(self, requests_mock):
+        ds = Dataset(config=config, auth=auth_default)
+        response = json.dumps(
+            [{"Id": "foo-bar", "title": "eide"}, {"Id": "foo-bar2", "title": "someone"}]
+        )
+        matcher = re.compile("datasets")
+        requests_mock.register_uri("GET", matcher, text=response, status_code=200)
+        res = ds.get_datasets("eide")
+        assert [d["Id"] for d in res] == ["foo-bar"]
 
     def test_getDataset(self, requests_mock):
         ds = Dataset(config=config, auth=auth_default)