oc4ids_datastore_pipeline/registry.py: Change to getting individual data records

This means we can get all the dataset fields datatig has,
previously we could only get fields in list_fields in the datatig config

Author: jarofgreen

oc4ids_datastore_pipeline/registry.py

@@ -16,13 +16,15 @@ def fetch_registered_datasets() -> dict[str, dict[str, str]]:
         r = requests.get(url)
         r.raise_for_status()
         json_data = r.json()
-        registered_datasets = {
-            key: {
-                "source_url": value["fields"]["url"]["value"],
-                "country": value["fields"]["country"]["value"],
+        registered_datasets = {}
+        for key, value in json_data["records"].items():
+            r_data = requests.get(value["api_url"])
+            r_data.raise_for_status()
+            r_data_json = r_data.json()
+            registered_datasets[key] = {
+                "source_url": r_data_json["fields"]["url"]["value"],
+                "country": r_data_json["fields"]["country"]["value"],
             }
-            for (key, value) in json_data["records"].items()
-        }
         registered_datasets_count = len(registered_datasets)
         logger.info(f"Fetched URLs for {registered_datasets_count} datasets")
     except Exception as e:

tests/test_registry.py

@@ -12,16 +12,25 @@
 
 def test_fetch_registered_datasets(mocker: MockerFixture) -> None:
     mock_response = MagicMock()
-    mock_response.json.return_value = {
-        "records": {
-            "test_dataset": {
-                "fields": {
-                    "url": {"value": "https://test_dataset.json"},
-                    "country": {"value": "ab"},
+    mock_response.json.side_effect = [
+        {
+            "records": {
+                "test_dataset": {
+                    "api_url": "http://www.example.com",
+                    "fields": {
+                        "url": {"value": "https://test_dataset.json"},
+                        "country": {"value": "ab"},
+                    },
                 }
             }
-        }
-    }
+        },
+        {
+            "fields": {
+                "url": {"value": "https://test_dataset.json"},
+                "country": {"value": "ab"},
+            }
+        },
+    ]
     patch_get = mocker.patch("oc4ids_datastore_pipeline.pipeline.requests.get")
     patch_get.return_value = mock_response