feat: fetch license mappings from registry

Author: tillywoodfield

oc4ids_datastore_pipeline/pipeline.py

@@ -30,6 +30,25 @@ def fetch_registered_datasets() -> dict[str, str]:
         raise Exception("Failed to fetch datasets list from registry", e)
 
 
+def fetch_license_mappings() -> dict[str, str]:
+    logger.info("Fetching license mappings from registry")
+    try:
+        url = "https://opendataservices.github.io/oc4ids-registry/datatig/type/license/records_api.json"  # noqa: E501
+        r = requests.get(url)
+        r.raise_for_status()
+        json_data = r.json()
+        return {
+            urls["fields"]["url"]["value"]: license["fields"]["title"]["value"]
+            for license in json_data["records"].values()
+            for urls in license["fields"]["urls"]["values"]
+        }
+    except Exception as e:
+        logger.warning(
+            "Failed to fetch license mappings from registry, with error: " + str(e),
+        )
+        return {}
+
+
 def download_json(url: str) -> Any:
     logger.info(f"Downloading json from {url}")
     try:

tests/test_pipeline.py

@@ -8,6 +8,7 @@
 
 from oc4ids_datastore_pipeline.pipeline import (
     download_json,
+    fetch_license_mappings,
     fetch_registered_datasets,
     process_dataset,
     validate_json,
@@ -43,6 +44,70 @@ def test_fetch_registered_datasets_raises_failure_exception(
     assert "Mocked exception" in str(exc_info.value)
 
 
+def test_fetch_license_mappings(mocker: MockerFixture) -> None:
+    mock_response = MagicMock()
+    mock_response.json.return_value = {
+        "records": {
+            "license_1": {
+                "fields": {
+                    "title": {"value": "License 1"},
+                    "urls": {
+                        "values": [
+                            {
+                                "fields": {
+                                    "url": {"value": "https://license_1.com/license"}
+                                }
+                            },
+                            {
+                                "fields": {
+                                    "url": {
+                                        "value": "https://license_1.com/different_url"
+                                    }
+                                }
+                            },
+                        ]
+                    },
+                }
+            },
+            "license_2": {
+                "fields": {
+                    "title": {"value": "License 2"},
+                    "urls": {
+                        "values": [
+                            {
+                                "fields": {
+                                    "url": {"value": "https://license_2.com/license"}
+                                }
+                            },
+                        ]
+                    },
+                }
+            },
+        }
+    }
+    patch_get = mocker.patch("oc4ids_datastore_pipeline.pipeline.requests.get")
+    patch_get.return_value = mock_response
+
+    result = fetch_license_mappings()
+
+    assert result == {
+        "https://license_1.com/license": "License 1",
+        "https://license_1.com/different_url": "License 1",
+        "https://license_2.com/license": "License 2",
+    }
+
+
+def test_fetch_license_mappings_catches_exception(
+    mocker: MockerFixture,
+) -> None:
+    patch_get = mocker.patch("oc4ids_datastore_pipeline.pipeline.requests.get")
+    patch_get.side_effect = Exception("Mocked exception")
+
+    result = fetch_license_mappings()
+
+    assert result == {}
+
+
 def test_download_json_raises_failure_exception(mocker: MockerFixture) -> None:
     patch_get = mocker.patch("oc4ids_datastore_pipeline.pipeline.requests.get")
     patch_get.side_effect = Exception("Mocked exception")