feat: download datasets

Author: tillywoodfield

oc4ids_datastore_pipeline/pipeline.py

@@ -1,7 +1,45 @@
 import logging
+from typing import Any
+
+import requests
 
 logger = logging.getLogger(__name__)
 
+REGISTERED_DATASETS = {
+    "uganda_gpp": "https://gpp.ppda.go.ug/adminapi/public/api/open-data/v1/infrastructure/projects/download?format=json",  # noqa: E501
+    "ghana_cost_sekondi_takoradi": "https://costsekondi-takoradigh.org/uploads/projectJson.json",  # noqa: E501
+    "mexico_cost_jalisco": "http://www.costjalisco.org.mx/jsonprojects",
+    "mexico_nuevo_leon": "http://si.nl.gob.mx/siasi_ws/api/edcapi/DescargarProjectPackage",  # noqa: E501
+    "indonesia_cost_west_lombok": "https://intras.lombokbaratkab.go.id/oc4ids",
+    "ukraine_cost_ukraine": "https://portal.costukraine.org/data.json",
+    "malawi_cost_malawi": "https://ippi.mw/api/projects/query",
+}
+
+
+def download_json(url: str) -> Any:
+    logger.info(f"Downloading json from {url}")
+    try:
+        r = requests.get(url)
+        r.raise_for_status()
+        response_size = len(r.content)
+        logger.info(f"Downloaded {url} ({response_size} bytes)")
+        return r.json()
+    except Exception as e:
+        raise Exception("Download failed", e)
+
+
+def process_dataset(dataset_name: str, dataset_url: str) -> None:
+    logger.info(f"Processing dataset {dataset_name}")
+    try:
+        download_json(dataset_url)
+    except Exception as e:
+        logger.warning(f"Failed to process dataset {dataset_name} with error {e}")
+
+
+def process_datasets() -> None:
+    for name, url in REGISTERED_DATASETS.items():
+        process_dataset(name, url)
+
 
 def run() -> None:
-    logger.info("Hello World!")
+    process_datasets()

pyproject.toml

@@ -7,7 +7,9 @@ name = "oc4ids-datastore-pipeline"
 description = "OC4IDS Datastore Pipeline"
 version = "0.1.0"
 readme = "README.md"
-dependencies = []
+dependencies = [
+  "requests"
+]
 
 [project.optional-dependencies]
 dev = [
@@ -17,6 +19,8 @@ dev = [
   "Flake8-pyproject",
   "mypy",
   "pytest",
+  "pytest-mock",
+  "types-requests",
 ]
 
 [project.scripts]
@@ -30,3 +34,7 @@ max-line-length = 88
 
 [tool.mypy]
 strict = true
+
+[tool.pytest.ini_options]
+log_cli = true
+log_cli_level = "INFO"

requirements_dev.txt

@@ -6,6 +6,10 @@
 #
 black==25.1.0
     # via oc4ids-datastore-pipeline (pyproject.toml)
+certifi==2025.1.31
+    # via requests
+charset-normalizer==3.4.1
+    # via requests
 click==8.1.8
     # via black
 flake8==7.1.1
@@ -14,6 +18,8 @@ flake8==7.1.1
     #   oc4ids-datastore-pipeline (pyproject.toml)
 flake8-pyproject==1.2.3
     # via oc4ids-datastore-pipeline (pyproject.toml)
+idna==3.10
+    # via requests
 iniconfig==2.0.0
     # via pytest
 isort==6.0.0
@@ -41,6 +47,18 @@ pycodestyle==2.12.1
 pyflakes==3.2.0
     # via flake8
 pytest==8.3.4
+    # via
+    #   oc4ids-datastore-pipeline (pyproject.toml)
+    #   pytest-mock
+pytest-mock==3.14.0
+    # via oc4ids-datastore-pipeline (pyproject.toml)
+requests==2.32.3
+    # via oc4ids-datastore-pipeline (pyproject.toml)
+types-requests==2.32.0.20241016
     # via oc4ids-datastore-pipeline (pyproject.toml)
 typing-extensions==4.12.2
     # via mypy
+urllib3==2.3.0
+    # via
+    #   requests
+    #   types-requests

tests/test_pipeline.py

@@ -1,2 +1,24 @@
-def test_hello_world() -> None:
-    pass
+import pytest
+from pytest_mock import MockerFixture
+
+from oc4ids_datastore_pipeline.pipeline import download_json, process_dataset
+
+
+def test_download_json_raises_failure_exception(mocker: MockerFixture) -> None:
+    patch_get = mocker.patch("oc4ids_datastore_pipeline.pipeline.requests.get")
+    patch_get.side_effect = Exception("Mocked exception")
+
+    with pytest.raises(Exception) as exc_info:
+        download_json(url="https://test_dataset.json")
+
+    assert "Download failed" in str(exc_info.value)
+    assert "Mocked exception" in str(exc_info.value)
+
+
+def test_process_dataset_catches_exception(mocker: MockerFixture) -> None:
+    patch_download_json = mocker.patch(
+        "oc4ids_datastore_pipeline.pipeline.download_json"
+    )
+    patch_download_json.side_effect = Exception("Download failed")
+
+    process_dataset("test_dataset", "https://test_dataset.json")