feat: save license metadata to database

Author: tillywoodfield

migrations/versions/85905d23accc_add_license_columns_to_dataset_table.py

@@ -0,0 +1,33 @@
+"""add license columns to dataset table
+
+Revision ID: 85905d23accc
+Revises: aaabf849b37f
+Create Date: 2025-02-05 09:45:04.056529
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "85905d23accc"
+down_revision: Union[str, None] = "aaabf849b37f"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("dataset", sa.Column("license_url", sa.String(), nullable=True))
+    op.add_column("dataset", sa.Column("license_name", sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("dataset", "license_name")
+    op.drop_column("dataset", "license_url")
+    # ### end Alembic commands ###

oc4ids_datastore_pipeline/database.py

@@ -1,6 +1,7 @@
 import datetime
 import logging
 import os
+from typing import Optional
 
 from sqlalchemy import (
     DateTime,
@@ -25,6 +26,8 @@ class Dataset(Base):
     dataset_id: Mapped[str] = mapped_column(String, primary_key=True)
     source_url: Mapped[str] = mapped_column(String)
     publisher_name: Mapped[str] = mapped_column(String)
+    license_url: Mapped[Optional[str]] = mapped_column(String, nullable=True)
+    license_name: Mapped[Optional[str]] = mapped_column(String, nullable=True)
     json_url: Mapped[str] = mapped_column(String)
     updated_at: Mapped[datetime.datetime] = mapped_column(DateTime(timezone=True))
 

oc4ids_datastore_pipeline/pipeline.py

@@ -8,7 +8,10 @@
 from libcoveoc4ids.api import oc4ids_json_output
 
 from oc4ids_datastore_pipeline.database import Dataset, save_dataset
-from oc4ids_datastore_pipeline.registry import fetch_registered_datasets
+from oc4ids_datastore_pipeline.registry import (
+    fetch_registered_datasets,
+    get_license_name_from_url,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -54,10 +57,14 @@ def save_dataset_metadata(
 ) -> None:
     logger.info(f"Saving metadata for dataset {dataset_name}")
     publisher_name = json_data.get("publisher", {}).get("name", "")
+    license_url = json_data.get("license", None)
+    license_name = get_license_name_from_url(license_url) if license_url else None
     dataset = Dataset(
         dataset_id=dataset_name,
         source_url=source_url,
         publisher_name=publisher_name,
+        license_url=license_url,
+        license_name=license_name,
         json_url=json_url,
         updated_at=datetime.datetime.now(datetime.UTC),
     )

oc4ids_datastore_pipeline/registry.py

@@ -1,10 +1,14 @@
 import logging
+from typing import Optional
 
 import requests
 
 logger = logging.getLogger(__name__)
 
 
+_license_mappings = None
+
+
 def fetch_registered_datasets() -> dict[str, str]:
     logger.info("Fetching registered datasets list from registry")
     try:
@@ -40,3 +44,12 @@ def fetch_license_mappings() -> dict[str, str]:
             "Failed to fetch license mappings from registry, with error: " + str(e),
         )
         return {}
+
+
+def get_license_name_from_url(
+    url: str, force_refresh: Optional[bool] = False
+) -> Optional[str]:
+    global _license_mappings
+    if force_refresh or (_license_mappings is None):
+        _license_mappings = fetch_license_mappings()
+    return _license_mappings.get(url, None)

tests/test_registry.py

@@ -6,6 +6,7 @@
 from oc4ids_datastore_pipeline.registry import (
     fetch_license_mappings,
     fetch_registered_datasets,
+    get_license_name_from_url,
 )
 
 
@@ -99,3 +100,34 @@ def test_fetch_license_mappings_catches_exception(
     result = fetch_license_mappings()
 
     assert result == {}
+
+
+def test_get_license_name_from_url(mocker: MockerFixture) -> None:
+    patch_license_mappings = mocker.patch(
+        "oc4ids_datastore_pipeline.registry.fetch_license_mappings"
+    )
+    patch_license_mappings.return_value = {
+        "https://license_1.com/license": "License 1",
+        "https://license_2.com/license": "License 2",
+    }
+
+    license_name = get_license_name_from_url(
+        "https://license_2.com/license", force_refresh=True
+    )
+
+    assert license_name == "License 2"
+
+
+def test_get_license_name_from_url_not_in_mapping(mocker: MockerFixture) -> None:
+    patch_license_mappings = mocker.patch(
+        "oc4ids_datastore_pipeline.registry.fetch_license_mappings"
+    )
+    patch_license_mappings.return_value = {
+        "https://license_1.com/license": "License 1",
+    }
+
+    license_name = get_license_name_from_url(
+        "https://license_2.com/license", force_refresh=True
+    )
+
+    assert license_name is None