feat: save license metadata to database

Author: tillywoodfield

migrations/versions/00238f37d113_add_license_columns_to_dataset_table.py

@@ -0,0 +1,33 @@
+"""add license columns to dataset table
+
+Revision ID: 00238f37d113
+Revises: aaabf849b37f
+Create Date: 2025-02-04 17:49:14.122704
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "00238f37d113"
+down_revision: Union[str, None] = "aaabf849b37f"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("dataset", sa.Column("license_url", sa.String(), nullable=False))
+    op.add_column("dataset", sa.Column("license_name", sa.String(), nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("dataset", "license_name")
+    op.drop_column("dataset", "license_url")
+    # ### end Alembic commands ###

oc4ids_datastore_pipeline/database.py

@@ -25,6 +25,8 @@ class Dataset(Base):
     dataset_id: Mapped[str] = mapped_column(String, primary_key=True)
     source_url: Mapped[str] = mapped_column(String)
     publisher_name: Mapped[str] = mapped_column(String)
+    license_url: Mapped[str] = mapped_column(String)
+    license_name: Mapped[str] = mapped_column(String)
     json_url: Mapped[str] = mapped_column(String)
     updated_at: Mapped[datetime.datetime] = mapped_column(DateTime(timezone=True))
 

oc4ids_datastore_pipeline/pipeline.py

@@ -8,7 +8,10 @@
 from libcoveoc4ids.api import oc4ids_json_output
 
 from oc4ids_datastore_pipeline.database import Dataset, save_dataset
-from oc4ids_datastore_pipeline.registry import fetch_registered_datasets
+from oc4ids_datastore_pipeline.registry import (
+    fetch_registered_datasets,
+    get_license_name_from_url,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -54,10 +57,14 @@ def save_dataset_metadata(
 ) -> None:
     logger.info(f"Saving metadata for dataset {dataset_name}")
     publisher_name = json_data.get("publisher", {}).get("name", "")
+    license_url = json_data.get("license", "")
+    license_name = get_license_name_from_url(license_url)
     dataset = Dataset(
         dataset_id=dataset_name,
         source_url=source_url,
         publisher_name=publisher_name,
+        license_url=license_url,
+        license_name=license_name,
         json_url=json_url,
         updated_at=datetime.datetime.now(datetime.UTC),
     )

oc4ids_datastore_pipeline/registry.py

@@ -1,3 +1,4 @@
+from functools import lru_cache
 import logging
 
 import requests
@@ -23,6 +24,7 @@ def fetch_registered_datasets() -> dict[str, str]:
         raise Exception("Failed to fetch datasets list from registry", e)
 
 
+@lru_cache(maxsize=None)
 def fetch_license_mappings() -> dict[str, str]:
     logger.info("Fetching license mappings from registry")
     try:
@@ -40,3 +42,8 @@ def fetch_license_mappings() -> dict[str, str]:
             "Failed to fetch license mappings from registry, with error: " + str(e),
         )
         return {}
+
+
+def get_license_name_from_url(url: str) -> str:
+    license_mappings = fetch_license_mappings()
+    return license_mappings.get(url, "")

tests/test_registry.py

@@ -99,3 +99,7 @@ def test_fetch_license_mappings_catches_exception(
     result = fetch_license_mappings()
 
     assert result == {}
+
+
+def test_get_license_name_from_url() -> None:
+    pass  # TODO: Implement