feat: 1343 populate license tables (#1445)

qcdyx · web-flow · commit 9f8d57d9d514 · 2025-11-10T11:29:52.000-05:00
populate license, rules and license_rules tables with spdx data
diff --git a/functions-python/tasks_executor/README.md b/functions-python/tasks_executor/README.md
@@ -78,3 +78,14 @@ To populate license rules:
   }
 }
 ```
+
+To populate licenses:
+
+```json
+{
+  "task": "populate_licenses",
+  "payload": {
+    "dry_run": true
+  }
+}
+```
diff --git a/functions-python/tasks_executor/src/main.py b/functions-python/tasks_executor/src/main.py
@@ -44,6 +44,10 @@
     populate_license_rules_handler,
 )
 
+from tasks.licenses.populate_licenses import (
+    populate_licenses_handler,
+)
+
 init_logger()
 LIST_COMMAND: Final[str] = "list"
 tasks = {
@@ -90,6 +94,10 @@
         "description": "Populates license rules in the database from a predefined JSON source.",
         "handler": populate_license_rules_handler,
     },
+    "populate_licenses": {
+        "description": "Populates licenses and license-rules in the database from a predefined JSON source.",
+        "handler": populate_licenses_handler,
+    },
 }
 
 
diff --git a/functions-python/tasks_executor/src/tasks/licenses/populate_license_rules.py b/functions-python/tasks_executor/src/tasks/licenses/populate_license_rules.py
@@ -1,3 +1,20 @@
+# This script defines a task to populate the 'rules' table in the database from a canonical
+# JSON file. It is designed to be triggered as a background task.
+#
+# The script performs the following steps:
+# 1. Fetches the 'rules.json' file from the MobilityData/licenses-aas GitHub repository.
+# 2. The JSON file categorizes rules into 'permissions', 'conditions', and 'limitations'.
+# 3. It processes this structure by:
+#    a. Iterating through each category.
+#    b. Mapping the plural category name (e.g., 'permissions') to a singular 'type'
+#       (e.g., 'permission') to satisfy a database check constraint on the Rule model.
+#    c. Combining all rules from all categories into a single list.
+# 4. For each rule in the combined list, it performs an "upsert" operation (insert or update)
+#    into the 'rules' table using SQLAlchemy's `merge` method, with the rule's 'name'
+#    acting as the primary key.
+# 5. Supports a 'dry_run' mode, which simulates the process and logs intended actions
+#    without committing any changes to the database.
+# 6. Includes error handling for network issues and database transactions.
 import logging
 
 import requests
diff --git a/functions-python/tasks_executor/src/tasks/licenses/populate_licenses.py b/functions-python/tasks_executor/src/tasks/licenses/populate_licenses.py
@@ -0,0 +1,155 @@
+# This script defines a task to populate the 'licenses' table and the 'license_rules'
+# association table in the database. It is designed to be triggered as a background task.
+#
+# The script performs the following steps:
+# 1. Fetches a list of license definition files from the MobilityData/licenses-aas GitHub repository
+#    using the GitHub API.
+# 2. Downloads the JSON content of each individual license file.
+# 3. For each license:
+#    a. Parses the JSON, extracting license details primarily from the nested 'spdx' object.
+#       This includes the license ID, name, URL, and full license text.
+#    b. Creates a new 'License' record or updates an existing one based on the license ID.
+#    c. Extracts the associated rule names from the 'permissions', 'conditions', and 'limitations'
+#       lists at the top level of the JSON.
+#    d. Queries the 'rules' table to find the corresponding Rule objects.
+#    e. Associates the found rules with the license. The SQLAlchemy ORM automatically
+#       manages the creation of records in the 'license_rules' join table to establish
+#       the many-to-many relationship.
+# 4. Supports a 'dry_run' mode, which simulates the process and logs intended
+#    actions without committing any changes to the database.
+# 5. Includes error handling for network issues and database transactions.
+import logging
+from datetime import datetime, timezone
+
+import requests
+from shared.database.database import with_db_session
+from shared.database_gen.sqlacodegen_models import License, Rule
+
+LICENSES_API_URL = (
+    "https://api.github.com/repos/MobilityData/licenses-aas/contents/data/licenses"
+)
+
+
+def populate_licenses_handler(payload):
+    """
+    Handler for populating licenses.
+
+    Args:
+        payload (dict): Incoming payload data.
+    """
+    dry_run = get_parameters(payload)
+    return populate_licenses_task(dry_run)
+
+
+@with_db_session
+def populate_licenses_task(dry_run, db_session):
+    """
+    Populates licenses and their associated rules in the database.
+
+    Args:
+        dry_run (bool): If True, simulates the operation without making changes.
+        db_session: Database session for executing queries.
+    """
+    logging.info("Starting populate_licenses_task with dry_run=%s", dry_run)
+
+    try:
+        logging.info("Downloading license list from %s", LICENSES_API_URL)
+        response = requests.get(LICENSES_API_URL, timeout=10)
+        response.raise_for_status()
+        files = response.json()
+
+        licenses_data = []
+        for file_info in files:
+            if file_info["type"] == "file" and file_info["name"].endswith(".json"):
+                download_url = file_info["download_url"]
+                license_response = requests.get(download_url, timeout=10)
+                license_response.raise_for_status()
+                licenses_data.append(license_response.json())
+
+        logging.info("Loaded %d licenses.", len(licenses_data))
+
+        if dry_run:
+            logging.info("Dry run: would process %d licenses.", len(licenses_data))
+        else:
+            for license_data in licenses_data:
+                spdx_data = license_data.get("spdx")
+                if not spdx_data:
+                    logging.warning("Skipping license record without 'spdx' data")
+                    continue
+                else:
+                    is_spdx = True
+                license_id = spdx_data.get("licenseId")
+                if not license_id:
+                    logging.warning("Skipping record without licenseId.")
+                    continue
+
+                logging.info("Processing license %s", license_id)
+
+                license_object = db_session.get(License, license_id)
+                if not license_object:
+                    license_object = License(id=license_id)
+                    license_object.created_at = datetime.now(timezone.utc)
+                    license_object.is_spdx = is_spdx
+                    license_object.type = "standard"
+                    # Add the new license object to the session immediately.
+                    # This makes it "pending" and allows SQLAlchemy to track relationship changes.
+                    db_session.add(license_object)
+                license_object.name = spdx_data.get("name")
+                license_object.updated_at = datetime.now(timezone.utc)
+                cross_ref_list = spdx_data.get("crossRef")
+                if (
+                    cross_ref_list
+                    and isinstance(cross_ref_list, list)
+                    and cross_ref_list
+                ):
+                    license_object.url = cross_ref_list[0].get("url")
+                else:
+                    license_object.url = None
+
+                license_object.content_txt = spdx_data.get("licenseText")
+                license_object.content_html = spdx_data.get("licenseTextHtml")
+
+                # Clear existing rules to handle updates
+                license_object.rules = []
+
+                all_rule_names = []
+                for rule_type in ["permissions", "conditions", "limitations"]:
+                    all_rule_names.extend(license_data.get(rule_type, []))
+
+                if all_rule_names:
+                    rules = (
+                        db_session.query(Rule)
+                        .filter(Rule.name.in_(all_rule_names))
+                        .all()
+                    )
+                    license_object.rules.extend(rules)
+                    if len(rules) != len(all_rule_names):
+                        logging.warning(
+                            "License '%s': Found %d of %d rules in the database.",
+                            license_id,
+                            len(rules),
+                            len(all_rule_names),
+                        )
+                # Merge the license object into the session. This handles both creating new licenses
+                # and updating existing ones (upsert), including their rule associations.
+                db_session.merge(license_object)
+
+            logging.info(
+                "Successfully upserted licenses into the database.",
+            )
+
+    except requests.exceptions.RequestException as e:
+        logging.error("Failed to download licenses JSON file: %s", e)
+        raise
+
+
+def get_parameters(payload):
+    """
+    Get parameters from the payload.
+
+    Args:
+        payload (dict): dictionary containing the payload data.
+    Returns:
+        bool: dry_run
+    """
+    return payload.get("dry_run", False)
diff --git a/functions-python/tasks_executor/tests/tasks/populate_licenses_and_rules/test_populate_license_rules.py b/functions-python/tasks_executor/tests/tasks/populate_licenses_and_rules/test_populate_license_rules.py
diff --git a/functions-python/tasks_executor/tests/tasks/populate_licenses_and_rules/test_populate_licenses.py b/functions-python/tasks_executor/tests/tasks/populate_licenses_and_rules/test_populate_licenses.py
@@ -0,0 +1,167 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+import requests
+
+from shared.database_gen.sqlacodegen_models import Rule
+from tasks.licenses.populate_licenses import (
+    LICENSES_API_URL,
+    populate_licenses_task,
+)
+
+# Mock data for GitHub API responses
+MOCK_LICENSE_LIST = [
+    {
+        "name": "MIT.json",
+        "type": "file",
+        "download_url": "http://mockurl/MIT.json",
+    },
+    {
+        "name": "BSD-3-Clause.json",
+        "type": "file",
+        "download_url": "http://mockurl/BSD-3-Clause.json",
+    },
+    {
+        "name": "no-spdx.json",
+        "type": "file",
+        "download_url": "http://mockurl/no-spdx.json",
+    },
+    {
+        "name": "README.md",
+        "type": "file",
+        "download_url": "http://mockurl/README.md",
+    },
+]
+
+MOCK_LICENSE_MIT = {
+    "spdx": {
+        "licenseId": "MIT",
+        "name": "MIT License",
+        "crossRef": [{"url": "https://opensource.org/licenses/MIT"}],
+        "licenseText": "MIT License text...",
+        "licenseTextHtml": "<p>MIT License text...</p>",
+    },
+    "permissions": ["commercial-use", "distribution"],
+    "conditions": ["include-copyright"],
+    "limitations": [],
+}
+
+MOCK_LICENSE_BSD = {
+    "spdx": {
+        "licenseId": "BSD-3-Clause",
+        "name": "BSD 3-Clause License",
+        "crossRef": [{"url": "https://opensource.org/licenses/BSD-3-Clause"}],
+        "licenseText": "BSD license text...",
+        "licenseTextHtml": "<p>BSD license text...</p>",
+    },
+    "permissions": ["commercial-use"],
+    "conditions": [],
+    "limitations": ["liability", "warranty"],
+}
+
+MOCK_LICENSE_NO_SPDX = {"licenseId": "NO-SPDX-ID", "name": "No SPDX License"}
+
+
+class TestPopulateLicenses(unittest.TestCase):
+    def _mock_requests_get(self, mock_get):
+        """Helper to configure mock for requests.get."""
+        mock_responses = {
+            LICENSES_API_URL: MagicMock(json=lambda: MOCK_LICENSE_LIST),
+            "http://mockurl/MIT.json": MagicMock(json=lambda: MOCK_LICENSE_MIT),
+            "http://mockurl/BSD-3-Clause.json": MagicMock(
+                json=lambda: MOCK_LICENSE_BSD
+            ),
+            "http://mockurl/no-spdx.json": MagicMock(json=lambda: MOCK_LICENSE_NO_SPDX),
+        }
+
+        def get_side_effect(url, timeout=None):
+            if url in mock_responses:
+                response = mock_responses[url]
+                response.raise_for_status.return_value = None
+                return response
+            raise requests.exceptions.RequestException(f"URL not mocked: {url}")
+
+        mock_get.side_effect = get_side_effect
+
+    @patch("tasks.licenses.populate_licenses.requests.get")
+    def test_populate_licenses_success(self, mock_get):
+        """Test successful population of licenses."""
+        # Arrange
+        self._mock_requests_get(mock_get)
+        mock_db_session = MagicMock()
+        mock_db_session.get.return_value = None  # Simulate no existing licenses
+
+        # Mock the rules query to return only the rules that are requested.
+        all_mock_rules = {
+            "commercial-use": Rule(name="commercial-use"),
+            "distribution": Rule(name="distribution"),
+            "include-copyright": Rule(name="include-copyright"),
+            "liability": Rule(name="liability"),
+            "warranty": Rule(name="warranty"),
+        }
+
+        def filter_side_effect(filter_condition):
+            # This simulates the `Rule.name.in_(...)` filter by inspecting the
+            # requested names from the filter condition's right-hand side.
+            requested_names = filter_condition.right.value
+            mock_query_result = [
+                all_mock_rules[name]
+                for name in requested_names
+                if name in all_mock_rules
+            ]
+            mock_filter = MagicMock()
+            mock_filter.all.return_value = mock_query_result
+            return mock_filter
+
+        mock_db_session.query.return_value.filter.side_effect = filter_side_effect
+
+        # Act
+        populate_licenses_task(dry_run=False, db_session=mock_db_session)
+
+        # Assert
+        self.assertEqual(mock_db_session.merge.call_count, 2)
+        mock_db_session.rollback.assert_not_called()
+
+        # Check that merge was called with correctly constructed License objects
+        call_args_list = mock_db_session.merge.call_args_list
+        merged_licenses = [arg.args[0] for arg in call_args_list]
+
+        mit_license = next((lic for lic in merged_licenses if lic.id == "MIT"), None)
+        self.assertIsNotNone(mit_license)
+        self.assertEqual(mit_license.name, "MIT License")
+        self.assertTrue(mit_license.is_spdx)
+        self.assertEqual(len(mit_license.rules), 3)
+
+    @patch("tasks.licenses.populate_licenses.requests.get")
+    def test_populate_licenses_dry_run(self, mock_get):
+        """Test that no database changes are made during a dry run."""
+        # Arrange
+        self._mock_requests_get(mock_get)
+        mock_db_session = MagicMock()
+
+        # Act
+        populate_licenses_task(dry_run=True, db_session=mock_db_session)
+
+        # Assert
+        mock_db_session.get.assert_not_called()
+        mock_db_session.merge.assert_not_called()
+        mock_db_session.rollback.assert_not_called()
+
+    @patch("tasks.licenses.populate_licenses.requests.get")
+    def test_request_exception_handling(self, mock_get):
+        """Test handling of a requests exception."""
+        # Arrange
+        mock_get.side_effect = requests.exceptions.RequestException("Network Error")
+        mock_db_session = MagicMock()
+
+        # Act & Assert
+        with self.assertRaises(requests.exceptions.RequestException):
+            populate_licenses_task(dry_run=False, db_session=mock_db_session)
+
+        mock_db_session.merge.assert_not_called()
+        # Rollback is not called because the exception happens before the db try/except block
+        mock_db_session.rollback.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()
