feat: 1343 populate spdx license tables from licenses as project (#1425)

* added license, rules, license-rules tables and populate rules table

Author: qcdyx

functions-python/tasks_executor/README.md

@@ -17,48 +17,64 @@ Examples:
 
 ```json
 {
-   "task": "rebuild_missing_validation_reports",
-   "payload": {
-      "dry_run": true,
-      "filter_after_in_days": 14,
-      "filter_statuses": ["active", "inactive", "future"]
-   }
+  "task": "rebuild_missing_validation_reports",
+  "payload": {
+    "dry_run": true,
+    "filter_after_in_days": 14,
+    "filter_statuses": ["active", "inactive", "future"]
+  }
 }
 ```
+
 ```json
 {
-   "task": "rebuild_missing_bounding_boxes",
-   "payload": {
-      "dry_run": true,
-      "after_date": "2025-06-01"
-   }
+  "task": "rebuild_missing_bounding_boxes",
+  "payload": {
+    "dry_run": true,
+    "after_date": "2025-06-01"
+  }
 }
 ```
+
 ```json
 {
-   "task": "refresh_materialized_view",
-   "payload": {
+  "task": "refresh_materialized_view",
+  "payload": {
     "dry_run": true
   }
 }
 ```
 
 To get the list of supported tasks use:
+
 ```json
 {
-   "name": "list_tasks",
-   "payload": {}
+  "name": "list_tasks",
+  "payload": {}
 }
 ```
+
 To update the geolocation files precision:
+
 ```json
 {
-   "task": "update_geojson_files_precision",
-   "payload": {
-      "dry_run": true,
-      "data_type": "gtfs",
-      "precision": 5,
-      "limit": 10
-   }
+  "task": "update_geojson_files_precision",
+  "payload": {
+    "dry_run": true,
+    "data_type": "gtfs",
+    "precision": 5,
+    "limit": 10
+  }
 }
-```
+```
+
+To populate license rules:
+
+```json
+{
+  "task": "populate_license_rules",
+  "payload": {
+    "dry_run": true
+  }
+}
+```

functions-python/tasks_executor/src/main.py

@@ -40,6 +40,10 @@
 )
 from tasks.data_import.import_jbda_feeds import import_jbda_handler
 
+from tasks.licenses.populate_license_rules import (
+    populate_license_rules_handler,
+)
+
 init_logger()
 LIST_COMMAND: Final[str] = "list"
 tasks = {
@@ -82,6 +86,10 @@
         "description": "Imports JBDA data into the system.",
         "handler": import_jbda_handler,
     },
+    "populate_license_rules": {
+        "description": "Populates license rules in the database from a predefined JSON source.",
+        "handler": populate_license_rules_handler,
+    },
 }
 
 

functions-python/tasks_executor/src/tasks/licenses/populate_license_rules.py

@@ -0,0 +1,94 @@
+import logging
+
+import requests
+from shared.database.database import with_db_session
+from shared.database_gen.sqlacodegen_models import Rule
+
+RULES_JSON_URL = (
+    "https://raw.githubusercontent.com/MobilityData/licenses-aas/main/data/rules.json"
+)
+
+
+def populate_license_rules_handler(payload):
+    """
+    Handler for populating license rules.
+
+    Args:
+        payload (dict): Incoming payload data.
+
+    """
+    (dry_run) = get_parameters(payload)
+    return populate_license_rules_task(dry_run)
+
+
+@with_db_session
+def populate_license_rules_task(dry_run, db_session):
+    """
+    Populates license rules in the database. This function is triggered by a Cloud Task.
+
+    Args:
+        dry_run (bool): If True, the function will simulate the operation without making changes.
+        db_session: Database session for executing queries.
+    """
+    logging.info("Starting populate_license_rules_task with dry_run=%s", dry_run)
+
+    try:
+        logging.info("Downloading rules from %s", RULES_JSON_URL)
+        response = requests.get(RULES_JSON_URL, timeout=10)
+        response.raise_for_status()
+        rules_json = response.json()
+        # Type mapping to satisfy the check constraint of Rule.type
+        TYPE_MAPPING = {
+            "permissions": "permission",
+            "conditions": "condition",
+            "limitations": "limitation",
+        }
+
+        # Combine all rule lists from the three categories
+        rules_data = []
+        for rule_type, rule_list in rules_json.items():
+            normalized_type = TYPE_MAPPING[rule_type]
+            for rule_data in rule_list:
+                rule_data["type"] = normalized_type
+                rules_data.append(rule_data)
+
+        logging.info(
+            "Loaded %d rules from %d categories.", len(rules_data), len(rules_json)
+        )
+
+        if dry_run:
+            logging.info("Dry run: would insert/update %d rules.", len(rules_data))
+        else:
+            for rule_data in rules_data:
+                rule_object = Rule(
+                    name=rule_data.get("name"),
+                    label=rule_data.get("label"),
+                    description=rule_data.get("description"),
+                    type=rule_data.get("type"),
+                )
+                db_session.merge(rule_object)
+
+            logging.info(
+                "Successfully upserted %d rules into the database.", len(rules_data)
+            )
+
+    except requests.exceptions.RequestException as e:
+        logging.error("Failed to download rules JSON file: %s", e)
+        raise
+    except Exception as e:
+        logging.error("An error occurred while populating license rules: %s", e)
+        db_session.rollback()
+        raise
+
+
+def get_parameters(payload):
+    """
+    Get parameters from the payload and environment variables.
+
+    Args:
+        payload (dict): dictionary containing the payload data.
+    Returns:
+        tuple: (dry_run, after_date)
+    """
+    dry_run = payload.get("dry_run", False)
+    return dry_run

functions-python/tasks_executor/tests/tasks/populate_license_rules/test_populate_license_rules.py

@@ -0,0 +1,125 @@
+import unittest
+from unittest.mock import patch, MagicMock
+import requests
+from sqlalchemy.exc import SQLAlchemyError
+
+from tasks.licenses.populate_license_rules import (
+    populate_license_rules_task,
+    RULES_JSON_URL,
+)
+from shared.database_gen.sqlacodegen_models import Rule
+
+
+class TestPopulateLicenseRules(unittest.TestCase):
+    def setUp(self):
+        """Set up mock data for tests."""
+        self.mock_rules_json = {
+            "permissions": [
+                {
+                    "name": "commercial-use",
+                    "label": "Commercial Use",
+                    "description": "The licensed material may be used for commercial purposes.",
+                }
+            ],
+            "conditions": [
+                {
+                    "name": "include-copyright",
+                    "label": "Include Copyright",
+                    "description": "A copy of the copyright and license notices must be included.",
+                }
+            ],
+            "limitations": [],
+        }
+
+    @patch("tasks.licenses.populate_license_rules.requests.get")
+    def test_populate_rules_success(self, mock_requests_get):
+        """Test successful population of license rules."""
+        # Arrange
+        mock_response = MagicMock()
+        mock_response.json.return_value = self.mock_rules_json
+        mock_response.raise_for_status.return_value = None
+        mock_requests_get.return_value = mock_response
+
+        mock_db_session = MagicMock()
+
+        # Act
+        populate_license_rules_task(dry_run=False, db_session=mock_db_session)
+
+        # Assert
+        mock_requests_get.assert_called_once_with(RULES_JSON_URL, timeout=10)
+        self.assertEqual(mock_db_session.merge.call_count, 2)
+
+        # Check that merge was called with correctly constructed Rule objects
+        call_args_list = mock_db_session.merge.call_args_list
+
+        # Check first call
+        arg1 = call_args_list[0].args[0]
+        self.assertIsInstance(arg1, Rule)
+        self.assertEqual(arg1.name, "commercial-use")
+        self.assertEqual(arg1.type, "permission")
+
+        # Check second call
+        arg2 = call_args_list[1].args[0]
+        self.assertIsInstance(arg2, Rule)
+        self.assertEqual(arg2.name, "include-copyright")
+        self.assertEqual(arg2.type, "condition")
+
+        mock_db_session.rollback.assert_not_called()
+
+    @patch("tasks.licenses.populate_license_rules.requests.get")
+    def test_populate_rules_dry_run(self, mock_requests_get):
+        """Test that no database changes are made during a dry run."""
+        # Arrange
+        mock_response = MagicMock()
+        mock_response.json.return_value = self.mock_rules_json
+        mock_response.raise_for_status.return_value = None
+        mock_requests_get.return_value = mock_response
+
+        mock_db_session = MagicMock()
+
+        # Act
+        populate_license_rules_task(dry_run=True, db_session=mock_db_session)
+
+        # Assert
+        mock_requests_get.assert_called_once_with(RULES_JSON_URL, timeout=10)
+        mock_db_session.merge.assert_not_called()
+        mock_db_session.rollback.assert_not_called()
+
+    @patch("tasks.licenses.populate_license_rules.requests.get")
+    def test_request_exception_handling(self, mock_requests_get):
+        """Test handling of a requests exception."""
+        # Arrange
+        mock_requests_get.side_effect = requests.exceptions.RequestException(
+            "Network Error"
+        )
+        mock_db_session = MagicMock()
+
+        # Act & Assert
+        with self.assertRaises(requests.exceptions.RequestException):
+            populate_license_rules_task(dry_run=False, db_session=mock_db_session)
+
+        mock_db_session.merge.assert_not_called()
+        mock_db_session.rollback.assert_not_called()
+
+    @patch("tasks.licenses.populate_license_rules.requests.get")
+    def test_database_exception_handling(self, mock_requests_get):
+        """Test handling of a database exception during merge."""
+        # Arrange
+        mock_response = MagicMock()
+        mock_response.json.return_value = self.mock_rules_json
+        mock_response.raise_for_status.return_value = None
+        mock_requests_get.return_value = mock_response
+
+        mock_db_session = MagicMock()
+        mock_db_session.merge.side_effect = SQLAlchemyError("DB connection failed")
+
+        # Act & Assert
+        with self.assertRaises(SQLAlchemyError):
+            populate_license_rules_task(dry_run=False, db_session=mock_db_session)
+
+        self.assertTrue(mock_db_session.merge.called)
+        mock_db_session.rollback.assert_called_once()
+
+
+if __name__ == "__main__":
+    unittest.main()

liquibase/changelog.xml

@@ -78,4 +78,5 @@
     <!-- Materialized view recreated because of note data type change -->
     <include file="changes/feat_pt_154.sql" relativeToChangelogFile="true"/>
     <include file="changes/feat_1249.sql" relativeToChangelogFile="true"/>
+    <include file="changes/feat_1343.sql" relativeToChangelogFile="true"/>
 </databaseChangeLog>

liquibase/changes/feat_1343.sql

@@ -0,0 +1,27 @@
+CREATE TABLE license (
+id TEXT PRIMARY KEY,
+type TEXT NOT NULL CHECK (type IN ('standard', 'custom')),
+is_spdx BOOLEAN NOT NULL DEFAULT FALSE,
+name TEXT NOT NULL,
+url TEXT,
+description TEXT,
+content_txt TEXT,
+content_html TEXT,
+created_at TIMESTAMP,
+updated_at TIMESTAMP
+);
+
+-- Unified rules table
+CREATE TABLE rules (
+name TEXT PRIMARY KEY,
+label TEXT NOT NULL,
+description TEXT,
+type TEXT NOT NULL CHECK (type IN ('permission', 'condition', 'limitation'))
+);
+
+-- Join table for license-rule mappings
+CREATE TABLE license_rules (
+license_id TEXT REFERENCES license(id) ON DELETE CASCADE,
+rule_id TEXT REFERENCES rules(name) ON DELETE CASCADE,
+PRIMARY KEY (license_id, rule_id)
+);