feat: batch task to reduce geojson maps precision and remove ids (#1353)

Author: davidgamez

api/tests/test_utils/db_utils.py

@@ -192,23 +192,14 @@ def is_test_db(url):
 
 def empty_database(db, url):
     if is_test_db(url):
-
-        metadata_tables = Base.metadata.tables
-
-        # Get all table names excluding those in the excluded_tables list
-        all_table_names = [table_name for table_name in metadata_tables.keys() if table_name not in excluded_tables]
-
-        # Sort the table names in reverse order of dependencies
-        tables_to_delete = sorted(
-            all_table_names, key=lambda name: len(metadata_tables[name].foreign_keys), reverse=True
-        )
-
         try:
             with db.start_db_session() as session:
-                for table_name in tables_to_delete:
-                    table = Base.metadata.tables[table_name]
-                    delete_stmt = delete(table)
-                    session.execute(delete_stmt)
-
+                # Using sorted_tables to respect foreign key constraints
+                for table in reversed(Base.metadata.sorted_tables):
+                    if table.name not in excluded_tables:
+                        table = Base.metadata.tables[table.name]
+                        delete_stmt = delete(table)
+                        session.execute(delete_stmt)
+                session.commit()
         except Exception as error:
             logging.error(f"Error while deleting from test db: {error}")

functions-python/helpers/locations.py

@@ -6,7 +6,6 @@
 from sqlalchemy import func, cast
 from geoalchemy2.types import Geography
 
-import pycountry
 from shared.database_gen.sqlacodegen_models import Feed, Location, Geopolygon
 import logging
 
@@ -35,6 +34,8 @@ def get_country_code(country_name: str) -> Optional[str]:
     Returns:
         Optional[str]: Two-letter ISO country code or None if not found
     """
+    import pycountry
+
     # Return None for empty or whitespace-only strings
     if not country_name or not country_name.strip():
         logging.error("Could not find country code for: empty string")

functions-python/process_validation_report/tests/conftest.py

@@ -143,7 +143,7 @@ def pytest_sessionfinish(session, exitstatus):
     returning the exit status to the system.
     """
     # Cleaned at the beginning instead of the end so we can examine the DB after the test.
-    # clean_testing_db()
+    clean_testing_db()
 
 
 def pytest_unconfigure(config):

functions-python/process_validation_report/tests/test_validation_report.py

@@ -82,6 +82,7 @@ def test_get_dataset(self, db_session):
         )
         try:
             db_session.add(feed)
+            db_session.flush()
             db_session.add(dataset)
             db_session.flush()
             returned_dataset = get_dataset(dataset_stable_id, db_session)
@@ -123,6 +124,7 @@ def test_create_validation_report_entities(self, mock_get, db_session):
         )
         try:
             db_session.add(feed)
+            db_session.flush()
             db_session.add(dataset)
             db_session.commit()
             create_validation_report_entities(feed_stable_id, dataset_stable_id, "1.0")
@@ -347,17 +349,18 @@ def test_create_validation_report_entities_missing_validator_version(
                 ],
             },
         )
-        feed_stable_id = faker.word()
-        dataset_stable_id = faker.word()
+        feed_stable_id = faker.uuid4()
+        dataset_stable_id = faker.uuid4()
 
         # Create GTFS Feed
-        feed = Gtfsfeed(id=faker.word(), data_type="gtfs", stable_id=feed_stable_id)
+        feed = Gtfsfeed(id=faker.uuid4(), data_type="gtfs", stable_id=feed_stable_id)
         # Create a new dataset
         dataset = Gtfsdataset(
-            id=faker.word(), feed_id=feed.id, stable_id=dataset_stable_id, latest=True
+            id=faker.uuid4(), feed_id=feed.id, stable_id=dataset_stable_id, latest=True
         )
         try:
             db_session.add(feed)
+            db_session.flush()
             db_session.add(dataset)
             db_session.commit()
             create_validation_report_entities(feed_stable_id, dataset_stable_id, "1.0")

functions-python/tasks_executor/README.md

@@ -50,3 +50,15 @@ To get the list of supported tasks use:
    "payload": {}
 }
 ```
+To update the geolocation files precision:
+```json
+{
+   "task": "update_geojson_files_precision",
+   "payload": {
+      "dry_run": true,
+      "data_type": "gtfs",
+      "precision": 5,
+      "limit": 10
+   }
+}
+```

functions-python/tasks_executor/requirements.txt

@@ -11,6 +11,7 @@ pluggy~=1.3.0
 certifi~=2025.8.3
 fastapi
 uvicorn[standard]
+psutil
 
 
 # SQL Alchemy and Geo Alchemy

functions-python/tasks_executor/src/main.py

@@ -35,6 +35,9 @@
 from tasks.visualization_files.rebuild_missing_visualization_files import (
     rebuild_missing_visualization_files_handler,
 )
+from tasks.geojson.update_geojson_files_precision import (
+    update_geojson_files_precision_handler,
+)
 
 init_logger()
 LIST_COMMAND: Final[str] = "list"
@@ -66,6 +69,10 @@
         "description": "Rebuilds missing dataset files for GTFS datasets.",
         "handler": rebuild_missing_dataset_files_handler,
     },
+    "update_geojson_files": {
+        "description": "Iterate over bucket looking for {feed_stable_id}/geolocation.geojson and update precision.",
+        "handler": update_geojson_files_precision_handler,
+    },
     "rebuild_missing_visualization_files": {
         "description": "Rebuilds missing visualization files for GTFS datasets.",
         "handler": rebuild_missing_visualization_files_handler,

functions-python/tasks_executor/src/tasks/geojson/README.md

@@ -0,0 +1,58 @@
+# Update GeoJSON files
+
+This task adjust the GeoJSON files removing the map IDs and reducing the precision of the coordinates to 5 decimal places. ALso updates the geolocation_file_created_date and geolocation_file_dataset_id fields in the Feed table.
+
+---
+
+## Task ID
+
+Use task ID: `update_geojson_files_precision`
+
+---
+
+## Usage
+
+The function accepts the following payload:
+
+```json
+{
+  "dry_run": true,                // [optional] If true, do not upload or modify the database (default: true)
+  "precision": 5,                 // [optional] Number of decimal places to keep in coordinates (default: 5)
+  "limit": 10,                    // [optional] Limit the number of feeds to process (default: no limit)
+  "data_type": "gtfs"             // [optional] Type of data to process, either "gtfs" or "gbfs" (default: "gtfs")
+}
+```
+
+### Example:
+
+```json
+{
+  "dry_run": true,
+  "data_type": "gtfs",
+  "limit": 10
+}
+```
+
+---
+
+## What It Does
+
+List all feeds with GeoJSON files, download each file, remove map IDs, reduce coordinate precision to the specified number of decimal places, and re-upload the modified file.
+Also updates the `geolocation_file_created_date` and `geolocation_file_dataset_id` fields in the `Feed` table.
+
+## GCP Environment Variables
+
+The function requires the following environment variables:
+
+| Variable                       | Description                                                             |
+|--------------------------------|-------------------------------------------------------------------------|
+| `DATASETS_BUCKET_NAME`         | The name of the GCS bucket used to store extracted GTFS files           | 
+| `GBFS_SNAPSHOTS_BUCKET_NAME`   | The name of the GCS bucket used to store extracted GBFS snapshots files | 
+
+---
+
+## Additional Notes
+
+* Commits to the database occur in batches of 100 feeds to improve performance and avoid large transaction blocks.
+* If `dry_run` is enabled, files are uploads or DB modifications are performed. Only the number of affected feeds is logged.
+* The function is safe to rerun. It will only affect feeds with missing geolocation_file_dataset_id.