feat: create a gcp function to generate the pmtiles of a single dataset (#1305)

Author: jcpitre

functions-python/tasks_executor/README.md

@@ -13,26 +13,28 @@ The function receive the following payload:
 }
 ```
 
-Example:
+Examples:
 
 ```json
 {
    "task": "rebuild_missing_validation_reports",
    "payload": {
-    "dry_run": true,
-    "filter_after_in_days": 14,
-    "filter_statuses": ["active", "inactive", "future"]
-  }
+      "dry_run": true,
+      "filter_after_in_days": 14,
+      "filter_statuses": ["active", "inactive", "future"]
+   }
 }
 ```
 ```json
 {
    "task": "rebuild_missing_bounding_boxes",
    "payload": {
-    "dry_run": true,
-    "after_date": "2025-06-01"
-  }
+      "dry_run": true,
+      "after_date": "2025-06-01"
+   }
 }
+```
+```json
 {
    "task": "refresh_materialized_view",
    "payload": {
@@ -44,7 +46,7 @@ Example:
 To get the list of supported tasks use:
 ```json
 {
-"name": "list_tasks",
-"payload": {}
+   "name": "list_tasks",
+   "payload": {}
 }
 ```

functions-python/tasks_executor/function_config.json

@@ -2,12 +2,16 @@
   "name": "tasks_executor",
   "description": "The Tasks Executor function runs maintenance tasks avoiding the creation of multiple functions for one-time execution",
   "entry_point": "tasks_executor",
-  "timeout": 540,
-  "memory": "4Gi",
+  "timeout": 1000,
+  "memory": "8Gi",
   "trigger_http": true,
   "include_folders": ["helpers"],
   "include_api_folders": ["database_gen", "database", "common"],
-  "environment_variables": [],
+  "environment_variables": [
+    {
+      "key": "DATASETS_BUCKET_NAME"
+    }
+  ],
   "secret_environment_variables": [
     {
       "key": "FEEDS_DATABASE_URL"
@@ -21,5 +25,5 @@
   "max_instance_request_concurrency": 1,
   "max_instance_count": 1,
   "min_instance_count": 0,
-  "available_cpu": 1
+  "available_cpu": 2
 }

functions-python/tasks_executor/requirements.txt

@@ -25,4 +25,6 @@ flask
 google-cloud-storage
 
 # Configuration
-python-dotenv==1.0.0
+python-dotenv==1.0.0
+tippecanoe
+

functions-python/tasks_executor/src/main.py

@@ -31,7 +31,7 @@
 from tasks.missing_bounding_boxes.rebuild_missing_bounding_boxes import (
     rebuild_missing_bounding_boxes_handler,
 )
-
+from tasks.pmtiles_builder.build_pmtiles import build_pmtiles_handler
 
 init_logger()
 LIST_COMMAND: Final[str] = "list"
@@ -63,6 +63,10 @@
         "description": "Rebuilds missing dataset files for GTFS datasets.",
         "handler": rebuild_missing_dataset_files_handler,
     },
+    "build_pmtiles": {
+        "description": "Build pmtiles for a given dataset.",
+        "handler": build_pmtiles_handler,
+    },
 }
 
 

functions-python/tasks_executor/src/tasks/pmtiles_builder/README.md

@@ -0,0 +1,31 @@
+# Build pmtile for a specific GTFS dataset
+
+This task generates the pmtiles for a provided dataset.
+pmtiles are used for displaying routes and stops in the UI
+
+## Task ID
+Use task Id: `build_pmtiles`
+
+## Usage
+The function receive the following payload:
+```
+   {
+      "feed_stable_id": str, 
+      "dataset_stable_id*: str 
+   }
+```
+
+Example:
+```json
+   {
+      "feed_stable_id": "mdb-1004", 
+      "dataset_stable_id": "mdb-1004-202507081807"
+   }
+```
+
+The task will verify that the dataset stable id starts with the feed stable id (mdb-1004 in our example)
+
+# GCP environment variables
+The function uses the following environment variables:
+- `ENV`: The environment to use. It can be `dev`, `staging` or `prod`. Default is `dev`.
+- `DATASETS_BUCKET_NAME`: The bucket name where the datasets are stored. The task will fail if this is not defined.  The variable has to include the suffix, like `-dev`, `-qa` or `-prod`.