fix: changing queue name + env vars

Author: cka-y

functions-python/batch_process_dataset/src/pipeline_tasks.py

@@ -19,7 +19,7 @@ def create_http_reverse_geolocation_processor_task(
     body = json.dumps(
         {"stable_id": stable_id, "stops_url": stops_url, "dataset_id": dataset_stable_id}
     ).encode()
-    queue_name = os.getenv("REVERSE_GEOLOCATION_QUEUE_NAME")
+    queue_name = os.getenv("REVERSE_GEOLOCATION_QUEUE")
     project_id = os.getenv("PROJECT_ID")
     gcp_region = os.getenv("GCP_REGION")
 
@@ -32,6 +32,7 @@ def create_http_reverse_geolocation_processor_task(
         queue_name,
     )
 
+
 def create_http_pmtiles_builder_task(
     stable_id: str,
     dataset_stable_id: str,
@@ -43,7 +44,7 @@ def create_http_pmtiles_builder_task(
     body = json.dumps(
         {"feed_stable_id": stable_id, "dataset_stable_id": dataset_stable_id}
     ).encode()
-    queue_name = os.getenv("PMTILES_BUILDER_QUEUE_NAME")
+    queue_name = os.getenv("PMTILES_BUILDER_QUEUE")
     project_id = os.getenv("PROJECT_ID")
     gcp_region = os.getenv("GCP_REGION")
 

infra/batch/main.tf

@@ -249,6 +249,27 @@ resource "google_pubsub_topic" "pubsub_topic" {
   name = "datasets-batch-topic-${var.environment}"
 }
 
+# Task queue to invoke pmtiles_builder function
+resource "google_cloud_tasks_queue" "pmtiles_builder_task_queue" {
+  project  = var.project_id
+  location = var.gcp_region
+  name     = "pmtiles-builder-task-queue-${var.environment}"
+
+  rate_limits {
+    max_concurrent_dispatches = 1
+    max_dispatches_per_second = 1
+  }
+
+  retry_config {
+    # This will make the cloud task retry for ~1 hour
+    max_attempts  = 31
+    min_backoff   = "120s"
+    max_backoff   = "120s"
+    max_doublings = 2
+  }
+}
+
+
 # Batch process dataset function
 resource "google_cloudfunctions2_function" "pubsub_function" {
   name        = "${local.function_batch_process_dataset_config.name}-${var.environment}"
@@ -284,7 +305,8 @@ resource "google_cloudfunctions2_function" "pubsub_function" {
       GCP_REGION = var.gcp_region
       SERVICE_ACCOUNT_EMAIL = google_service_account.functions_service_account.email
       MATERIALIZED_VIEW_QUEUE = google_cloud_tasks_queue.refresh_materialized_view_task_queue.name
-
+      PMTILES_BUILDER_QUEUE = google_cloud_tasks_queue.pmtiles_builder_task_queue.name
+      REVERSE_GEOLOCATION_QUEUE = "reverse-geolocation-processor-task-queue"
     }
     dynamic "secret_environment_variables" {
       for_each = local.function_batch_process_dataset_config.secret_environment_variables

infra/functions-python/main.tf

@@ -1491,27 +1491,6 @@ resource "google_cloud_tasks_queue" "refresh_materialized_view_task_queue" {
   }
 }
 
-# Task queue to invoke pmtiles_builder function
-resource "google_cloud_tasks_queue" "pmtiles_builder_task_queue" {
-  project  = var.project_id
-  location = var.gcp_region
-  name     = "pmtiles-builder-task-queue"
-
-  rate_limits {
-    max_concurrent_dispatches = 1
-    max_dispatches_per_second = 1
-  }
-
-  retry_config {
-    # This will make the cloud task retry for ~1 hour
-    max_attempts  = 31
-    min_backoff   = "120s"
-    max_backoff   = "120s"
-    max_doublings = 2
-  }
-}
-
-
 # Task queue to invoke gbfs_validator_batch function for the scheduler
 resource "google_cloudfunctions2_function_iam_member" "gbfs_validator_batch_invoker" {
   project        = var.project_id