set cloud name dynamic

Author: davidgamez

infra/functions-python/main.tf

@@ -370,8 +370,12 @@ resource "google_cloud_tasks_queue" "cloud_tasks_dead_letter_queue" {
 
 # Create a queue for the cloud tasks
 # The 2X rate is defined as 4*2 concurrent dispatches and 1 dispatch per second
+# The name of the queue need to be dynamic due to GCP limitations
+# references:
+#   - https://cloud.google.com/tasks/docs/deleting-appengine-queues-and-tasks#deleting_queues
+#   - https://issuetracker.google.com/issues/263947953
 resource "google_cloud_tasks_queue" "cloud_tasks_2x_rate_queue" {
-  name     = "cloud-tasks-2x-rate-queue-${var.environment}"
+  name     = "cloud-tasks-2x-rate-queue-${var.environment}-${formatdate("YYYYMMDDhhmmss", timestamp())}"
   location = var.gcp_region
 
   rate_limits {
@@ -388,6 +392,10 @@ resource "google_cloud_tasks_queue" "cloud_tasks_2x_rate_queue" {
   }
 }
 
+output "processing_report_cloud_task_name" {
+  value = google_cloud_tasks_queue.cloud_tasks_2x_rate_queue.name
+}
+
 resource "google_cloudfunctions2_function" "process_validation_report" {
   name        = local.function_process_validation_report_config.name
   description = local.function_process_validation_report_config.description

infra/main.tf

@@ -116,6 +116,7 @@ module "workflows" {
   gcp_region         = var.gcp_region
   environment        = var.environment
   validator_endpoint = var.validator_endpoint
+  processing_report_cloud_task_name = module.functions-python.processing_report_cloud_task_name
 }
 
 module "feed-api-load-balancer" {

infra/workflows/main.tf

@@ -90,6 +90,7 @@ resource "google_workflows_workflow" "gtfs_validator_execution" {
     reports_bucket_name   = lower(var.environment) == "prod" ? var.reports_bucket_name : "stg-${var.reports_bucket_name}"
     validator_endpoint    = var.validator_endpoint
     environment           = lower(var.environment)
+    processing_report_cloud_task_name       = var.processing_report_cloud_task_name
   }
   source_contents         = file("${path.module}../../../workflows/gtfs_validator_execution.yml")
 }

infra/workflows/vars.tf

@@ -51,3 +51,8 @@ variable "validator_endpoint" {
   type = string
   description = "URL of the validator endpoint"
 }
+
+variable "processing_report_cloud_task_name" {
+  type = string
+  description = "The cloud task name to call the process report task"
+}

workflows/gtfs_validator_execution.yml

@@ -19,9 +19,10 @@ main:
             - datasetID: ${text.split(data.resourceName, "/")[6]}
             - region: ${resource.labels.location}
             - projectID: ${resource.labels.project_id}
-            # Extracts the environment from the project ID. Ex: mobility-feeds-prod -> prod. Values: prod, qa, dev
-            - environment: ${text.split(projectID, "-")[2]}
-            - cloudTaskName: ${"cloud-tasks-2x-rate-queue-" + environment}
+            - environment: ${sys.get_env("environment")}
+            # The cloud tasks are sensitive to deletion.
+            # This is why we need to have this as a parameter set at deploying time.
+            - cloudTaskName: ${sys.get_env("processing_report_cloud_task_name")}
             - serviceAccountEmail: ${"workflows-service-account@mobility-feeds-" + environment + ".iam.gserviceaccount.com"}
             - url: ${"https://storage.googleapis.com/" + datasetsBucket + "/" + feedID + "/" + datasetID + "/" + datasetID + ".zip"}
             - headers: