diff --git a/modules/airflow_pool/README.md b/modules/airflow_pool/README.md index e739a41..dd082ab 100644 --- a/modules/airflow_pool/README.md +++ b/modules/airflow_pool/README.md @@ -6,15 +6,20 @@ This optional module is used to create a Cloud Composer Pool. module "pool" { source = "terraform-google-modules/composer/google//modules/airflow_pool" - project_id = "project-123" - environment = "Composer-Prod-Env" - location = "us-central1" - name = "beta" - slots = 1000 - description = "Pool used by beta DAGs" + project = "project-123" + environment = "Composer-Prod-Env" + location = "us-central1" + composer_env_name = "beta" + slot_count = 1000 + description = "Pool used by beta DAGs" } ``` +> [!IMPORTANT] +> to delete a pool you first need to run +> ``` terraform destroy --target module.however_you_named_it ``` +> if you just remove the module configuration from your code the pool will not be deleted. + ``` ## Inputs @@ -24,9 +29,10 @@ module "pool" { | composer\_env\_name | Name of the Cloud Composer Environment. | `string` | n/a | yes | | description | The description of the pool | `string` | `"Managed by Terraform"` | no | | pool\_name | The name of the pool | `string` | n/a | yes | -| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes | +| project | Project name where Cloud Composer Environment is created. | `string` | n/a | yes | | region | Region where the Cloud Composer Environment is created. | `string` | n/a | yes | | slot\_count | The number of slots in this pool | `number` | n/a | yes | +| include_deferred | Whether the pool should include deferred tasks in its calculation of occupied slots | `bool` | false | no | ## Outputs diff --git a/modules/airflow_pool/main.tf b/modules/airflow_pool/main.tf index 5dbb93c..8b921e5 100644 --- a/modules/airflow_pool/main.tf +++ b/modules/airflow_pool/main.tf @@ -15,14 +15,15 @@ */ locals { - gcloud_cmd_body = "composer environments run --project=${var.project_id} --location=${var.region} ${var.composer_env_name} pool" - create_cmd_body = "${local.gcloud_cmd_body} -- --set ${jsonencode(var.pool_name)} ${jsonencode(var.slot_count)} ${jsonencode(var.description)}" - destroy_cmd_body = "${local.gcloud_cmd_body} -- --delete ${jsonencode(var.pool_name)}" + include_deferred_flag = var.include_deferred ? " --include-deferred" : "" + gcloud_cmd_body = "composer environments run --project=${var.project} --location=${var.region} ${var.composer_env_name} pools" + create_cmd_body = "${local.gcloud_cmd_body} -- set ${jsonencode(var.pool_name)} ${jsonencode(var.slot_count)} ${jsonencode(var.description)} ${local.include_deferred_flag}" + destroy_cmd_body = "${local.gcloud_cmd_body} -- delete ${jsonencode(var.pool_name)}" } module "gcloud" { source = "terraform-google-modules/gcloud/google" - version = "~> 3.1" + version = "~> 4.0" platform = "linux" create_cmd_body = local.create_cmd_body destroy_cmd_body = local.destroy_cmd_body diff --git a/modules/airflow_pool/variables.tf b/modules/airflow_pool/variables.tf index ca47a20..46f251e 100644 --- a/modules/airflow_pool/variables.tf +++ b/modules/airflow_pool/variables.tf @@ -14,9 +14,9 @@ * limitations under the License. */ -variable "project_id" { +variable "project" { type = string - description = "Project ID where Cloud Composer Environment is created." + description = "Project name where Cloud Composer Environment is created." } variable "region" { @@ -39,6 +39,12 @@ variable "slot_count" { description = "The number of slots in this pool" } +variable "include_deferred" { + type = bool + default = false + description = "Whether the pool should include deferred tasks in its calculation of occupied slots" +} + variable "description" { type = string description = "The description of the pool"