diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 07b56b2a..dadf50cf 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -1,4 +1,4 @@ -# Copyright 2020 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,22 +25,17 @@ steps: - 'TF_VAR_folder_id=$_FOLDER_ID' - 'TF_VAR_billing_account=$_BILLING_ACCOUNT' -- id: create all +- id: init-all waitFor: - prepare name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create'] + args: ['/bin/bash', '-c', 'cft test run all --stage init --verbose'] # ----- SUITE simple-composer-env-v2 -- id: init-simple-composer-env-v2 - waitFor: - - create all - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage init --verbose'] - id: apply-simple-composer-env-v2 waitFor: - - init-simple-composer-env-v2 + - init-all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage apply --verbose'] - id: verify-simple-composer-env-v2 @@ -48,23 +43,12 @@ steps: - apply-simple-composer-env-v2 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose'] -- id: destroy-simple-composer-env-v2 - waitFor: - - verify-simple-composer-env-v2 - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose'] - # ----- SUITE composer-v2-sharedvpc-prereq-local -- id: init-composer-v2-sharedvpc-prereq - waitFor: - - destroy-simple-composer-env-v2 - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage init --verbose'] - id: apply-composer-v2-sharedvpc-prereq waitFor: - - init-composer-v2-sharedvpc-prereq + - init-all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage apply --verbose'] - id: verify-composer-v2-sharedvpc-prereq @@ -72,11 +56,37 @@ steps: - apply-composer-v2-sharedvpc-prereq name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage verify --verbose'] -- id: destroy-composer-v2-sharedvpc-prereq + + # ----- SUITE simple-composer-env-v3 + +- id: apply-simple-composer-env-v3 + waitFor: + - init-all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage apply --verbose'] +- id: verify-simple-composer-env-v3 + waitFor: + - apply-simple-composer-env-v3 + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage verify --verbose'] + +- id: destroy-v2 + waitFor: + - verify-simple-composer-env-v2 + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose'] + +- id: destroy-v2-sharedvpc-prereq waitFor: - verify-composer-v2-sharedvpc-prereq name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage destroy --verbose'] + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage verify --verbose'] + +- id: destroy-v3 + waitFor: + - verify-simple-composer-env-v3 + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage verify --verbose'] tags: - 'ci' diff --git a/examples/simple_composer_env_v3/README.md b/examples/simple_composer_env_v3/README.md new file mode 100644 index 00000000..d4b35fd4 --- /dev/null +++ b/examples/simple_composer_env_v3/README.md @@ -0,0 +1,33 @@ +# Simple Cloud Composer Environment (V3) Example + +This example illustrates how to use the `composer` V2 module to deploy private composer environment with private service connect (PSC) endpoint to connect network attachments. + +This example also creates a Cloud Storage Bucket for scheduled snapshots and assign appropriate permission(s) to Composer Service Account on the bucket. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| composer\_env\_name | Name of Cloud Composer Environment. | `string` | `"ci-composer"` | no | +| composer\_sa | Service Account to be used for running Cloud Composer Environment. | `string` | n/a | yes | +| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes | +| region | Region where Cloud Composer Environment is created. | `string` | `"us-central1"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| airflow\_uri | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. | +| composer\_env\_id | ID of Cloud Composer Environment. | +| composer\_env\_name | Name of the Cloud Composer Environment. | +| gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. | +| project\_id | Project ID where Cloud Composer Environment is created. | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_composer_env_v3/main.tf b/examples/simple_composer_env_v3/main.tf new file mode 100644 index 00000000..c7c08cdf --- /dev/null +++ b/examples/simple_composer_env_v3/main.tf @@ -0,0 +1,83 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "key_suffix" { + length = 5 + special = false + upper = false +} + +# Create a bucket to store the snapshots +resource "google_storage_bucket" "my_bucket" { + project = var.project_id + name = "snapshot-bucket-${random_string.key_suffix.result}" + location = var.region + force_destroy = true + uniform_bucket_level_access = true +} + +resource "google_storage_bucket_iam_member" "object_admin" { + bucket = google_storage_bucket.my_bucket.name + role = "roles/storage.objectAdmin" + member = "serviceAccount:${var.composer_sa}" +} + +module "simple-composer-environment" { + source = "terraform-google-modules/composer/google//modules/create_environment_v3" + version = "~> 6.0" + + project_id = var.project_id + composer_env_name = var.composer_env_name + region = var.region + composer_service_account = var.composer_sa + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + create_network_attachment = true + + grant_sa_agent_permission = false + environment_size = "ENVIRONMENT_SIZE_SMALL" + + use_private_environment = true + enable_private_builds_only = true + cloud_data_lineage_integration = true + resilience_mode = "STANDARD_RESILIENCE" + + scheduled_snapshots_config = { + enabled = true + snapshot_location = google_storage_bucket.my_bucket.url + snapshot_creation_schedule = "0 4 * * *" + time_zone = "UTC+01" + } + + maintenance_start_time = "2025-02-01T00:00:00Z" + maintenance_end_time = "2025-05-01T12:00:00Z" + maintenance_recurrence = "FREQ=WEEKLY;BYDAY=SU,SA" + + depends_on = [ + google_storage_bucket_iam_member.object_admin, + ] + + web_server_network_access_control = [ + { + allowed_ip_range = "192.0.2.0/24" + description = "office net 1" + }, + { + allowed_ip_range = "192.0.4.0/24" + description = "office net 2" + }, + ] +} diff --git a/examples/simple_composer_env_v3/network.tf b/examples/simple_composer_env_v3/network.tf new file mode 100644 index 00000000..c911d635 --- /dev/null +++ b/examples/simple_composer_env_v3/network.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_network" "main" { + project = var.project_id + name = "ci-composer-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + project = var.project_id + name = "ci-composer-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + private_ip_google_access = true +} + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} diff --git a/examples/simple_composer_env_v3/outputs.tf b/examples/simple_composer_env_v3/outputs.tf new file mode 100644 index 00000000..78223c33 --- /dev/null +++ b/examples/simple_composer_env_v3/outputs.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + description = "Project ID where Cloud Composer Environment is created." + value = var.project_id +} + +output "composer_env_name" { + description = "Name of the Cloud Composer Environment." + value = module.simple-composer-environment.composer_env_name +} + +output "composer_env_id" { + description = "ID of Cloud Composer Environment." + value = module.simple-composer-environment.composer_env_id +} + +output "gcs_bucket" { + description = "Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment." + value = module.simple-composer-environment.gcs_bucket +} + +output "airflow_uri" { + description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment." + value = module.simple-composer-environment.airflow_uri +} diff --git a/examples/simple_composer_env_v3/variables.tf b/examples/simple_composer_env_v3/variables.tf new file mode 100644 index 00000000..f8138770 --- /dev/null +++ b/examples/simple_composer_env_v3/variables.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project ID where Cloud Composer Environment is created." + type = string +} + +variable "composer_env_name" { + description = "Name of Cloud Composer Environment." + default = "ci-composer" + type = string +} + +variable "region" { + description = "Region where Cloud Composer Environment is created." + type = string + default = "us-central1" +} + +variable "composer_sa" { + description = "Service Account to be used for running Cloud Composer Environment." + type = string +} diff --git a/modules/create_environment_v3/README.md b/modules/create_environment_v3/README.md new file mode 100644 index 00000000..a4e8e751 --- /dev/null +++ b/modules/create_environment_v3/README.md @@ -0,0 +1,113 @@ +# Module Cloud Composer Environment ([v3](https://cloud.google.com/composer/docs/composer-3/composer-overview)) + +This module is used to create a Cloud Composer v3 environment. + +## Compatibility + +This module is meant for use with Terraform 1.3+ and tested using Terraform 1.3+. If you find incompatibilities using Terraform >=1.3, please open an issue. + + +```hcl +module "simple-composer-environment" { + source = "terraform-google-modules/composer/google//modules/create_environment_v3" + version = "~> 7.0" + project_id = var.project_id + composer_env_name = "test-composer-env" + region = "us-central1" + composer_service_account = var.composer_service_account + network = "test-vpc" + subnetwork = "test-subnet" + grant_sa_agent_permission = false + environment_size = "ENVIRONMENT_SIZE_SMALL" + use_private_environment = true + composer_network_attachment_name = "composer-na" + + scheduler = { + cpu = 0.5 + memory_gb = 1 + storage_gb = 1 + count = 2 + } + + dag_processor = { + cpu = 0.5 + memory_gb = 1 + storage_gb = 1 + count = 2 + } + + web_server = { + cpu = 0.5 + memory_gb = 1 + storage_gb = 1 + } + + worker = { + cpu = 0.5 + memory_gb = 1 + storage_gb = 1 + min_count = 2 + max_count = 3 + } + + triggerer = { + cpu = 1 + memory_gb = 1 + count = 2 + } +} + +``` + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| airflow\_config\_overrides | Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example "core-dags\_are\_paused\_at\_creation". | `map(string)` | `{}` | no | +| cloud\_data\_lineage\_integration | Whether or not Dataplex data lineage integration is enabled. Cloud Composer environments in versions composer-2.1.2-airflow-..* and newer) | `bool` | `false` | no | +| composer\_env\_name | Name of Cloud Composer Environment | `string` | n/a | yes | +| composer\_network\_attachment\_name | Name for PSC (Private Service Connect) Network entry point. | `string` | `null` | no | +| composer\_service\_account | Service Account for running Cloud Composer. | `string` | `null` | no | +| create\_network\_attachment | Either create a new network attachment or use existing one. If true, provide the subnet details. | `bool` | `true` | no | +| dag\_processor | Configuration for resources used by Airflow workers. |
object({
cpu = string
memory_gb = number
storage_gb = number
count = number
}) | {
"count": 2,
"cpu": 2,
"memory_gb": 7.5,
"storage_gb": 5
} | no |
+| enable\_private\_builds\_only | If true, builds performed during operations that install Python packages have only private connectivity to Google services. If false, the builds also have access to the internet. | `bool` | `false` | no |
+| env\_variables | Variables of the airflow environment. | `map(string)` | `{}` | no |
+| environment\_size | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. Values for environment size are: `ENVIRONMENT_SIZE_SMALL`, `ENVIRONMENT_SIZE_MEDIUM`, and `ENVIRONMENT_SIZE_LARGE`. | `string` | `"ENVIRONMENT_SIZE_MEDIUM"` | no |
+| grant\_sa\_agent\_permission | Cloud Composer relies on Workload Identity as Google API authentication mechanism for Airflow. | `bool` | `true` | no |
+| image\_version | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-3-airflow-2.10.2-build.7"` | no |
+| kms\_key\_name | Customer-managed Encryption Key fully qualified resource name, i.e. projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key. | `string` | `null` | no |
+| labels | The resource labels (a map of key/value pairs) to be applied to the Cloud Composer. | `map(string)` | `{}` | no |
+| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `null` | no |
+| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `null` | no |
+| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no |
+| network | The VPC network to host the composer cluster. | `string` | n/a | yes |
+| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no |
+| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes |
+| pypi\_packages | Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. "numpy"). | `map(string)` | `{}` | no |
+| region | Region where the Cloud Composer Environment is created. | `string` | `"us-central1"` | no |
+| resilience\_mode | Cloud Composer 2.1.15 or newer only. The resilience mode states whether high resilience is enabled for the environment or not. Values for resilience mode are `HIGH_RESILIENCE` for high resilience and `STANDARD_RESILIENCE` for standard resilience | `string` | `null` | no |
+| scheduled\_snapshots\_config | The recovery configuration settings for the Cloud Composer environment | object({
enabled = optional(bool, false)
snapshot_location = optional(string)
snapshot_creation_schedule = optional(string)
time_zone = optional(string)
}) | `null` | no |
+| scheduler | Configuration for resources used by Airflow schedulers. | object({
cpu = string
memory_gb = number
storage_gb = number
count = number
}) | {
"count": 2,
"cpu": 1,
"memory_gb": 4,
"storage_gb": 5
} | no |
+| storage\_bucket | Name of an existing Cloud Storage bucket to be used by the environment | `string` | `null` | no |
+| subnetwork | The name of the subnetwork to host the composer cluster. | `string` | n/a | yes |
+| subnetwork\_region | The subnetwork region of the shared VPC's host (for shared vpc support) | `string` | `""` | no |
+| tags | Tags applied to all nodes. Tags are used to identify valid sources or targets for network firewalls. | `set(string)` | `[]` | no |
+| task\_logs\_retention\_storage\_mode | The mode of storage for Airflow workers task logs. Values for storage mode are CLOUD\_LOGGING\_ONLY to only store logs in cloud logging and CLOUD\_LOGGING\_AND\_CLOUD\_STORAGE to store logs in cloud logging and cloud storage. Cloud Composer 2.0.23 or newer only | `string` | `null` | no |
+| triggerer | Configuration for resources used by Airflow triggerer | object({
cpu = string
memory_gb = number
count = number
}) | `null` | no |
+| use\_private\_environment | Create a private environment. If true, a private Composer environment will be created. | `bool` | `false` | no |
+| web\_server | Configuration for resources used by Airflow web server. | object({
cpu = string
memory_gb = number
storage_gb = number
}) | {
"cpu": 2,
"memory_gb": 7.5,
"storage_gb": 5
} | no |
+| web\_server\_network\_access\_control | The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions are applied | list(object({
allowed_ip_range = string
description = string
})) | `null` | no |
+| web\_server\_plugins\_mode | Web server plugins configuration. Can be either 'ENABLED' or 'DISABLED'. Defaults to 'ENABLED'. | `string` | `"ENABLED"` | no |
+| worker | Configuration for resources used by Airflow workers. | object({
cpu = string
memory_gb = number
storage_gb = number
min_count = number
max_count = number
}) | {
"cpu": 2,
"max_count": 6,
"memory_gb": 7.5,
"min_count": 2,
"storage_gb": 5
} | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| airflow\_uri | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. |
+| composer\_env | Cloud Composer Environment |
+| composer\_env\_id | ID of Cloud Composer Environment. |
+| composer\_env\_name | Name of the Cloud Composer Environment. |
+| gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. |
+
+
diff --git a/modules/create_environment_v3/iam.tf b/modules/create_environment_v3/iam.tf
new file mode 100644
index 00000000..e09dc52f
--- /dev/null
+++ b/modules/create_environment_v3/iam.tf
@@ -0,0 +1,26 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+data "google_project" "project" {
+ project_id = var.project_id
+}
+
+resource "google_project_iam_member" "composer_agent_service_account" {
+ count = var.grant_sa_agent_permission ? 1 : 0
+ project = data.google_project.project.project_id
+ role = "roles/composer.ServiceAgentV2Ext"
+ member = format("serviceAccount:%s", local.cloud_composer_sa)
+}
diff --git a/modules/create_environment_v3/main.tf b/modules/create_environment_v3/main.tf
new file mode 100644
index 00000000..b01f1c82
--- /dev/null
+++ b/modules/create_environment_v3/main.tf
@@ -0,0 +1,192 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+ network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id
+ subnetwork_region = var.subnetwork_region != "" ? var.subnetwork_region : var.region
+ cloud_composer_sa = format("service-%s@cloudcomposer-accounts.iam.gserviceaccount.com", data.google_project.project.number)
+}
+
+resource "google_composer_environment" "composer_env" {
+ provider = google-beta
+
+ project = var.project_id
+ name = var.composer_env_name
+ region = var.region
+ labels = var.labels
+
+ dynamic "storage_config" {
+ for_each = var.storage_bucket != null ? ["storage_config"] : []
+ content {
+ bucket = var.storage_bucket
+ }
+ }
+
+ config {
+
+ enable_private_environment = var.use_private_environment # reusing the existing variable name from previous versions
+ enable_private_builds_only = var.enable_private_builds_only
+
+ environment_size = var.environment_size
+ resilience_mode = var.resilience_mode
+
+ node_config {
+ service_account = var.composer_service_account
+ tags = var.tags
+ network = var.create_network_attachment ? "projects/${local.network_project_id}/global/networks/${var.network}" : null
+ subnetwork = var.create_network_attachment ? "projects/${local.network_project_id}/regions/${local.subnetwork_region}/subnetworks/${var.subnetwork}" : null
+ composer_network_attachment = var.create_network_attachment ? null : "projects/{var.project_id}/regions/${var.region}/networkAttachments/${var.composer_network_attachment_name}"
+ }
+
+ dynamic "software_config" {
+ for_each = [
+ {
+ airflow_config_overrides = var.airflow_config_overrides
+ pypi_packages = var.pypi_packages
+ env_variables = var.env_variables
+ image_version = var.image_version
+ web_server_plugins_mode = var.web_server_plugins_mode
+ }]
+ content {
+ airflow_config_overrides = software_config.value["airflow_config_overrides"]
+ pypi_packages = software_config.value["pypi_packages"]
+ env_variables = software_config.value["env_variables"]
+ image_version = software_config.value["image_version"]
+ web_server_plugins_mode = software_config.value["web_server_plugins_mode"]
+ dynamic "cloud_data_lineage_integration" {
+ for_each = var.cloud_data_lineage_integration ? ["cloud_data_lineage_integration"] : []
+ content {
+ enabled = var.cloud_data_lineage_integration
+ }
+ }
+ }
+ }
+
+ dynamic "maintenance_window" {
+ for_each = (var.maintenance_end_time != null && var.maintenance_recurrence != null) ? [
+ {
+ start_time = var.maintenance_start_time
+ end_time = var.maintenance_end_time
+ recurrence = var.maintenance_recurrence
+ }] : []
+ content {
+ start_time = maintenance_window.value["start_time"]
+ end_time = maintenance_window.value["end_time"]
+ recurrence = maintenance_window.value["recurrence"]
+ }
+ }
+
+ workloads_config {
+
+ dynamic "scheduler" {
+ for_each = var.scheduler != null ? [var.scheduler] : []
+ content {
+ cpu = scheduler.value["cpu"]
+ memory_gb = scheduler.value["memory_gb"]
+ storage_gb = scheduler.value["storage_gb"]
+ count = scheduler.value["count"]
+ }
+ }
+
+ dynamic "web_server" {
+ for_each = var.web_server != null ? [var.web_server] : []
+ content {
+ cpu = web_server.value["cpu"]
+ memory_gb = web_server.value["memory_gb"]
+ storage_gb = web_server.value["storage_gb"]
+ }
+ }
+
+ dynamic "worker" {
+ for_each = var.worker != null ? [var.worker] : []
+ content {
+ cpu = worker.value["cpu"]
+ memory_gb = worker.value["memory_gb"]
+ storage_gb = worker.value["storage_gb"]
+ min_count = worker.value["min_count"]
+ max_count = worker.value["max_count"]
+ }
+ }
+
+ dynamic "triggerer" {
+ for_each = var.triggerer != null ? [var.triggerer] : []
+ content {
+ cpu = triggerer.value["cpu"]
+ memory_gb = triggerer.value["memory_gb"]
+ count = triggerer.value["count"]
+ }
+ }
+
+ dynamic "dag_processor" {
+ for_each = var.dag_processor != null ? [var.dag_processor] : []
+ content {
+ cpu = dag_processor.value["cpu"]
+ memory_gb = dag_processor.value["memory_gb"]
+ storage_gb = dag_processor.value["storage_gb"]
+ count = dag_processor.value["count"]
+ }
+ }
+
+ }
+
+ dynamic "recovery_config" {
+ for_each = var.scheduled_snapshots_config != null ? ["recovery_config"] : []
+ content {
+ dynamic "scheduled_snapshots_config" {
+ for_each = var.scheduled_snapshots_config != null ? [var.scheduled_snapshots_config] : []
+ content {
+ enabled = scheduled_snapshots_config.value["enabled"]
+ snapshot_location = scheduled_snapshots_config.value["snapshot_location"]
+ snapshot_creation_schedule = scheduled_snapshots_config.value["snapshot_creation_schedule"]
+ time_zone = scheduled_snapshots_config.value["time_zone"]
+ }
+ }
+ }
+ }
+
+ dynamic "web_server_network_access_control" {
+ for_each = var.web_server_network_access_control == null ? [] : ["web_server_network_access_control"]
+ content {
+ dynamic "allowed_ip_range" {
+ for_each = { for x in var.web_server_network_access_control : x.allowed_ip_range => x }
+ content {
+ value = allowed_ip_range.value["allowed_ip_range"]
+ description = allowed_ip_range.value["description"]
+ }
+ }
+ }
+ }
+
+ dynamic "encryption_config" {
+ for_each = var.kms_key_name != null ? ["encryption_config"] : []
+ content {
+ kms_key_name = var.kms_key_name
+ }
+ }
+
+ dynamic "data_retention_config" {
+ for_each = var.task_logs_retention_storage_mode == null ? [] : ["data_retention_config"]
+ content {
+ task_logs_retention_config {
+ storage_mode = var.task_logs_retention_storage_mode
+ }
+ }
+ }
+ }
+
+ depends_on = [google_project_iam_member.composer_agent_service_account]
+
+}
diff --git a/modules/create_environment_v3/outputs.tf b/modules/create_environment_v3/outputs.tf
new file mode 100644
index 00000000..41d33bc0
--- /dev/null
+++ b/modules/create_environment_v3/outputs.tf
@@ -0,0 +1,40 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "composer_env_name" {
+ value = google_composer_environment.composer_env.name
+ description = "Name of the Cloud Composer Environment."
+}
+
+output "composer_env_id" {
+ value = google_composer_environment.composer_env.id
+ description = "ID of Cloud Composer Environment."
+}
+
+output "gcs_bucket" {
+ value = google_composer_environment.composer_env.config[0].dag_gcs_prefix
+ description = "Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment."
+}
+
+output "airflow_uri" {
+ value = google_composer_environment.composer_env.config[0].airflow_uri
+ description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment."
+}
+
+output "composer_env" {
+ value = google_composer_environment.composer_env
+ description = "Cloud Composer Environment"
+}
diff --git a/modules/create_environment_v3/variables.tf b/modules/create_environment_v3/variables.tf
new file mode 100644
index 00000000..6f72e6bd
--- /dev/null
+++ b/modules/create_environment_v3/variables.tf
@@ -0,0 +1,279 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+ description = "Project ID where Cloud Composer Environment is created."
+ type = string
+}
+
+variable "composer_env_name" {
+ description = "Name of Cloud Composer Environment"
+ type = string
+}
+
+variable "region" {
+ description = "Region where the Cloud Composer Environment is created."
+ type = string
+ default = "us-central1"
+}
+
+variable "labels" {
+ type = map(string)
+ description = "The resource labels (a map of key/value pairs) to be applied to the Cloud Composer."
+ default = {}
+}
+
+variable "tags" {
+ description = "Tags applied to all nodes. Tags are used to identify valid sources or targets for network firewalls."
+ type = set(string)
+ default = []
+}
+
+variable "network" {
+ type = string
+ description = "The VPC network to host the composer cluster."
+}
+
+variable "network_project_id" {
+ type = string
+ description = "The project ID of the shared VPC's host (for shared vpc support)"
+ default = ""
+}
+
+variable "subnetwork" {
+ type = string
+ description = "The name of the subnetwork to host the composer cluster."
+}
+
+variable "subnetwork_region" {
+ type = string
+ description = "The subnetwork region of the shared VPC's host (for shared vpc support)"
+ default = ""
+}
+
+variable "create_network_attachment" {
+ type = bool
+ description = "Either create a new network attachment or use existing one. If true, provide the subnet details."
+ default = true
+}
+
+variable "composer_network_attachment_name" {
+ type = string
+ description = "Name for PSC (Private Service Connect) Network entry point."
+ default = null
+}
+
+variable "composer_service_account" {
+ description = "Service Account for running Cloud Composer."
+ type = string
+ default = null
+}
+
+variable "airflow_config_overrides" {
+ type = map(string)
+ description = "Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example \"core-dags_are_paused_at_creation\"."
+ default = {}
+}
+
+variable "env_variables" {
+ type = map(string)
+ description = "Variables of the airflow environment."
+ default = {}
+}
+
+variable "image_version" {
+ type = string
+ description = "The version of the aiflow running in the cloud composer environment."
+ default = "composer-3-airflow-2.10.2-build.7"
+}
+
+variable "web_server_plugins_mode" {
+ type = string
+ description = "Web server plugins configuration. Can be either 'ENABLED' or 'DISABLED'. Defaults to 'ENABLED'."
+ default = "ENABLED"
+}
+
+variable "pypi_packages" {
+ type = map(string)
+ description = " Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. \"numpy\")."
+ default = {}
+}
+
+variable "use_private_environment" {
+ description = "Create a private environment. If true, a private Composer environment will be created."
+ type = bool
+ default = false
+}
+
+variable "enable_private_builds_only" {
+ type = bool
+ description = "If true, builds performed during operations that install Python packages have only private connectivity to Google services. If false, the builds also have access to the internet."
+ default = false
+}
+
+variable "maintenance_start_time" {
+ description = "Time window specified for daily or recurring maintenance operations in RFC3339 format"
+ type = string
+ default = "05:00"
+}
+
+variable "maintenance_end_time" {
+ description = "Time window specified for recurring maintenance operations in RFC3339 format"
+ type = string
+ default = null
+}
+
+variable "maintenance_recurrence" {
+ description = "Frequency of the recurring maintenance window in RFC5545 format."
+ type = string
+ default = null
+}
+
+variable "environment_size" {
+ type = string
+ description = "The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. Values for environment size are: `ENVIRONMENT_SIZE_SMALL`, `ENVIRONMENT_SIZE_MEDIUM`, and `ENVIRONMENT_SIZE_LARGE`."
+ default = "ENVIRONMENT_SIZE_MEDIUM"
+}
+
+variable "scheduler" {
+ type = object({
+ cpu = string
+ memory_gb = number
+ storage_gb = number
+ count = number
+ })
+ default = {
+ cpu = 1
+ memory_gb = 4
+ storage_gb = 5
+ count = 2
+ }
+ description = "Configuration for resources used by Airflow schedulers."
+}
+
+variable "web_server" {
+ type = object({
+ cpu = string
+ memory_gb = number
+ storage_gb = number
+ })
+ default = {
+ cpu = 2
+ memory_gb = 7.5
+ storage_gb = 5
+ }
+ description = "Configuration for resources used by Airflow web server."
+}
+
+variable "worker" {
+ type = object({
+ cpu = string
+ memory_gb = number
+ storage_gb = number
+ min_count = number
+ max_count = number
+ })
+ default = {
+ cpu = 2
+ memory_gb = 7.5
+ storage_gb = 5
+ min_count = 2
+ max_count = 6
+ }
+ description = "Configuration for resources used by Airflow workers."
+}
+
+variable "triggerer" {
+ type = object({
+ cpu = string
+ memory_gb = number
+ count = number
+ })
+ default = null
+ description = " Configuration for resources used by Airflow triggerer"
+}
+
+variable "dag_processor" {
+ type = object({
+ cpu = string
+ memory_gb = number
+ storage_gb = number
+ count = number
+ })
+ default = {
+ cpu = 2
+ memory_gb = 7.5
+ storage_gb = 5
+ count = 2
+ }
+ description = "Configuration for resources used by Airflow workers."
+}
+
+variable "grant_sa_agent_permission" {
+ type = bool
+ default = true
+ description = "Cloud Composer relies on Workload Identity as Google API authentication mechanism for Airflow. "
+}
+
+variable "scheduled_snapshots_config" {
+ type = object({
+ enabled = optional(bool, false)
+ snapshot_location = optional(string)
+ snapshot_creation_schedule = optional(string)
+ time_zone = optional(string)
+ })
+ default = null
+ description = "The recovery configuration settings for the Cloud Composer environment"
+}
+
+variable "storage_bucket" {
+ description = "Name of an existing Cloud Storage bucket to be used by the environment"
+ type = string
+ default = null
+}
+
+variable "resilience_mode" {
+ description = "Cloud Composer 2.1.15 or newer only. The resilience mode states whether high resilience is enabled for the environment or not. Values for resilience mode are `HIGH_RESILIENCE` for high resilience and `STANDARD_RESILIENCE` for standard resilience"
+ type = string
+ default = null
+}
+
+variable "cloud_data_lineage_integration" {
+ description = "Whether or not Dataplex data lineage integration is enabled. Cloud Composer environments in versions composer-2.1.2-airflow-..* and newer)"
+ type = bool
+ default = false
+}
+
+variable "web_server_network_access_control" {
+ type = list(object({
+ allowed_ip_range = string
+ description = string
+ }))
+ default = null
+ description = "The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions are applied"
+}
+
+variable "kms_key_name" {
+ description = "Customer-managed Encryption Key fully qualified resource name, i.e. projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key."
+ type = string
+ default = null
+}
+
+variable "task_logs_retention_storage_mode" {
+ description = "The mode of storage for Airflow workers task logs. Values for storage mode are CLOUD_LOGGING_ONLY to only store logs in cloud logging and CLOUD_LOGGING_AND_CLOUD_STORAGE to store logs in cloud logging and cloud storage. Cloud Composer 2.0.23 or newer only"
+ type = string
+ default = null
+}
diff --git a/modules/create_environment_v3/versions.tf b/modules/create_environment_v3/versions.tf
new file mode 100644
index 00000000..0685edea
--- /dev/null
+++ b/modules/create_environment_v3/versions.tf
@@ -0,0 +1,40 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+ required_version = ">= 1.3"
+ required_providers {
+
+ google = {
+ source = "hashicorp/google"
+ version = ">= 6.0, < 7"
+ }
+
+ google-beta = {
+ source = "hashicorp/google-beta"
+ version = ">= 6.0, < 7"
+ }
+ }
+
+ provider_meta "google" {
+ module_name = "blueprints/terraform/terraform-google-composer:create_environment_v3/v6.0.0"
+ }
+
+ provider_meta "google-beta" {
+ module_name = "blueprints/terraform/terraform-google-composer:create_environment_v3/v6.0.0"
+ }
+
+}
diff --git a/test/integration/simple_composer_env_v3/simple_composer_env_v3_test.go b/test/integration/simple_composer_env_v3/simple_composer_env_v3_test.go
new file mode 100644
index 00000000..ed84fe1d
--- /dev/null
+++ b/test/integration/simple_composer_env_v3/simple_composer_env_v3_test.go
@@ -0,0 +1,40 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package simple_composer_env_v3
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
+ "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestSimpleComposerEnvV3Module(t *testing.T) {
+ composer := tft.NewTFBlueprintTest(t)
+
+ composer.DefineVerify(func(assert *assert.Assertions) {
+ composer.DefaultVerify(assert)
+
+ projectID := composer.GetStringOutput("project_id")
+
+ op := gcloud.Runf(t, "composer environments describe %s --project=%s --location=us-central1", composer.GetStringOutput("composer_env_name"), projectID)
+ assert.Equal(fmt.Sprintf("projects/%s/locations/us-central1/environments/%s", projectID, composer.GetStringOutput("composer_env_name")), op.Get("name").String(), "Composer name is valid")
+ assert.Equal(composer.GetStringOutput("airflow_uri"), op.Get("config.airflowUri").String(), "AirflowUri is valid")
+ assert.Equal(composer.GetStringOutput("gcs_bucket"), op.Get("config.dagGcsPrefix").String(), "GCS-Dag is valid")
+ })
+ composer.Test()
+}