diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 07b56b2a..dadf50cf 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -1,4 +1,4 @@ -# Copyright 2020 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,22 +25,17 @@ steps: - 'TF_VAR_folder_id=$_FOLDER_ID' - 'TF_VAR_billing_account=$_BILLING_ACCOUNT' -- id: create all +- id: init-all waitFor: - prepare name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create'] + args: ['/bin/bash', '-c', 'cft test run all --stage init --verbose'] # ----- SUITE simple-composer-env-v2 -- id: init-simple-composer-env-v2 - waitFor: - - create all - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage init --verbose'] - id: apply-simple-composer-env-v2 waitFor: - - init-simple-composer-env-v2 + - init-all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage apply --verbose'] - id: verify-simple-composer-env-v2 @@ -48,23 +43,12 @@ steps: - apply-simple-composer-env-v2 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose'] -- id: destroy-simple-composer-env-v2 - waitFor: - - verify-simple-composer-env-v2 - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose'] - # ----- SUITE composer-v2-sharedvpc-prereq-local -- id: init-composer-v2-sharedvpc-prereq - waitFor: - - destroy-simple-composer-env-v2 - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage init --verbose'] - id: apply-composer-v2-sharedvpc-prereq waitFor: - - init-composer-v2-sharedvpc-prereq + - init-all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage apply --verbose'] - id: verify-composer-v2-sharedvpc-prereq @@ -72,11 +56,37 @@ steps: - apply-composer-v2-sharedvpc-prereq name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage verify --verbose'] -- id: destroy-composer-v2-sharedvpc-prereq + + # ----- SUITE simple-composer-env-v3 + +- id: apply-simple-composer-env-v3 + waitFor: + - init-all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage apply --verbose'] +- id: verify-simple-composer-env-v3 + waitFor: + - apply-simple-composer-env-v3 + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage verify --verbose'] + +- id: destroy-v2 + waitFor: + - verify-simple-composer-env-v2 + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose'] + +- id: destroy-v2-sharedvpc-prereq waitFor: - verify-composer-v2-sharedvpc-prereq name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage destroy --verbose'] + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage verify --verbose'] + +- id: destroy-v3 + waitFor: + - verify-simple-composer-env-v3 + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage verify --verbose'] tags: - 'ci' diff --git a/examples/simple_composer_env_v3/README.md b/examples/simple_composer_env_v3/README.md new file mode 100644 index 00000000..d4b35fd4 --- /dev/null +++ b/examples/simple_composer_env_v3/README.md @@ -0,0 +1,33 @@ +# Simple Cloud Composer Environment (V3) Example + +This example illustrates how to use the `composer` V2 module to deploy private composer environment with private service connect (PSC) endpoint to connect network attachments. + +This example also creates a Cloud Storage Bucket for scheduled snapshots and assign appropriate permission(s) to Composer Service Account on the bucket. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| composer\_env\_name | Name of Cloud Composer Environment. | `string` | `"ci-composer"` | no | +| composer\_sa | Service Account to be used for running Cloud Composer Environment. | `string` | n/a | yes | +| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes | +| region | Region where Cloud Composer Environment is created. | `string` | `"us-central1"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| airflow\_uri | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. | +| composer\_env\_id | ID of Cloud Composer Environment. | +| composer\_env\_name | Name of the Cloud Composer Environment. | +| gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. | +| project\_id | Project ID where Cloud Composer Environment is created. | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_composer_env_v3/main.tf b/examples/simple_composer_env_v3/main.tf new file mode 100644 index 00000000..c7c08cdf --- /dev/null +++ b/examples/simple_composer_env_v3/main.tf @@ -0,0 +1,83 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "key_suffix" { + length = 5 + special = false + upper = false +} + +# Create a bucket to store the snapshots +resource "google_storage_bucket" "my_bucket" { + project = var.project_id + name = "snapshot-bucket-${random_string.key_suffix.result}" + location = var.region + force_destroy = true + uniform_bucket_level_access = true +} + +resource "google_storage_bucket_iam_member" "object_admin" { + bucket = google_storage_bucket.my_bucket.name + role = "roles/storage.objectAdmin" + member = "serviceAccount:${var.composer_sa}" +} + +module "simple-composer-environment" { + source = "terraform-google-modules/composer/google//modules/create_environment_v3" + version = "~> 6.0" + + project_id = var.project_id + composer_env_name = var.composer_env_name + region = var.region + composer_service_account = var.composer_sa + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + create_network_attachment = true + + grant_sa_agent_permission = false + environment_size = "ENVIRONMENT_SIZE_SMALL" + + use_private_environment = true + enable_private_builds_only = true + cloud_data_lineage_integration = true + resilience_mode = "STANDARD_RESILIENCE" + + scheduled_snapshots_config = { + enabled = true + snapshot_location = google_storage_bucket.my_bucket.url + snapshot_creation_schedule = "0 4 * * *" + time_zone = "UTC+01" + } + + maintenance_start_time = "2025-02-01T00:00:00Z" + maintenance_end_time = "2025-05-01T12:00:00Z" + maintenance_recurrence = "FREQ=WEEKLY;BYDAY=SU,SA" + + depends_on = [ + google_storage_bucket_iam_member.object_admin, + ] + + web_server_network_access_control = [ + { + allowed_ip_range = "192.0.2.0/24" + description = "office net 1" + }, + { + allowed_ip_range = "192.0.4.0/24" + description = "office net 2" + }, + ] +} diff --git a/examples/simple_composer_env_v3/network.tf b/examples/simple_composer_env_v3/network.tf new file mode 100644 index 00000000..c911d635 --- /dev/null +++ b/examples/simple_composer_env_v3/network.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_network" "main" { + project = var.project_id + name = "ci-composer-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + project = var.project_id + name = "ci-composer-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + private_ip_google_access = true +} + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} diff --git a/examples/simple_composer_env_v3/outputs.tf b/examples/simple_composer_env_v3/outputs.tf new file mode 100644 index 00000000..78223c33 --- /dev/null +++ b/examples/simple_composer_env_v3/outputs.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + description = "Project ID where Cloud Composer Environment is created." + value = var.project_id +} + +output "composer_env_name" { + description = "Name of the Cloud Composer Environment." + value = module.simple-composer-environment.composer_env_name +} + +output "composer_env_id" { + description = "ID of Cloud Composer Environment." + value = module.simple-composer-environment.composer_env_id +} + +output "gcs_bucket" { + description = "Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment." + value = module.simple-composer-environment.gcs_bucket +} + +output "airflow_uri" { + description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment." + value = module.simple-composer-environment.airflow_uri +} diff --git a/examples/simple_composer_env_v3/variables.tf b/examples/simple_composer_env_v3/variables.tf new file mode 100644 index 00000000..f8138770 --- /dev/null +++ b/examples/simple_composer_env_v3/variables.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project ID where Cloud Composer Environment is created." + type = string +} + +variable "composer_env_name" { + description = "Name of Cloud Composer Environment." + default = "ci-composer" + type = string +} + +variable "region" { + description = "Region where Cloud Composer Environment is created." + type = string + default = "us-central1" +} + +variable "composer_sa" { + description = "Service Account to be used for running Cloud Composer Environment." + type = string +} diff --git a/modules/create_environment_v3/README.md b/modules/create_environment_v3/README.md new file mode 100644 index 00000000..a4e8e751 --- /dev/null +++ b/modules/create_environment_v3/README.md @@ -0,0 +1,113 @@ +# Module Cloud Composer Environment ([v3](https://cloud.google.com/composer/docs/composer-3/composer-overview)) + +This module is used to create a Cloud Composer v3 environment. + +## Compatibility + +This module is meant for use with Terraform 1.3+ and tested using Terraform 1.3+. If you find incompatibilities using Terraform >=1.3, please open an issue. + + +```hcl +module "simple-composer-environment" { + source = "terraform-google-modules/composer/google//modules/create_environment_v3" + version = "~> 7.0" + project_id = var.project_id + composer_env_name = "test-composer-env" + region = "us-central1" + composer_service_account = var.composer_service_account + network = "test-vpc" + subnetwork = "test-subnet" + grant_sa_agent_permission = false + environment_size = "ENVIRONMENT_SIZE_SMALL" + use_private_environment = true + composer_network_attachment_name = "composer-na" + + scheduler = { + cpu = 0.5 + memory_gb = 1 + storage_gb = 1 + count = 2 + } + + dag_processor = { + cpu = 0.5 + memory_gb = 1 + storage_gb = 1 + count = 2 + } + + web_server = { + cpu = 0.5 + memory_gb = 1 + storage_gb = 1 + } + + worker = { + cpu = 0.5 + memory_gb = 1 + storage_gb = 1 + min_count = 2 + max_count = 3 + } + + triggerer = { + cpu = 1 + memory_gb = 1 + count = 2 + } +} + +``` + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| airflow\_config\_overrides | Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example "core-dags\_are\_paused\_at\_creation". | `map(string)` | `{}` | no | +| cloud\_data\_lineage\_integration | Whether or not Dataplex data lineage integration is enabled. Cloud Composer environments in versions composer-2.1.2-airflow-..* and newer) | `bool` | `false` | no | +| composer\_env\_name | Name of Cloud Composer Environment | `string` | n/a | yes | +| composer\_network\_attachment\_name | Name for PSC (Private Service Connect) Network entry point. | `string` | `null` | no | +| composer\_service\_account | Service Account for running Cloud Composer. | `string` | `null` | no | +| create\_network\_attachment | Either create a new network attachment or use existing one. If true, provide the subnet details. | `bool` | `true` | no | +| dag\_processor | Configuration for resources used by Airflow workers. |
object({
cpu = string
memory_gb = number
storage_gb = number
count = number
})
|
{
"count": 2,
"cpu": 2,
"memory_gb": 7.5,
"storage_gb": 5
}
| no | +| enable\_private\_builds\_only | If true, builds performed during operations that install Python packages have only private connectivity to Google services. If false, the builds also have access to the internet. | `bool` | `false` | no | +| env\_variables | Variables of the airflow environment. | `map(string)` | `{}` | no | +| environment\_size | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. Values for environment size are: `ENVIRONMENT_SIZE_SMALL`, `ENVIRONMENT_SIZE_MEDIUM`, and `ENVIRONMENT_SIZE_LARGE`. | `string` | `"ENVIRONMENT_SIZE_MEDIUM"` | no | +| grant\_sa\_agent\_permission | Cloud Composer relies on Workload Identity as Google API authentication mechanism for Airflow. | `bool` | `true` | no | +| image\_version | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-3-airflow-2.10.2-build.7"` | no | +| kms\_key\_name | Customer-managed Encryption Key fully qualified resource name, i.e. projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key. | `string` | `null` | no | +| labels | The resource labels (a map of key/value pairs) to be applied to the Cloud Composer. | `map(string)` | `{}` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `null` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `null` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| network | The VPC network to host the composer cluster. | `string` | n/a | yes | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes | +| pypi\_packages | Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. "numpy"). | `map(string)` | `{}` | no | +| region | Region where the Cloud Composer Environment is created. | `string` | `"us-central1"` | no | +| resilience\_mode | Cloud Composer 2.1.15 or newer only. The resilience mode states whether high resilience is enabled for the environment or not. Values for resilience mode are `HIGH_RESILIENCE` for high resilience and `STANDARD_RESILIENCE` for standard resilience | `string` | `null` | no | +| scheduled\_snapshots\_config | The recovery configuration settings for the Cloud Composer environment |
object({
enabled = optional(bool, false)
snapshot_location = optional(string)
snapshot_creation_schedule = optional(string)
time_zone = optional(string)
})
| `null` | no | +| scheduler | Configuration for resources used by Airflow schedulers. |
object({
cpu = string
memory_gb = number
storage_gb = number
count = number
})
|
{
"count": 2,
"cpu": 1,
"memory_gb": 4,
"storage_gb": 5
}
| no | +| storage\_bucket | Name of an existing Cloud Storage bucket to be used by the environment | `string` | `null` | no | +| subnetwork | The name of the subnetwork to host the composer cluster. | `string` | n/a | yes | +| subnetwork\_region | The subnetwork region of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| tags | Tags applied to all nodes. Tags are used to identify valid sources or targets for network firewalls. | `set(string)` | `[]` | no | +| task\_logs\_retention\_storage\_mode | The mode of storage for Airflow workers task logs. Values for storage mode are CLOUD\_LOGGING\_ONLY to only store logs in cloud logging and CLOUD\_LOGGING\_AND\_CLOUD\_STORAGE to store logs in cloud logging and cloud storage. Cloud Composer 2.0.23 or newer only | `string` | `null` | no | +| triggerer | Configuration for resources used by Airflow triggerer |
object({
cpu = string
memory_gb = number
count = number
})
| `null` | no | +| use\_private\_environment | Create a private environment. If true, a private Composer environment will be created. | `bool` | `false` | no | +| web\_server | Configuration for resources used by Airflow web server. |
object({
cpu = string
memory_gb = number
storage_gb = number
})
|
{
"cpu": 2,
"memory_gb": 7.5,
"storage_gb": 5
}
| no | +| web\_server\_network\_access\_control | The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions are applied |
list(object({
allowed_ip_range = string
description = string
}))
| `null` | no | +| web\_server\_plugins\_mode | Web server plugins configuration. Can be either 'ENABLED' or 'DISABLED'. Defaults to 'ENABLED'. | `string` | `"ENABLED"` | no | +| worker | Configuration for resources used by Airflow workers. |
object({
cpu = string
memory_gb = number
storage_gb = number
min_count = number
max_count = number
})
|
{
"cpu": 2,
"max_count": 6,
"memory_gb": 7.5,
"min_count": 2,
"storage_gb": 5
}
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| airflow\_uri | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. | +| composer\_env | Cloud Composer Environment | +| composer\_env\_id | ID of Cloud Composer Environment. | +| composer\_env\_name | Name of the Cloud Composer Environment. | +| gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. | + + diff --git a/modules/create_environment_v3/iam.tf b/modules/create_environment_v3/iam.tf new file mode 100644 index 00000000..e09dc52f --- /dev/null +++ b/modules/create_environment_v3/iam.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_project" "project" { + project_id = var.project_id +} + +resource "google_project_iam_member" "composer_agent_service_account" { + count = var.grant_sa_agent_permission ? 1 : 0 + project = data.google_project.project.project_id + role = "roles/composer.ServiceAgentV2Ext" + member = format("serviceAccount:%s", local.cloud_composer_sa) +} diff --git a/modules/create_environment_v3/main.tf b/modules/create_environment_v3/main.tf new file mode 100644 index 00000000..b01f1c82 --- /dev/null +++ b/modules/create_environment_v3/main.tf @@ -0,0 +1,192 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id + subnetwork_region = var.subnetwork_region != "" ? var.subnetwork_region : var.region + cloud_composer_sa = format("service-%s@cloudcomposer-accounts.iam.gserviceaccount.com", data.google_project.project.number) +} + +resource "google_composer_environment" "composer_env" { + provider = google-beta + + project = var.project_id + name = var.composer_env_name + region = var.region + labels = var.labels + + dynamic "storage_config" { + for_each = var.storage_bucket != null ? ["storage_config"] : [] + content { + bucket = var.storage_bucket + } + } + + config { + + enable_private_environment = var.use_private_environment # reusing the existing variable name from previous versions + enable_private_builds_only = var.enable_private_builds_only + + environment_size = var.environment_size + resilience_mode = var.resilience_mode + + node_config { + service_account = var.composer_service_account + tags = var.tags + network = var.create_network_attachment ? "projects/${local.network_project_id}/global/networks/${var.network}" : null + subnetwork = var.create_network_attachment ? "projects/${local.network_project_id}/regions/${local.subnetwork_region}/subnetworks/${var.subnetwork}" : null + composer_network_attachment = var.create_network_attachment ? null : "projects/{var.project_id}/regions/${var.region}/networkAttachments/${var.composer_network_attachment_name}" + } + + dynamic "software_config" { + for_each = [ + { + airflow_config_overrides = var.airflow_config_overrides + pypi_packages = var.pypi_packages + env_variables = var.env_variables + image_version = var.image_version + web_server_plugins_mode = var.web_server_plugins_mode + }] + content { + airflow_config_overrides = software_config.value["airflow_config_overrides"] + pypi_packages = software_config.value["pypi_packages"] + env_variables = software_config.value["env_variables"] + image_version = software_config.value["image_version"] + web_server_plugins_mode = software_config.value["web_server_plugins_mode"] + dynamic "cloud_data_lineage_integration" { + for_each = var.cloud_data_lineage_integration ? ["cloud_data_lineage_integration"] : [] + content { + enabled = var.cloud_data_lineage_integration + } + } + } + } + + dynamic "maintenance_window" { + for_each = (var.maintenance_end_time != null && var.maintenance_recurrence != null) ? [ + { + start_time = var.maintenance_start_time + end_time = var.maintenance_end_time + recurrence = var.maintenance_recurrence + }] : [] + content { + start_time = maintenance_window.value["start_time"] + end_time = maintenance_window.value["end_time"] + recurrence = maintenance_window.value["recurrence"] + } + } + + workloads_config { + + dynamic "scheduler" { + for_each = var.scheduler != null ? [var.scheduler] : [] + content { + cpu = scheduler.value["cpu"] + memory_gb = scheduler.value["memory_gb"] + storage_gb = scheduler.value["storage_gb"] + count = scheduler.value["count"] + } + } + + dynamic "web_server" { + for_each = var.web_server != null ? [var.web_server] : [] + content { + cpu = web_server.value["cpu"] + memory_gb = web_server.value["memory_gb"] + storage_gb = web_server.value["storage_gb"] + } + } + + dynamic "worker" { + for_each = var.worker != null ? [var.worker] : [] + content { + cpu = worker.value["cpu"] + memory_gb = worker.value["memory_gb"] + storage_gb = worker.value["storage_gb"] + min_count = worker.value["min_count"] + max_count = worker.value["max_count"] + } + } + + dynamic "triggerer" { + for_each = var.triggerer != null ? [var.triggerer] : [] + content { + cpu = triggerer.value["cpu"] + memory_gb = triggerer.value["memory_gb"] + count = triggerer.value["count"] + } + } + + dynamic "dag_processor" { + for_each = var.dag_processor != null ? [var.dag_processor] : [] + content { + cpu = dag_processor.value["cpu"] + memory_gb = dag_processor.value["memory_gb"] + storage_gb = dag_processor.value["storage_gb"] + count = dag_processor.value["count"] + } + } + + } + + dynamic "recovery_config" { + for_each = var.scheduled_snapshots_config != null ? ["recovery_config"] : [] + content { + dynamic "scheduled_snapshots_config" { + for_each = var.scheduled_snapshots_config != null ? [var.scheduled_snapshots_config] : [] + content { + enabled = scheduled_snapshots_config.value["enabled"] + snapshot_location = scheduled_snapshots_config.value["snapshot_location"] + snapshot_creation_schedule = scheduled_snapshots_config.value["snapshot_creation_schedule"] + time_zone = scheduled_snapshots_config.value["time_zone"] + } + } + } + } + + dynamic "web_server_network_access_control" { + for_each = var.web_server_network_access_control == null ? [] : ["web_server_network_access_control"] + content { + dynamic "allowed_ip_range" { + for_each = { for x in var.web_server_network_access_control : x.allowed_ip_range => x } + content { + value = allowed_ip_range.value["allowed_ip_range"] + description = allowed_ip_range.value["description"] + } + } + } + } + + dynamic "encryption_config" { + for_each = var.kms_key_name != null ? ["encryption_config"] : [] + content { + kms_key_name = var.kms_key_name + } + } + + dynamic "data_retention_config" { + for_each = var.task_logs_retention_storage_mode == null ? [] : ["data_retention_config"] + content { + task_logs_retention_config { + storage_mode = var.task_logs_retention_storage_mode + } + } + } + } + + depends_on = [google_project_iam_member.composer_agent_service_account] + +} diff --git a/modules/create_environment_v3/outputs.tf b/modules/create_environment_v3/outputs.tf new file mode 100644 index 00000000..41d33bc0 --- /dev/null +++ b/modules/create_environment_v3/outputs.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "composer_env_name" { + value = google_composer_environment.composer_env.name + description = "Name of the Cloud Composer Environment." +} + +output "composer_env_id" { + value = google_composer_environment.composer_env.id + description = "ID of Cloud Composer Environment." +} + +output "gcs_bucket" { + value = google_composer_environment.composer_env.config[0].dag_gcs_prefix + description = "Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment." +} + +output "airflow_uri" { + value = google_composer_environment.composer_env.config[0].airflow_uri + description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment." +} + +output "composer_env" { + value = google_composer_environment.composer_env + description = "Cloud Composer Environment" +} diff --git a/modules/create_environment_v3/variables.tf b/modules/create_environment_v3/variables.tf new file mode 100644 index 00000000..6f72e6bd --- /dev/null +++ b/modules/create_environment_v3/variables.tf @@ -0,0 +1,279 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project ID where Cloud Composer Environment is created." + type = string +} + +variable "composer_env_name" { + description = "Name of Cloud Composer Environment" + type = string +} + +variable "region" { + description = "Region where the Cloud Composer Environment is created." + type = string + default = "us-central1" +} + +variable "labels" { + type = map(string) + description = "The resource labels (a map of key/value pairs) to be applied to the Cloud Composer." + default = {} +} + +variable "tags" { + description = "Tags applied to all nodes. Tags are used to identify valid sources or targets for network firewalls." + type = set(string) + default = [] +} + +variable "network" { + type = string + description = "The VPC network to host the composer cluster." +} + +variable "network_project_id" { + type = string + description = "The project ID of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "subnetwork" { + type = string + description = "The name of the subnetwork to host the composer cluster." +} + +variable "subnetwork_region" { + type = string + description = "The subnetwork region of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "create_network_attachment" { + type = bool + description = "Either create a new network attachment or use existing one. If true, provide the subnet details." + default = true +} + +variable "composer_network_attachment_name" { + type = string + description = "Name for PSC (Private Service Connect) Network entry point." + default = null +} + +variable "composer_service_account" { + description = "Service Account for running Cloud Composer." + type = string + default = null +} + +variable "airflow_config_overrides" { + type = map(string) + description = "Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example \"core-dags_are_paused_at_creation\"." + default = {} +} + +variable "env_variables" { + type = map(string) + description = "Variables of the airflow environment." + default = {} +} + +variable "image_version" { + type = string + description = "The version of the aiflow running in the cloud composer environment." + default = "composer-3-airflow-2.10.2-build.7" +} + +variable "web_server_plugins_mode" { + type = string + description = "Web server plugins configuration. Can be either 'ENABLED' or 'DISABLED'. Defaults to 'ENABLED'." + default = "ENABLED" +} + +variable "pypi_packages" { + type = map(string) + description = " Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. \"numpy\")." + default = {} +} + +variable "use_private_environment" { + description = "Create a private environment. If true, a private Composer environment will be created." + type = bool + default = false +} + +variable "enable_private_builds_only" { + type = bool + description = "If true, builds performed during operations that install Python packages have only private connectivity to Google services. If false, the builds also have access to the internet." + default = false +} + +variable "maintenance_start_time" { + description = "Time window specified for daily or recurring maintenance operations in RFC3339 format" + type = string + default = "05:00" +} + +variable "maintenance_end_time" { + description = "Time window specified for recurring maintenance operations in RFC3339 format" + type = string + default = null +} + +variable "maintenance_recurrence" { + description = "Frequency of the recurring maintenance window in RFC5545 format." + type = string + default = null +} + +variable "environment_size" { + type = string + description = "The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. Values for environment size are: `ENVIRONMENT_SIZE_SMALL`, `ENVIRONMENT_SIZE_MEDIUM`, and `ENVIRONMENT_SIZE_LARGE`." + default = "ENVIRONMENT_SIZE_MEDIUM" +} + +variable "scheduler" { + type = object({ + cpu = string + memory_gb = number + storage_gb = number + count = number + }) + default = { + cpu = 1 + memory_gb = 4 + storage_gb = 5 + count = 2 + } + description = "Configuration for resources used by Airflow schedulers." +} + +variable "web_server" { + type = object({ + cpu = string + memory_gb = number + storage_gb = number + }) + default = { + cpu = 2 + memory_gb = 7.5 + storage_gb = 5 + } + description = "Configuration for resources used by Airflow web server." +} + +variable "worker" { + type = object({ + cpu = string + memory_gb = number + storage_gb = number + min_count = number + max_count = number + }) + default = { + cpu = 2 + memory_gb = 7.5 + storage_gb = 5 + min_count = 2 + max_count = 6 + } + description = "Configuration for resources used by Airflow workers." +} + +variable "triggerer" { + type = object({ + cpu = string + memory_gb = number + count = number + }) + default = null + description = " Configuration for resources used by Airflow triggerer" +} + +variable "dag_processor" { + type = object({ + cpu = string + memory_gb = number + storage_gb = number + count = number + }) + default = { + cpu = 2 + memory_gb = 7.5 + storage_gb = 5 + count = 2 + } + description = "Configuration for resources used by Airflow workers." +} + +variable "grant_sa_agent_permission" { + type = bool + default = true + description = "Cloud Composer relies on Workload Identity as Google API authentication mechanism for Airflow. " +} + +variable "scheduled_snapshots_config" { + type = object({ + enabled = optional(bool, false) + snapshot_location = optional(string) + snapshot_creation_schedule = optional(string) + time_zone = optional(string) + }) + default = null + description = "The recovery configuration settings for the Cloud Composer environment" +} + +variable "storage_bucket" { + description = "Name of an existing Cloud Storage bucket to be used by the environment" + type = string + default = null +} + +variable "resilience_mode" { + description = "Cloud Composer 2.1.15 or newer only. The resilience mode states whether high resilience is enabled for the environment or not. Values for resilience mode are `HIGH_RESILIENCE` for high resilience and `STANDARD_RESILIENCE` for standard resilience" + type = string + default = null +} + +variable "cloud_data_lineage_integration" { + description = "Whether or not Dataplex data lineage integration is enabled. Cloud Composer environments in versions composer-2.1.2-airflow-..* and newer)" + type = bool + default = false +} + +variable "web_server_network_access_control" { + type = list(object({ + allowed_ip_range = string + description = string + })) + default = null + description = "The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions are applied" +} + +variable "kms_key_name" { + description = "Customer-managed Encryption Key fully qualified resource name, i.e. projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key." + type = string + default = null +} + +variable "task_logs_retention_storage_mode" { + description = "The mode of storage for Airflow workers task logs. Values for storage mode are CLOUD_LOGGING_ONLY to only store logs in cloud logging and CLOUD_LOGGING_AND_CLOUD_STORAGE to store logs in cloud logging and cloud storage. Cloud Composer 2.0.23 or newer only" + type = string + default = null +} diff --git a/modules/create_environment_v3/versions.tf b/modules/create_environment_v3/versions.tf new file mode 100644 index 00000000..0685edea --- /dev/null +++ b/modules/create_environment_v3/versions.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.3" + required_providers { + + google = { + source = "hashicorp/google" + version = ">= 6.0, < 7" + } + + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.0, < 7" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/terraform-google-composer:create_environment_v3/v6.0.0" + } + + provider_meta "google-beta" { + module_name = "blueprints/terraform/terraform-google-composer:create_environment_v3/v6.0.0" + } + +} diff --git a/test/integration/simple_composer_env_v3/simple_composer_env_v3_test.go b/test/integration/simple_composer_env_v3/simple_composer_env_v3_test.go new file mode 100644 index 00000000..ed84fe1d --- /dev/null +++ b/test/integration/simple_composer_env_v3/simple_composer_env_v3_test.go @@ -0,0 +1,40 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package simple_composer_env_v3 + +import ( + "fmt" + "testing" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" +) + +func TestSimpleComposerEnvV3Module(t *testing.T) { + composer := tft.NewTFBlueprintTest(t) + + composer.DefineVerify(func(assert *assert.Assertions) { + composer.DefaultVerify(assert) + + projectID := composer.GetStringOutput("project_id") + + op := gcloud.Runf(t, "composer environments describe %s --project=%s --location=us-central1", composer.GetStringOutput("composer_env_name"), projectID) + assert.Equal(fmt.Sprintf("projects/%s/locations/us-central1/environments/%s", projectID, composer.GetStringOutput("composer_env_name")), op.Get("name").String(), "Composer name is valid") + assert.Equal(composer.GetStringOutput("airflow_uri"), op.Get("config.airflowUri").String(), "AirflowUri is valid") + assert.Equal(composer.GetStringOutput("gcs_bucket"), op.Get("config.dagGcsPrefix").String(), "GCS-Dag is valid") + }) + composer.Test() +}