feat: Add Composer v3 sub-module

build/int.cloudbuild.yaml

@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2025 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,58 +25,68 @@ steps:
   - 'TF_VAR_folder_id=$_FOLDER_ID'
   - 'TF_VAR_billing_account=$_BILLING_ACCOUNT'
 
-- id: create all
+- id: init-all
   waitFor:
     - prepare
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create']
+  args: ['/bin/bash', '-c', 'cft test run all --stage init --verbose']
 
   # ----- SUITE simple-composer-env-v2
 
-- id: init-simple-composer-env-v2
-  waitFor:
-    - create all
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage init --verbose']
 - id: apply-simple-composer-env-v2
   waitFor:
-    - init-simple-composer-env-v2
+    - init-all
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage apply --verbose']
 - id: verify-simple-composer-env-v2
   waitFor:
     - apply-simple-composer-env-v2
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose']
-- id: destroy-simple-composer-env-v2
-  waitFor:
-    - verify-simple-composer-env-v2
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose']
-
 
 # ----- SUITE composer-v2-sharedvpc-prereq-local
 
-- id: init-composer-v2-sharedvpc-prereq
-  waitFor:
-    - destroy-simple-composer-env-v2
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage init --verbose']
 - id: apply-composer-v2-sharedvpc-prereq
   waitFor:
-    - init-composer-v2-sharedvpc-prereq
+    - init-all
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage apply --verbose']
 - id: verify-composer-v2-sharedvpc-prereq
   waitFor:
     - apply-composer-v2-sharedvpc-prereq
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage verify --verbose']
-- id: destroy-composer-v2-sharedvpc-prereq
+
+  # ----- SUITE simple-composer-env-v3
+
+- id: apply-simple-composer-env-v3
+  waitFor:
+    - init-all
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage apply --verbose']
+- id: verify-simple-composer-env-v3
+  waitFor:
+    - apply-simple-composer-env-v3
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage verify --verbose']
+
+- id: destroy-v2
+  waitFor:
+    - verify-simple-composer-env-v2
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose']
+
+- id: destroy-v2-sharedvpc-prereq
   waitFor:
     - verify-composer-v2-sharedvpc-prereq
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage destroy --verbose']
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage verify --verbose']
+
+- id: destroy-v3
+  waitFor:
+    - verify-simple-composer-env-v3
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage verify --verbose']
 
 tags:
 - 'ci'

examples/simple_composer_env_v3/README.md

@@ -0,0 +1,33 @@
+# Simple Cloud Composer Environment (V3) Example
+
+This example illustrates how to use the `composer` V2 module to deploy private composer environment with private service connect (PSC) endpoint to connect network attachments.
+
+This example also creates a Cloud Storage Bucket for scheduled snapshots and assign appropriate permission(s) to Composer Service Account on the bucket.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| composer\_env\_name | Name of Cloud Composer Environment. | `string` | `"ci-composer"` | no |
+| composer\_sa | Service Account to be used for running Cloud Composer Environment. | `string` | n/a | yes |
+| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes |
+| region | Region where Cloud Composer Environment is created. | `string` | `"us-central1"` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| airflow\_uri | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. |
+| composer\_env\_id | ID of Cloud Composer Environment. |
+| composer\_env\_name | Name of the Cloud Composer Environment. |
+| gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. |
+| project\_id | Project ID where Cloud Composer Environment is created. |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+To provision this example, run the following from within this directory:
+- `terraform init` to get the plugins
+- `terraform plan` to see the infrastructure plan
+- `terraform apply` to apply the infrastructure build
+- `terraform destroy` to destroy the built infrastructure

examples/simple_composer_env_v3/main.tf

@@ -0,0 +1,83 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "random_string" "key_suffix" {
+  length  = 5
+  special = false
+  upper   = false
+}
+
+# Create a bucket to store the snapshots
+resource "google_storage_bucket" "my_bucket" {
+  project                     = var.project_id
+  name                        = "snapshot-bucket-${random_string.key_suffix.result}"
+  location                    = var.region
+  force_destroy               = true
+  uniform_bucket_level_access = true
+}
+
+resource "google_storage_bucket_iam_member" "object_admin" {
+  bucket = google_storage_bucket.my_bucket.name
+  role   = "roles/storage.objectAdmin"
+  member = "serviceAccount:${var.composer_sa}"
+}
+
+module "simple-composer-environment" {
+  source  = "terraform-google-modules/composer/google//modules/create_environment_v3"
+  version = "~> 6.0"
+
+  project_id                = var.project_id
+  composer_env_name         = var.composer_env_name
+  region                    = var.region
+  composer_service_account  = var.composer_sa
+  network                   = google_compute_network.main.name
+  subnetwork                = google_compute_subnetwork.main.name
+  create_network_attachment = true
+
+  grant_sa_agent_permission = false
+  environment_size          = "ENVIRONMENT_SIZE_SMALL"
+
+  use_private_environment        = true
+  enable_private_builds_only     = true
+  cloud_data_lineage_integration = true
+  resilience_mode                = "STANDARD_RESILIENCE"
+
+  scheduled_snapshots_config = {
+    enabled                    = true
+    snapshot_location          = google_storage_bucket.my_bucket.url
+    snapshot_creation_schedule = "0 4 * * *"
+    time_zone                  = "UTC+01"
+  }
+
+  maintenance_start_time = "2025-02-01T00:00:00Z"
+  maintenance_end_time   = "2025-05-01T12:00:00Z"
+  maintenance_recurrence = "FREQ=WEEKLY;BYDAY=SU,SA"
+
+  depends_on = [
+    google_storage_bucket_iam_member.object_admin,
+  ]
+
+  web_server_network_access_control = [
+    {
+      allowed_ip_range = "192.0.2.0/24"
+      description      = "office net 1"
+    },
+    {
+      allowed_ip_range = "192.0.4.0/24"
+      description      = "office net 2"
+    },
+  ]
+}

examples/simple_composer_env_v3/network.tf

@@ -0,0 +1,36 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "google_compute_network" "main" {
+  project                 = var.project_id
+  name                    = "ci-composer-test-${random_string.suffix.result}"
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "main" {
+  project                  = var.project_id
+  name                     = "ci-composer-test-${random_string.suffix.result}"
+  ip_cidr_range            = "10.0.0.0/17"
+  region                   = var.region
+  network                  = google_compute_network.main.self_link
+  private_ip_google_access = true
+}
+
+resource "random_string" "suffix" {
+  length  = 4
+  special = false
+  upper   = false
+}

examples/simple_composer_env_v3/outputs.tf

@@ -0,0 +1,40 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "project_id" {
+  description = "Project ID where Cloud Composer Environment is created."
+  value       = var.project_id
+}
+
+output "composer_env_name" {
+  description = "Name of the Cloud Composer Environment."
+  value       = module.simple-composer-environment.composer_env_name
+}
+
+output "composer_env_id" {
+  description = "ID of Cloud Composer Environment."
+  value       = module.simple-composer-environment.composer_env_id
+}
+
+output "gcs_bucket" {
+  description = "Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment."
+  value       = module.simple-composer-environment.gcs_bucket
+}
+
+output "airflow_uri" {
+  description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment."
+  value       = module.simple-composer-environment.airflow_uri
+}

examples/simple_composer_env_v3/variables.tf

@@ -0,0 +1,37 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "Project ID where Cloud Composer Environment is created."
+  type        = string
+}
+
+variable "composer_env_name" {
+  description = "Name of Cloud Composer Environment."
+  default     = "ci-composer"
+  type        = string
+}
+
+variable "region" {
+  description = "Region where Cloud Composer Environment is created."
+  type        = string
+  default     = "us-central1"
+}
+
+variable "composer_sa" {
+  description = "Service Account to be used for running Cloud Composer Environment."
+  type        = string
+}