feat: fixed composer 3 env

Author: prabhu34

build/int.cloudbuild.yaml

@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2025 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,22 +25,17 @@ steps:
   - 'TF_VAR_folder_id=$_FOLDER_ID'
   - 'TF_VAR_billing_account=$_BILLING_ACCOUNT'
 
-- id: create all
+- id: init-all
   waitFor:
     - prepare
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create']
+  args: ['/bin/bash', '-c', 'cft test run all --stage init --verbose']
 
   # ----- SUITE simple-composer-env-v2
 
-- id: init-simple-composer-env-v2
-  waitFor:
-    - create all
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage init --verbose']
 - id: apply-simple-composer-env-v2
   waitFor:
-    - init-simple-composer-env-v2
+    - init-all
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage apply --verbose']
 - id: verify-simple-composer-env-v2
@@ -54,17 +49,12 @@ steps:
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose']
 
-
 # ----- SUITE composer-v2-sharedvpc-prereq-local
 
-- id: init-composer-v2-sharedvpc-prereq
-  waitFor:
-    - destroy-simple-composer-env-v2
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage init --verbose']
 - id: apply-composer-v2-sharedvpc-prereq
   waitFor:
-    - init-composer-v2-sharedvpc-prereq
+    - init-all
+    - destroy-simple-composer-env-v2
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage apply --verbose']
 - id: verify-composer-v2-sharedvpc-prereq
@@ -78,30 +68,23 @@ steps:
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage destroy --verbose']
 
-
   # ----- SUITE simple-composer-env-v3
 
-- id: init-simple-composer-env-v3
-  waitFor:
-    - create all
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage init --verbose']
 - id: apply-simple-composer-env-v3
   waitFor:
-    - init-simple-composer-env-v3
+    - init-all
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage apply --verbose']
-- id: verify-simple-composer-env-v3
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage apply --verbose']
+- id: verify-simple-composer-env-v2
   waitFor:
     - apply-simple-composer-env-v3
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage verify --verbose']
-- id: destroy-simple-composer-env-v3
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose']
+- id: destroy-simple-composer-env-v2
   waitFor:
     - verify-simple-composer-env-v3
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage destroy --verbose']
-
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose']
 
 tags:
 - 'ci'

examples/simple_composer_env_v3/main.tf

@@ -51,43 +51,10 @@ module "simple-composer-environment" {
   environment_size          = "ENVIRONMENT_SIZE_SMALL"
 
   use_private_environment        = true
+  enable_private_builds_only     = true
   cloud_data_lineage_integration = true
   resilience_mode                = "STANDARD_RESILIENCE"
 
-  scheduler = {
-    cpu        = 0.5
-    memory_gb  = 1
-    storage_gb = 1
-    count      = 2
-  }
-
-  dag_processor = {
-    cpu        = 0.5
-    memory_gb  = 1
-    storage_gb = 1
-    count      = 2
-  }
-
-  web_server = {
-    cpu        = 0.5
-    memory_gb  = 1
-    storage_gb = 1
-  }
-
-  worker = {
-    cpu        = 0.5
-    memory_gb  = 1
-    storage_gb = 1
-    min_count  = 2
-    max_count  = 3
-  }
-
-  triggerer = {
-    cpu       = 1
-    memory_gb = 1
-    count     = 2
-  }
-
   scheduled_snapshots_config = {
     enabled                    = true
     snapshot_location          = google_storage_bucket.my_bucket.url
@@ -96,7 +63,7 @@ module "simple-composer-environment" {
   }
 
   maintenance_start_time = "2025-02-01T00:00:00Z"
-  maintenance_end_time   = "2025-03-01T12:00:00Z"
+  maintenance_end_time   = "2025-05-01T12:00:00Z"
   maintenance_recurrence = "FREQ=WEEKLY;BYDAY=SU,SA"
 
   depends_on = [

modules/create_environment_v3/README.md

@@ -70,10 +70,11 @@ module "simple-composer-environment" {
 | composer\_service\_account | Service Account for running Cloud Composer. | `string` | `null` | no |
 | create\_network\_attachment | Either create a new network attachment or use existing one. If true, provide the subnet details. | `bool` | `true` | no |
 | dag\_processor | Configuration for resources used by Airflow workers. | <pre>object({<br>    cpu        = string<br>    memory_gb  = number<br>    storage_gb = number<br>    count      = number<br>  })</pre> | <pre>{<br>  "count": 2,<br>  "cpu": 2,<br>  "memory_gb": 7.5,<br>  "storage_gb": 5<br>}</pre> | no |
+| enable\_private\_builds\_only | If true, builds performed during operations that install Python packages have only private connectivity to Google services. If false, the builds also have access to the internet. | `bool` | `false` | no |
 | env\_variables | Variables of the airflow environment. | `map(string)` | `{}` | no |
 | environment\_size | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. Values for environment size are: `ENVIRONMENT_SIZE_SMALL`, `ENVIRONMENT_SIZE_MEDIUM`, and `ENVIRONMENT_SIZE_LARGE`. | `string` | `"ENVIRONMENT_SIZE_MEDIUM"` | no |
 | grant\_sa\_agent\_permission | Cloud Composer relies on Workload Identity as Google API authentication mechanism for Airflow. | `bool` | `true` | no |
-| image\_version | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-3-airflow-2.10.2-build.5"` | no |
+| image\_version | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-3-airflow-2.10.2-build.7"` | no |
 | kms\_key\_name | Customer-managed Encryption Key fully qualified resource name, i.e. projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key. | `string` | `null` | no |
 | labels | The resource labels (a map of key/value pairs) to be applied to the Cloud Composer. | `map(string)` | `{}` | no |
 | maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `null` | no |

modules/create_environment_v3/main.tf

@@ -38,6 +38,7 @@ resource "google_composer_environment" "composer_env" {
   config {
 
     enable_private_environment = var.use_private_environment # reusing the existing variable name from previous versions
+    enable_private_builds_only = var.enable_private_builds_only
 
     environment_size = var.environment_size
     resilience_mode  = var.resilience_mode

modules/create_environment_v3/variables.tf

@@ -97,7 +97,7 @@ variable "env_variables" {
 variable "image_version" {
   type        = string
   description = "The version of the aiflow running in the cloud composer environment."
-  default     = "composer-3-airflow-2.10.2-build.5"
+  default     = "composer-3-airflow-2.10.2-build.7"
 }
 
 variable "web_server_plugins_mode" {
@@ -118,6 +118,12 @@ variable "use_private_environment" {
   default     = false
 }
 
+variable "enable_private_builds_only" {
+  type        = bool
+  description = "If true, builds performed during operations that install Python packages have only private connectivity to Google services. If false, the builds also have access to the internet."
+  default     = false
+}
+
 variable "maintenance_start_time" {
   description = "Time window specified for daily or recurring maintenance operations in RFC3339 format"
   type        = string