Skip to content

Commit 444aa02

Browse files
prabhu34prabhu34
authored
feat: Add Composer v3 sub-module (#155)
1 parent ba0eb61 commit 444aa02

File tree

13 files changed

+992
-23
lines changed

13 files changed

+992
-23
lines changed

‎build/int.cloudbuild.yaml‎

Lines changed: 33 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2020 Google LLC
1+
# Copyright 2025 Google LLC
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -25,58 +25,68 @@ steps:
2525
- 'TF_VAR_folder_id=$_FOLDER_ID'
2626
- 'TF_VAR_billing_account=$_BILLING_ACCOUNT'
2727

28-
- id: create all
28+
- id: init-all
2929
waitFor:
3030
- prepare
3131
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
32-
args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create']
32+
args: ['/bin/bash', '-c', 'cft test run all --stage init --verbose']
3333

3434
# ----- SUITE simple-composer-env-v2
3535

36-
- id: init-simple-composer-env-v2
37-
waitFor:
38-
- create all
39-
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
40-
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage init --verbose']
4136
- id: apply-simple-composer-env-v2
4237
waitFor:
43-
- init-simple-composer-env-v2
38+
- init-all
4439
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
4540
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage apply --verbose']
4641
- id: verify-simple-composer-env-v2
4742
waitFor:
4843
- apply-simple-composer-env-v2
4944
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
5045
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose']
51-
- id: destroy-simple-composer-env-v2
52-
waitFor:
53-
- verify-simple-composer-env-v2
54-
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
55-
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose']
56-
5746

5847
# ----- SUITE composer-v2-sharedvpc-prereq-local
5948

60-
- id: init-composer-v2-sharedvpc-prereq
61-
waitFor:
62-
- destroy-simple-composer-env-v2
63-
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
64-
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage init --verbose']
6549
- id: apply-composer-v2-sharedvpc-prereq
6650
waitFor:
67-
- init-composer-v2-sharedvpc-prereq
51+
- init-all
6852
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
6953
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage apply --verbose']
7054
- id: verify-composer-v2-sharedvpc-prereq
7155
waitFor:
7256
- apply-composer-v2-sharedvpc-prereq
7357
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
7458
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage verify --verbose']
75-
- id: destroy-composer-v2-sharedvpc-prereq
59+
60+
# ----- SUITE simple-composer-env-v3
61+
62+
- id: apply-simple-composer-env-v3
63+
waitFor:
64+
- init-all
65+
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
66+
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage apply --verbose']
67+
- id: verify-simple-composer-env-v3
68+
waitFor:
69+
- apply-simple-composer-env-v3
70+
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
71+
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage verify --verbose']
72+
73+
- id: destroy-v2
74+
waitFor:
75+
- verify-simple-composer-env-v2
76+
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
77+
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose']
78+
79+
- id: destroy-v2-sharedvpc-prereq
7680
waitFor:
7781
- verify-composer-v2-sharedvpc-prereq
7882
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
79-
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage destroy --verbose']
83+
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage verify --verbose']
84+
85+
- id: destroy-v3
86+
waitFor:
87+
- verify-simple-composer-env-v3
88+
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
89+
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV3Module --stage verify --verbose']
8090

8191
tags:
8292
- 'ci'

‎examples/simple_composer_env_v3/README.md‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# Simple Cloud Composer Environment (V3) Example
2+
3+
This example illustrates how to use the `composer` V2 module to deploy private composer environment with private service connect (PSC) endpoint to connect network attachments.
4+
5+
This example also creates a Cloud Storage Bucket for scheduled snapshots and assign appropriate permission(s) to Composer Service Account on the bucket.
6+
7+
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
8+
## Inputs
9+
10+
| Name | Description | Type | Default | Required |
11+
|------|-------------|------|---------|:--------:|
12+
| composer\_env\_name | Name of Cloud Composer Environment. | `string` | `"ci-composer"` | no |
13+
| composer\_sa | Service Account to be used for running Cloud Composer Environment. | `string` | n/a | yes |
14+
| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes |
15+
| region | Region where Cloud Composer Environment is created. | `string` | `"us-central1"` | no |
16+
17+
## Outputs
18+
19+
| Name | Description |
20+
|------|-------------|
21+
| airflow\_uri | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. |
22+
| composer\_env\_id | ID of Cloud Composer Environment. |
23+
| composer\_env\_name | Name of the Cloud Composer Environment. |
24+
| gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. |
25+
| project\_id | Project ID where Cloud Composer Environment is created. |
26+
27+
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
28+
29+
To provision this example, run the following from within this directory:
30+
- `terraform init` to get the plugins
31+
- `terraform plan` to see the infrastructure plan
32+
- `terraform apply` to apply the infrastructure build
33+
- `terraform destroy` to destroy the built infrastructure

‎examples/simple_composer_env_v3/main.tf‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
resource "random_string" "key_suffix" {
18+
length = 5
19+
special = false
20+
upper = false
21+
}
22+
23+
# Create a bucket to store the snapshots
24+
resource "google_storage_bucket" "my_bucket" {
25+
project = var.project_id
26+
name = "snapshot-bucket-${random_string.key_suffix.result}"
27+
location = var.region
28+
force_destroy = true
29+
uniform_bucket_level_access = true
30+
}
31+
32+
resource "google_storage_bucket_iam_member" "object_admin" {
33+
bucket = google_storage_bucket.my_bucket.name
34+
role = "roles/storage.objectAdmin"
35+
member = "serviceAccount:${var.composer_sa}"
36+
}
37+
38+
module "simple-composer-environment" {
39+
source = "terraform-google-modules/composer/google//modules/create_environment_v3"
40+
version = "~> 6.0"
41+
42+
project_id = var.project_id
43+
composer_env_name = var.composer_env_name
44+
region = var.region
45+
composer_service_account = var.composer_sa
46+
network = google_compute_network.main.name
47+
subnetwork = google_compute_subnetwork.main.name
48+
create_network_attachment = true
49+
50+
grant_sa_agent_permission = false
51+
environment_size = "ENVIRONMENT_SIZE_SMALL"
52+
53+
use_private_environment = true
54+
enable_private_builds_only = true
55+
cloud_data_lineage_integration = true
56+
resilience_mode = "STANDARD_RESILIENCE"
57+
58+
scheduled_snapshots_config = {
59+
enabled = true
60+
snapshot_location = google_storage_bucket.my_bucket.url
61+
snapshot_creation_schedule = "0 4 * * *"
62+
time_zone = "UTC+01"
63+
}
64+
65+
maintenance_start_time = "2025-02-01T00:00:00Z"
66+
maintenance_end_time = "2025-05-01T12:00:00Z"
67+
maintenance_recurrence = "FREQ=WEEKLY;BYDAY=SU,SA"
68+
69+
depends_on = [
70+
google_storage_bucket_iam_member.object_admin,
71+
]
72+
73+
web_server_network_access_control = [
74+
{
75+
allowed_ip_range = "192.0.2.0/24"
76+
description = "office net 1"
77+
},
78+
{
79+
allowed_ip_range = "192.0.4.0/24"
80+
description = "office net 2"
81+
},
82+
]
83+
}

‎examples/simple_composer_env_v3/network.tf‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/**
2+
* Copyright 2022 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
resource "google_compute_network" "main" {
18+
project = var.project_id
19+
name = "ci-composer-test-${random_string.suffix.result}"
20+
auto_create_subnetworks = false
21+
}
22+
23+
resource "google_compute_subnetwork" "main" {
24+
project = var.project_id
25+
name = "ci-composer-test-${random_string.suffix.result}"
26+
ip_cidr_range = "10.0.0.0/17"
27+
region = var.region
28+
network = google_compute_network.main.self_link
29+
private_ip_google_access = true
30+
}
31+
32+
resource "random_string" "suffix" {
33+
length = 4
34+
special = false
35+
upper = false
36+
}

‎examples/simple_composer_env_v3/outputs.tf‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
output "project_id" {
18+
description = "Project ID where Cloud Composer Environment is created."
19+
value = var.project_id
20+
}
21+
22+
output "composer_env_name" {
23+
description = "Name of the Cloud Composer Environment."
24+
value = module.simple-composer-environment.composer_env_name
25+
}
26+
27+
output "composer_env_id" {
28+
description = "ID of Cloud Composer Environment."
29+
value = module.simple-composer-environment.composer_env_id
30+
}
31+
32+
output "gcs_bucket" {
33+
description = "Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment."
34+
value = module.simple-composer-environment.gcs_bucket
35+
}
36+
37+
output "airflow_uri" {
38+
description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment."
39+
value = module.simple-composer-environment.airflow_uri
40+
}

‎examples/simple_composer_env_v3/variables.tf‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
variable "project_id" {
18+
description = "Project ID where Cloud Composer Environment is created."
19+
type = string
20+
}
21+
22+
variable "composer_env_name" {
23+
description = "Name of Cloud Composer Environment."
24+
default = "ci-composer"
25+
type = string
26+
}
27+
28+
variable "region" {
29+
description = "Region where Cloud Composer Environment is created."
30+
type = string
31+
default = "us-central1"
32+
}
33+
34+
variable "composer_sa" {
35+
description = "Service Account to be used for running Cloud Composer Environment."
36+
type = string
37+
}

0 commit comments

Comments
 (0)