Skip to content

Commit 4e4dc9d

Browse files
prabhu34prabhu34
committed
feat: added examples and tests
1 parent 72ef715 commit 4e4dc9d

File tree

6 files changed

+278
-14
lines changed

6 files changed

+278
-14
lines changed

‎examples/simple_composer_env_v3/README.md‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Simple Cloud Composer Environment (V3) Example
2+
3+
This example illustrates how to use the `composer` V2 module to deploy private composer environment with private service connect (PSC) endpoint to connect network attachments.
4+
5+
This example also creates a Cloud Storage Bucket for scheduled snapshots and assign appropriate permission(s) to Composer Service Account on the bucket.
6+
7+
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
8+
## Inputs
9+
10+
| Name | Description | Type | Default | Required |
11+
|------|-------------|------|---------|:--------:|
12+
| composer\_env\_name | Name of Cloud Composer Environment. | `string` | `"ci-composer"` | no |
13+
| composer\_service\_account | Service Account to be used for running Cloud Composer Environment. | `string` | n/a | yes |
14+
| network | Network where Cloud Composer is created. | `string` | n/a | yes |
15+
| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes |
16+
| region | Region where Cloud Composer Environment is created. | `string` | n/a | yes |
17+
| subnetwork | Name of the Subetwork where Cloud Composer is created. | `string` | n/a | yes |
18+
19+
## Outputs
20+
21+
| Name | Description |
22+
|------|-------------|
23+
| airflow\_uri | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. |
24+
| composer\_env\_id | ID of Cloud Composer Environment. |
25+
| composer\_env\_name | Name of the Cloud Composer Environment. |
26+
| gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. |
27+
28+
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
29+
30+
To provision this example, run the following from within this directory:
31+
- `terraform init` to get the plugins
32+
- `terraform plan` to see the infrastructure plan
33+
- `terraform apply` to apply the infrastructure build
34+
- `terraform destroy` to destroy the built infrastructure

‎examples/simple_composer_env_v3/main.tf‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
resource "random_string" "key_suffix" {
18+
length = 5
19+
special = false
20+
upper = false
21+
}
22+
23+
# Create a bucket to store the snapshots
24+
resource "google_storage_bucket" "my_bucket" {
25+
project = var.project_id
26+
name = "snapshot-bucket-${random_string.key_suffix.result}"
27+
location = var.region
28+
force_destroy = true
29+
uniform_bucket_level_access = true
30+
}
31+
32+
resource "google_storage_bucket_iam_member" "object_admin" {
33+
bucket = google_storage_bucket.my_bucket.name
34+
role = "roles/storage.objectAdmin"
35+
member = "serviceAccount:${var.composer_service_account}"
36+
}
37+
38+
module "simple-composer-environment" {
39+
source = "terraform-google-modules/composer/google//modules/create_environment_v3"
40+
version = "~> 6.0"
41+
42+
project_id = var.project_id
43+
composer_env_name = var.composer_env_name
44+
region = var.region
45+
composer_service_account = var.composer_service_account
46+
network = var.network
47+
subnetwork = var.subnetwork
48+
create_network_attachment = true
49+
50+
grant_sa_agent_permission = false
51+
environment_size = "ENVIRONMENT_SIZE_SMALL"
52+
53+
use_private_environment = true
54+
cloud_data_lineage_integration = true
55+
resilience_mode = "STANDARD_RESILIENCE"
56+
57+
scheduler = {
58+
cpu = 0.5
59+
memory_gb = 1.875
60+
storage_gb = 1
61+
count = 2
62+
}
63+
64+
dag_processor = {
65+
cpu = 0.5
66+
memory_gb = 1.875
67+
storage_gb = 1
68+
count = 2
69+
}
70+
71+
web_server = {
72+
cpu = 0.5
73+
memory_gb = 1.875
74+
storage_gb = 1
75+
}
76+
77+
worker = {
78+
cpu = 0.5
79+
memory_gb = 1.875
80+
storage_gb = 1
81+
min_count = 2
82+
max_count = 3
83+
}
84+
85+
triggerer = {
86+
cpu = 1
87+
memory_gb = 1
88+
count = 2
89+
}
90+
91+
scheduled_snapshots_config = {
92+
enabled = true
93+
snapshot_location = google_storage_bucket.my_bucket.url
94+
snapshot_creation_schedule = "0 4 * * *"
95+
time_zone = "UTC+01"
96+
}
97+
98+
maintenance_start_time = "2025-02-01T00:00:00Z"
99+
maintenance_end_time = "2025-03-01T12:00:00Z"
100+
maintenance_recurrence = "FREQ=WEEKLY;BYDAY=SU,SA"
101+
102+
depends_on = [
103+
google_storage_bucket_iam_member.object_admin,
104+
]
105+
106+
web_server_network_access_control = [
107+
{
108+
allowed_ip_range = "192.0.2.0/24"
109+
description = "office net 1"
110+
},
111+
{
112+
allowed_ip_range = "192.0.4.0/24"
113+
description = "office net 2"
114+
},
115+
]
116+
}

‎examples/simple_composer_env_v3/outputs.tf‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
output "composer_env_name" {
18+
description = "Name of the Cloud Composer Environment."
19+
value = module.simple-composer-environment.composer_env_name
20+
}
21+
22+
output "composer_env_id" {
23+
description = "ID of Cloud Composer Environment."
24+
value = module.simple-composer-environment.composer_env_id
25+
}
26+
27+
output "gcs_bucket" {
28+
description = "Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment."
29+
value = module.simple-composer-environment.gcs_bucket
30+
}
31+
32+
output "airflow_uri" {
33+
description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment."
34+
value = module.simple-composer-environment.airflow_uri
35+
}

‎examples/simple_composer_env_v3/variables.tf‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/**
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
variable "project_id" {
18+
description = "Project ID where Cloud Composer Environment is created."
19+
type = string
20+
}
21+
22+
variable "composer_env_name" {
23+
description = "Name of Cloud Composer Environment."
24+
default = "ci-composer"
25+
type = string
26+
}
27+
28+
variable "region" {
29+
description = "Region where Cloud Composer Environment is created."
30+
type = string
31+
}
32+
33+
variable "composer_service_account" {
34+
description = "Service Account to be used for running Cloud Composer Environment."
35+
type = string
36+
}
37+
38+
variable "network" {
39+
description = "Network where Cloud Composer is created."
40+
type = string
41+
}
42+
43+
variable "subnetwork" {
44+
description = "Name of the Subetwork where Cloud Composer is created."
45+
type = string
46+
}

‎modules/create_environment_v3/README.md‎

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -64,45 +64,39 @@ module "simple-composer-environment" {
6464
| Name | Description | Type | Default | Required |
6565
|------|-------------|------|---------|:--------:|
6666
| airflow\_config\_overrides | Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example "core-dags\_are\_paused\_at\_creation". | `map(string)` | `{}` | no |
67-
| cloud\_composer\_connection\_subnetwork | Subnetwork self-link. When specified, the environment will use Private Service Connect instead of VPC peerings to connect to CloudSQL in the Tenant Project. IP address of psc endpoint is allocated from this subnet | `string` | `null` | no |
68-
| cloud\_composer\_network\_ipv4\_cidr\_block | The CIDR block from which IP range in tenant project will be reserved. Required if VPC peering is used to connect to CloudSql instead of PSC | `string` | `null` | no |
6967
| cloud\_data\_lineage\_integration | Whether or not Dataplex data lineage integration is enabled. Cloud Composer environments in versions composer-2.1.2-airflow-..* and newer) | `bool` | `false` | no |
70-
| cloud\_sql\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for Cloud SQL private service access. Required if VPC peering is used to connect to CloudSql instead of PSC | `string` | `null` | no |
7168
| composer\_env\_name | Name of Cloud Composer Environment | `string` | n/a | yes |
69+
| composer\_network\_attachment\_name | Name for PSC (Private Service Connect) Network entry point. | `string` | `null` | no |
7270
| composer\_service\_account | Service Account for running Cloud Composer. | `string` | `null` | no |
73-
| enable\_ip\_masq\_agent | Deploys 'ip-masq-agent' daemon set in the GKE cluster and defines nonMasqueradeCIDRs equals to pod IP range so IP masquerading is used for all destination addresses, except between pods traffic. | `bool` | `false` | no |
74-
| enable\_private\_endpoint | Configure private access to the cluster endpoint. If true, access to the public endpoint of the GKE cluster is denied | `bool` | `false` | no |
75-
| enable\_privately\_used\_public\_ips | When enabled, IPs from public (non-RFC1918) ranges can be used for pod\_ip\_allocation\_range\_name and service\_ip\_allocation\_range\_name. | `bool` | `false` | no |
71+
| create\_network\_attachment | Either create a new network attachment or use existing one. If true, provide the subnet details. | `bool` | `true` | no |
72+
| dag\_processor | Configuration for resources used by Airflow workers. | <pre>object({<br> cpu = string<br> memory_gb = number<br> storage_gb = number<br> count = number<br> })</pre> | <pre>{<br> "count": 2,<br> "cpu": 2,<br> "memory_gb": 7.5,<br> "storage_gb": 5<br>}</pre> | no |
7673
| env\_variables | Variables of the airflow environment. | `map(string)` | `{}` | no |
7774
| environment\_size | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. Values for environment size are: `ENVIRONMENT_SIZE_SMALL`, `ENVIRONMENT_SIZE_MEDIUM`, and `ENVIRONMENT_SIZE_LARGE`. | `string` | `"ENVIRONMENT_SIZE_MEDIUM"` | no |
7875
| grant\_sa\_agent\_permission | Cloud Composer relies on Workload Identity as Google API authentication mechanism for Airflow. | `bool` | `true` | no |
79-
| image\_version | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-2.10.2-airflow-2.10.2"` | no |
76+
| image\_version | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-3-airflow-2.10.2-build.5"` | no |
8077
| kms\_key\_name | Customer-managed Encryption Key fully qualified resource name, i.e. projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key. | `string` | `null` | no |
8178
| labels | The resource labels (a map of key/value pairs) to be applied to the Cloud Composer. | `map(string)` | `{}` | no |
8279
| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `null` | no |
8380
| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `null` | no |
8481
| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no |
85-
| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | <pre>list(object({<br> cidr_block = string<br> display_name = string<br> }))</pre> | `[]` | no |
86-
| master\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for the GKE master. Required when `use_private_environment` and `enable_private_endpoint` is `true` | `string` | `null` | no |
8782
| network | The VPC network to host the composer cluster. | `string` | n/a | yes |
8883
| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no |
89-
| pod\_ip\_allocation\_range\_name | The name of the subnet secondary range, used to allocate IP addresses for the pods. | `string` | `null` | no |
9084
| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes |
9185
| pypi\_packages | Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. "numpy"). | `map(string)` | `{}` | no |
9286
| region | Region where the Cloud Composer Environment is created. | `string` | `"us-central1"` | no |
9387
| resilience\_mode | Cloud Composer 2.1.15 or newer only. The resilience mode states whether high resilience is enabled for the environment or not. Values for resilience mode are `HIGH_RESILIENCE` for high resilience and `STANDARD_RESILIENCE` for standard resilience | `string` | `null` | no |
9488
| scheduled\_snapshots\_config | The recovery configuration settings for the Cloud Composer environment | <pre>object({<br> enabled = optional(bool, false)<br> snapshot_location = optional(string)<br> snapshot_creation_schedule = optional(string)<br> time_zone = optional(string)<br> })</pre> | `null` | no |
95-
| scheduler | Configuration for resources used by Airflow schedulers. | <pre>object({<br> cpu = string<br> memory_gb = number<br> storage_gb = number<br> count = number<br> })</pre> | <pre>{<br> "count": 2,<br> "cpu": 2,<br> "memory_gb": 7.5,<br> "storage_gb": 5<br>}</pre> | no |
96-
| service\_ip\_allocation\_range\_name | The name of the subnet secondary range, used to allocate IP addresses for the Services. | `string` | `null` | no |
89+
| scheduler | Configuration for resources used by Airflow schedulers. | <pre>object({<br> cpu = string<br> memory_gb = number<br> storage_gb = number<br> count = number<br> })</pre> | <pre>{<br> "count": 2,<br> "cpu": 1,<br> "memory_gb": 4,<br> "storage_gb": 5<br>}</pre> | no |
9790
| storage\_bucket | Name of an existing Cloud Storage bucket to be used by the environment | `string` | `null` | no |
9891
| subnetwork | The name of the subnetwork to host the composer cluster. | `string` | n/a | yes |
9992
| subnetwork\_region | The subnetwork region of the shared VPC's host (for shared vpc support) | `string` | `""` | no |
10093
| tags | Tags applied to all nodes. Tags are used to identify valid sources or targets for network firewalls. | `set(string)` | `[]` | no |
10194
| task\_logs\_retention\_storage\_mode | The mode of storage for Airflow workers task logs. Values for storage mode are CLOUD\_LOGGING\_ONLY to only store logs in cloud logging and CLOUD\_LOGGING\_AND\_CLOUD\_STORAGE to store logs in cloud logging and cloud storage. Cloud Composer 2.0.23 or newer only | `string` | `null` | no |
10295
| triggerer | Configuration for resources used by Airflow triggerer | <pre>object({<br> cpu = string<br> memory_gb = number<br> count = number<br> })</pre> | `null` | no |
103-
| use\_private\_environment | Create a private environment. | `bool` | `false` | no |
96+
| use\_private\_environment | Create a private environment. If true, a private Composer environment will be created. | `bool` | `false` | no |
10497
| web\_server | Configuration for resources used by Airflow web server. | <pre>object({<br> cpu = string<br> memory_gb = number<br> storage_gb = number<br> })</pre> | <pre>{<br> "cpu": 2,<br> "memory_gb": 7.5,<br> "storage_gb": 5<br>}</pre> | no |
10598
| web\_server\_network\_access\_control | The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions are applied | <pre>list(object({<br> allowed_ip_range = string<br> description = string<br> }))</pre> | `null` | no |
99+
| web\_server\_plugins\_mode | Web server plugins configuration. Can be either 'ENABLED' or 'DISABLED'. Defaults to 'ENABLED'. | `string` | `"ENABLED"` | no |
106100
| worker | Configuration for resources used by Airflow workers. | <pre>object({<br> cpu = string<br> memory_gb = number<br> storage_gb = number<br> min_count = number<br> max_count = number<br> })</pre> | <pre>{<br> "cpu": 2,<br> "max_count": 6,<br> "memory_gb": 7.5,<br> "min_count": 2,<br> "storage_gb": 5<br>}</pre> | no |
107101

108102
## Outputs
@@ -114,6 +108,5 @@ module "simple-composer-environment" {
114108
| composer\_env\_id | ID of Cloud Composer Environment. |
115109
| composer\_env\_name | Name of the Cloud Composer Environment. |
116110
| gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. |
117-
| gke\_cluster | Google Kubernetes Engine cluster used to run the Cloud Composer Environment. |
118111

119112
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

‎test/integration/simple-composer-env-v3/simple_composer_env_v3_test.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package simple_composer_env_v3
16+
17+
import (
18+
"fmt"
19+
"testing"
20+
21+
"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
22+
"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
23+
"github.com/stretchr/testify/assert"
24+
)
25+
26+
func TestSimpleComposerEnvV3Module(t *testing.T) {
27+
composer := tft.NewTFBlueprintTest(t)
28+
29+
composer.DefineVerify(func(assert *assert.Assertions) {
30+
composer.DefaultVerify(assert)
31+
32+
projectID := composer.GetStringOutput("project_id")
33+
34+
op := gcloud.Runf(t, "composer environments describe %s --project=%s --location=us-central1", composer.GetStringOutput("composer_env_name"), projectID)
35+
assert.Equal(fmt.Sprintf("projects/%s/locations/us-central1/environments/%s", projectID, composer.GetStringOutput("composer_env_name")), op.Get("name").String(), "Composer name is valid")
36+
assert.Equal(composer.GetStringOutput("airflow_uri"), op.Get("config.airflowUri").String(), "AirflowUri is valid")
37+
assert.Equal(composer.GetStringOutput("gcs_bucket"), op.Get("config.dagGcsPrefix").String(), "GCS-Dag is valid")
38+
})
39+
composer.Test()
40+
}

0 commit comments

Comments
 (0)