Skip to content

Commit ea834f7

Browse files
imrannayerimrannayer
authored
feat(TPG >=5.3)!: add new features. update provider requirements (#98)
1 parent e975c68 commit ea834f7

File tree

27 files changed

+806
-972
lines changed

27 files changed

+806
-972
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
# Make will use bash instead of sh
1919
SHELL := /usr/bin/env bash
2020

21-
DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.5
21+
DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.17
2222
DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools
2323
REGISTRY_URL := gcr.io/cloud-foundation-cicd
2424

‎README.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,16 @@ The resources/services/activations/deletions that this module will create/trigge
66

77
- Create a GCP Composer Environment
88

9+
## Compatibility
10+
11+
This module is meant for use with Terraform 1.3+ and tested using Terraform 1.3+. If you find incompatibilities using Terraform >=1.3, please open an issue.
12+
13+
## Version
14+
15+
Current version is 4.0. Upgrade guides:
16+
17+
- [3.X -> 4.0.](/docs/upgrading_to_v4.0.md)
18+
919
## Usage
1020
Cloud Composer currently has two versions: V2 has greater flexibility in the Airflow core services
1121
(scheduler, webserver, worker) and has a more practical and scalable infrastructure. Therefore, we recommend prioritizing the

‎build/int.cloudbuild.yaml‎

Lines changed: 49 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -28,52 +28,6 @@ steps:
2828
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
2929
args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create']
3030

31-
# ----- SUITE simple-composer-env-v1-local
32-
33-
- id: init-simple-composer-env-v1
34-
waitFor:
35-
- prepare
36-
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
37-
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage init --verbose']
38-
- id: apply-simple-composer-env-v1
39-
waitFor:
40-
- init-simple-composer-env-v1
41-
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
42-
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage apply --verbose']
43-
- id: verify-simple-composer-env-v1
44-
waitFor:
45-
- apply-simple-composer-env-v1
46-
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
47-
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage verify --verbose']
48-
- id: destroy-simple-composer-env-v1
49-
waitFor:
50-
- verify-simple-composer-env-v1
51-
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
52-
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage destroy --verbose']
53-
54-
# ----- SUITE simple-composer-env-v2-local
55-
56-
- id: init-simple-composer-env-v2
57-
waitFor:
58-
- prepare
59-
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
60-
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage init --verbose']
61-
- id: apply-simple-composer-env-v2
62-
waitFor:
63-
- init-simple-composer-env-v2
64-
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
65-
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage apply --verbose']
66-
- id: verify-simple-composer-env-v2
67-
waitFor:
68-
- apply-simple-composer-env-v2
69-
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
70-
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose']
71-
- id: destroy-simple-composer-env-v2
72-
waitFor:
73-
- verify-simple-composer-env-v2
74-
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
75-
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose']
76-
7731
# ----- SUITE airflow-connection-local
7832

7933
- id: converge airflow-connection-local
@@ -111,11 +65,34 @@ steps:
11165
# name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
11266
# args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy airflow-pool-local']
11367

114-
# ----- SUITE composer-v2-sharedvpc-prereq-local
68+
# ----- SUITE simple-composer-env-v2-local
69+
70+
- id: init-simple-composer-env-v2
71+
waitFor:
72+
- verify airflow-pool-local
73+
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
74+
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage init --verbose']
75+
- id: apply-simple-composer-env-v2
76+
waitFor:
77+
- init-simple-composer-env-v2
78+
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
79+
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage apply --verbose']
80+
- id: verify-simple-composer-env-v2
81+
waitFor:
82+
- apply-simple-composer-env-v2
83+
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
84+
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose']
85+
- id: destroy-simple-composer-env-v2
86+
waitFor:
87+
- verify-simple-composer-env-v2
88+
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
89+
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose']
90+
91+
# ----- SUITE composer-v2-sharedvpc-prereq-local
11592

11693
- id: init-composer-v2-sharedvpc-prereq
11794
waitFor:
118-
- prepare
95+
- destroy-simple-composer-env-v2
11996
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
12097
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage init --verbose']
12198
- id: apply-composer-v2-sharedvpc-prereq
@@ -133,9 +110,32 @@ steps:
133110
- verify-composer-v2-sharedvpc-prereq
134111
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
135112
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage destroy --verbose']
113+
114+
# ----- SUITE simple-composer-env-v1-local
115+
116+
- id: init-simple-composer-env-v1
117+
waitFor:
118+
- destroy-composer-v2-sharedvpc-prereq
119+
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
120+
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage init --verbose']
121+
- id: apply-simple-composer-env-v1
122+
waitFor:
123+
- init-simple-composer-env-v1
124+
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
125+
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage apply --verbose']
126+
- id: verify-simple-composer-env-v1
127+
waitFor:
128+
- apply-simple-composer-env-v1
129+
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
130+
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage verify --verbose']
131+
- id: destroy-simple-composer-env-v1
132+
waitFor:
133+
- verify-simple-composer-env-v1
134+
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
135+
args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage destroy --verbose']
136136
tags:
137137
- 'ci'
138138
- 'integration'
139139
substitutions:
140140
_DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
141-
_DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.5'
141+
_DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.17'

‎docs/upgrading_to_v4.0.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Upgrading to v4.0.0
2+
3+
The v4.0 release contains backwards-incompatible changes.
4+
5+
This update requires upgrading the minimum provider version to `5.3`.
6+
7+
### TPG max version is bumped to 6.x

‎examples/airflow_connection/main.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ module "simple-composer-environment" {
2727
service_ip_allocation_range_name = var.service_ip_allocation_range_name
2828
node_count = 3
2929
machine_type = "n1-standard-1"
30+
image_version = "composer-1.20.12-airflow-1.10.15"
3031
}
3132

3233
# Making the k8s master globally available is only to make the integration testing portable and should be removed

‎examples/airflow_pool/main.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ module "simple-composer-environment" {
2727
service_ip_allocation_range_name = var.service_ip_allocation_range_name
2828
node_count = 3
2929
machine_type = "n1-standard-1"
30+
image_version = "composer-1.20.12-airflow-1.10.15"
3031
}
3132

3233
# Making the k8s master globally available is only to make the integration testing portable and should be removed

‎examples/composer_v2_sharedvpc_prereq/composer.tf‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ module "composer_net" {
2222
subnetwork = "composer-subnet"
2323
cloud_composer_network_ipv4_cidr_block = "192.168.192.0/24"
2424
master_ipv4_cidr = "192.168.193.0/28"
25-
cloud_sql_ipv4_cidr = "192.168.0.0/17"
2625
gke_subnet_ip_range = ["10.100.232.0/27"]
2726
gke_pods_services_ip_ranges = ["10.1.0.0/16", "10.4.0.0/16", "10.10.10.0/24", "10.10.14.0/24"]
2827
}

‎examples/simple_composer_env_v1/main.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,5 @@ module "simple-composer-environment" {
2727
service_ip_allocation_range_name = var.service_ip_allocation_range_name
2828
node_count = 3
2929
machine_type = "n1-standard-1"
30+
image_version = "composer-1.20.12-airflow-1.10.15"
3031
}

‎examples/simple_composer_env_v2/main.tf‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,27 @@
1414
* limitations under the License.
1515
*/
1616

17+
resource "random_string" "key_suffix" {
18+
length = 5
19+
special = false
20+
upper = false
21+
}
22+
23+
# Create a bucket to store the snapshots
24+
resource "google_storage_bucket" "my_bucket" {
25+
project = var.project_id
26+
name = "snapshot-bucket-${random_string.key_suffix.result}"
27+
location = var.region
28+
force_destroy = true
29+
uniform_bucket_level_access = true
30+
}
31+
32+
resource "google_storage_bucket_iam_member" "object_admin" {
33+
bucket = google_storage_bucket.my_bucket.name
34+
role = "roles/storage.objectAdmin"
35+
member = "serviceAccount:${var.composer_service_account}"
36+
}
37+
1738
module "simple-composer-environment" {
1839
source = "../../modules/create_environment_v2"
1940
project_id = var.project_id
@@ -26,11 +47,14 @@ module "simple-composer-environment" {
2647
service_ip_allocation_range_name = var.service_ip_allocation_range_name
2748
grant_sa_agent_permission = false
2849
environment_size = "ENVIRONMENT_SIZE_SMALL"
50+
enable_private_endpoint = true
51+
use_private_environment = true
52+
2953
scheduler = {
3054
cpu = 0.5
3155
memory_gb = 1.875
3256
storage_gb = 1
33-
count = 1
57+
count = 2
3458
}
3559
web_server = {
3660
cpu = 0.5
@@ -41,7 +65,24 @@ module "simple-composer-environment" {
4165
cpu = 0.5
4266
memory_gb = 1.875
4367
storage_gb = 1
44-
min_count = 1
68+
min_count = 2
4569
max_count = 3
4670
}
71+
72+
scheduled_snapshots_config = {
73+
enabled = true
74+
snapshot_location = google_storage_bucket.my_bucket.url
75+
snapshot_creation_schedule = "0 4 * * *"
76+
time_zone = "UTC+01"
77+
}
78+
79+
maintenance_window = {
80+
start_time = "2023-01-01T00:00:00Z"
81+
end_time = "2023-01-01T12:00:00Z"
82+
recurrence = "FREQ=WEEKLY;BYDAY=SU,WE,SA"
83+
}
84+
85+
depends_on = [
86+
google_storage_bucket_iam_member.object_admin,
87+
]
4788
}

‎modules/airflow_storage/README.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,14 @@ See the examples for cloudsql support.
2424
| Name | Description | Type | Default | Required |
2525
|------|-------------|------|---------|:--------:|
2626
| destination\_path | The optional destination path | `string` | `null` | no |
27-
| environment | n/a | `string` | n/a | yes |
28-
| location | n/a | `string` | n/a | yes |
29-
| project\_id | n/a | `string` | n/a | yes |
27+
| environment | Environment | `string` | n/a | yes |
28+
| location | Location of the resource | `string` | n/a | yes |
29+
| project\_id | Project ID | `string` | n/a | yes |
3030
| source\_path | The source on the local file system | `string` | n/a | yes |
3131
| type | The type of resource to upload. Either dag, plugin or data | `string` | n/a | yes |
3232
3333
## Outputs
3434
35-
No output.
35+
No outputs.
3636
3737
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

0 commit comments

Comments
 (0)