Skip to content

Commit 549ba62

Browse files
diegolnascdiegolnasc
authored
feat: Composer module improvement (#12)
* feat: composer module * feat: cloud composer module improvement * fix: white spaces * fix: fmt * fix: test modules * fix: test modules * feat: readme * feat: cloud composer module improvement * fix: Vpc native * feat: Makefile + tests * feat: additional resources for testing. * fix: provider version
1 parent d45205e commit 549ba62

File tree

13 files changed

+316
-53
lines changed

13 files changed

+316
-53
lines changed

‎README.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ module "composer" {
1919
region = "us-central1"
2020
composer_env_name = "composer-env-test"
2121
composer_sa = "project-service-account@<PROJECT_ID>.iam.gserviceaccount.com"
22+
network = "test-network"
23+
subnetwork = "composer-subnet"
2224
}
2325
```
2426

@@ -31,9 +33,10 @@ Functional examples are included in the
3133
| Name | Description | Type | Default | Required |
3234
|------|-------------|------|---------|:--------:|
3335
| composer\_env\_name | Name of Cloud Composer Environment | `string` | n/a | yes |
34-
| composer\_sa | Service Account to be used for running Cloud Composer Environment. | `string` | n/a | yes |
36+
| network | Network where Cloud Composer is created. | `string` | n/a | yes |
3537
| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes |
3638
| region | Region where the Cloud Composer Environment is created. | `string` | n/a | yes |
39+
| subnetwork | Subetwork where Cloud Composer is created. | `string` | n/a | yes |
3740

3841
## Outputs
3942

‎examples/simple_composer_env/README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,12 @@ This example illustrates how to use the `composer` module.
99
|------|-------------|------|---------|:--------:|
1010
| composer\_env\_name | Name of Cloud Composer Environment. | `string` | n/a | yes |
1111
| composer\_service\_account | Service Account to be used for running Cloud Composer Environment. | `string` | n/a | yes |
12+
| network | Network where Cloud Composer is created. | `string` | n/a | yes |
13+
| pod\_ip\_allocation\_range\_name | The name of the cluster's secondary range used to allocate IP addresses to pods. | `string` | n/a | yes |
1214
| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes |
1315
| region | Region where Cloud Composer Environment is created. | `string` | n/a | yes |
16+
| service\_ip\_allocation\_range\_name | The name of the services' secondary range used to allocate IP addresses to the cluster. | `string` | n/a | yes |
17+
| subnetwork | Subetwork where Cloud Composer is created. | `string` | n/a | yes |
1418

1519
## Outputs
1620

‎examples/simple_composer_env/main.tf‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,13 @@ provider "google-beta" {
2828
module "simple-composer-environment" {
2929
source = "../../modules/create_environment"
3030

31-
project_id = var.project_id
32-
composer_env_name = var.composer_env_name
33-
region = var.region
34-
35-
composer_service_account = var.composer_service_account
31+
project_id = var.project_id
32+
composer_env_name = var.composer_env_name
33+
region = var.region
34+
composer_service_account = var.composer_service_account
35+
network = var.network
36+
subnetwork = var.subnetwork
37+
use_ip_aliases = true
38+
pod_ip_allocation_range_name = var.pod_ip_allocation_range_name
39+
service_ip_allocation_range_name = var.service_ip_allocation_range_name
3640
}

‎examples/simple_composer_env/variables.tf‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,23 @@ variable "composer_service_account" {
3333
description = "Service Account to be used for running Cloud Composer Environment."
3434
type = string
3535
}
36+
37+
variable "network" {
38+
description = "Network where Cloud Composer is created."
39+
type = string
40+
}
41+
42+
variable "subnetwork" {
43+
description = "Subetwork where Cloud Composer is created."
44+
type = string
45+
}
46+
47+
variable "pod_ip_allocation_range_name" {
48+
description = "The name of the cluster's secondary range used to allocate IP addresses to pods."
49+
type = string
50+
}
51+
52+
variable "service_ip_allocation_range_name" {
53+
type = string
54+
description = "The name of the services' secondary range used to allocate IP addresses to the cluster."
55+
}

‎main.tf‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,9 @@
1717
module "composer-environment" {
1818
source = "./modules/create_environment"
1919

20-
project_id = var.project_id
21-
composer_env_name = var.composer_env_name
22-
region = var.region
23-
composer_service_account = var.composer_sa
20+
project_id = var.project_id
21+
composer_env_name = var.composer_env_name
22+
region = var.region
23+
network = var.network
24+
subnetwork = var.subnetwork
2425
}

‎modules/create_environment/README.md‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,8 @@ module "composer" {
1616
node_config {
1717
zone = "us-central1-f"
1818
machine_type = "n1-standard-1"
19-
2019
network = "test-network"
2120
subnetwork = "test-subnetwork"
22-
23-
service_account = [email protected]
2421
}
2522
}
2623
}
@@ -30,15 +27,32 @@ module "composer" {
3027

3128
| Name | Description | Type | Default | Required |
3229
|------|-------------|------|---------|:--------:|
30+
| airflow\_config\_overrides | Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example "core-dags\_are\_paused\_at\_creation". | `map(string)` | `{}` | no |
31+
| cloud\_sql\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for Cloud SQL. | `string` | `null` | no |
3332
| composer\_env\_name | Name of Cloud Composer Environment | `string` | n/a | yes |
34-
| composer\_service\_account | Service Account for running Cloud Composer. | `string` | n/a | yes |
35-
| ip\_cidr\_range | CIDR range for the Cloud Composer Subnet. | `string` | `"10.0.0.0/14"` | no |
33+
| composer\_service\_account | Service Account for running Cloud Composer. | `string` | `null` | no |
34+
| disk\_size | The disk size for nodes. | `string` | `"100"` | no |
35+
| enable\_private\_endpoint | Configure public access to the cluster endpoint. | `bool` | `false` | no |
36+
| env\_variables | Variables of the airflow environment. | `map(string)` | `{}` | no |
37+
| image\_version | The version of the aiflow running in the cloud composer environment. | `string` | `null` | no |
38+
| labels | The resource labels (a map of key/value pairs) to be applied to the Cloud Composer. | `map(string)` | `{}` | no |
3639
| machine\_type | Machine type of Cloud Composer nodes. | `string` | `"n1-standard-8"` | no |
37-
| network\_name | Name of network created for Cloud Composer Environment. | `string` | `"composer-network-01"` | no |
40+
| master\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for the master. | `string` | `null` | no |
41+
| network | The VPC network to host the composer cluster. | `string` | n/a | yes |
42+
| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no |
3843
| node\_count | Number of worker nodes in Cloud Composer Environment. | `number` | `3` | no |
44+
| oauth\_scopes | Google API scopes to be made available on all node. | `set(string)` | <pre>[<br> "https://www.googleapis.com/auth/cloud-platform"<br>]</pre> | no |
45+
| pod\_ip\_allocation\_range\_name | The name of the cluster's secondary range used to allocate IP addresses to pods. | `string` | `null` | no |
3946
| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes |
47+
| pypi\_packages | Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. "numpy"). | `map(string)` | `{}` | no |
48+
| python\_version | The default version of Python used to run the Airflow scheduler, worker, and webserver processes. | `string` | `"3"` | no |
4049
| region | Region where the Cloud Composer Environment is created. | `string` | `"us-central1"` | no |
41-
| subnet\_name | Name of subnetwork created for Cloud Composer Environment. | `string` | `"composer-subnet-01"` | no |
50+
| service\_ip\_allocation\_range\_name | The name of the services' secondary range used to allocate IP addresses to the cluster. | `string` | `null` | no |
51+
| subnetwork | The subnetwork to host the composer cluster. | `string` | n/a | yes |
52+
| subnetwork\_region | The subnetwork region of the shared VPC's host (for shared vpc support) | `string` | `""` | no |
53+
| tags | Tags applied to all nodes. Tags are used to identify valid sources or targets for network firewalls. | `set(string)` | `[]` | no |
54+
| use\_ip\_aliases | Enable Alias IPs in the GKE cluster. If true, a VPC-native cluster is created. | `bool` | `false` | no |
55+
| web\_server\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for the web server. | `string` | `null` | no |
4256
| zone | Zone where the Cloud Composer nodes are created. | `string` | `"us-central1-f"` | no |
4357

4458
## Outputs

‎modules/create_environment/main.tf‎

Lines changed: 55 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -14,36 +14,72 @@
1414
* limitations under the License.
1515
*/
1616

17+
locals {
18+
network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id
19+
subnetwork_region = var.subnetwork_region != "" ? var.subnetwork_region : join("-", slice(split("-", var.zone), 0, 2))
20+
}
21+
1722
resource "google_composer_environment" "composer_env" {
1823
project = var.project_id
1924
name = var.composer_env_name
2025
region = var.region
26+
labels = var.labels
2127

2228
config {
2329
node_count = var.node_count
2430

2531
node_config {
26-
zone = var.zone
27-
machine_type = var.machine_type
28-
29-
network = google_compute_network.composer_network.name
30-
subnetwork = google_compute_subnetwork.composer_subnetwork.name
31-
32+
zone = var.zone
33+
machine_type = var.machine_type
34+
network = "projects/${local.network_project_id}/global/networks/${var.network}"
35+
subnetwork = "projects/${local.network_project_id}/regions/${local.subnetwork_region}/subnetworks/${var.subnetwork}"
3236
service_account = var.composer_service_account
37+
disk_size_gb = var.disk_size
38+
oauth_scopes = var.oauth_scopes
39+
tags = var.tags
40+
41+
dynamic "ip_allocation_policy" {
42+
for_each = var.use_ip_aliases ? [1] : []
43+
content {
44+
use_ip_aliases = var.use_ip_aliases
45+
cluster_secondary_range_name = var.pod_ip_allocation_range_name
46+
services_secondary_range_name = var.service_ip_allocation_range_name
47+
}
48+
}
3349
}
34-
}
35-
}
3650

37-
resource "google_compute_network" "composer_network" {
38-
project = var.project_id
39-
name = var.network_name
40-
auto_create_subnetworks = false
41-
}
51+
dynamic "private_environment_config" {
52+
for_each = var.use_ip_aliases ? [
53+
{
54+
enable_private_endpoint = var.enable_private_endpoint
55+
cloud_sql_ipv4_cidr_block = var.cloud_sql_ipv4_cidr
56+
web_server_ipv4_cidr_block = var.web_server_ipv4_cidr
57+
master_ipv4_cidr_block = var.master_ipv4_cidr
58+
}] : []
59+
content {
60+
enable_private_endpoint = private_environment_config.value["enable_private_endpoint"]
61+
cloud_sql_ipv4_cidr_block = private_environment_config.value["cloud_sql_ipv4_cidr_block"]
62+
web_server_ipv4_cidr_block = private_environment_config.value["web_server_ipv4_cidr_block"]
63+
master_ipv4_cidr_block = private_environment_config.value["master_ipv4_cidr_block"]
64+
}
65+
}
4266

43-
resource "google_compute_subnetwork" "composer_subnetwork" {
44-
project = var.project_id
45-
name = var.subnet_name
46-
ip_cidr_range = var.ip_cidr_range
47-
region = var.region
48-
network = google_compute_network.composer_network.self_link
67+
dynamic "software_config" {
68+
for_each = var.python_version != "" ? [
69+
{
70+
airflow_config_overrides = var.airflow_config_overrides
71+
env_variables = var.env_variables
72+
image_version = var.image_version
73+
pypi_packages = var.pypi_packages
74+
python_version = var.python_version
75+
}] : []
76+
content {
77+
airflow_config_overrides = software_config.value["airflow_config_overrides"]
78+
env_variables = software_config.value["env_variables"]
79+
image_version = software_config.value["image_version"]
80+
pypi_packages = software_config.value["pypi_packages"]
81+
python_version = software_config.value["python_version"]
82+
}
83+
}
84+
}
4985
}

‎modules/create_environment/variables.tf‎

Lines changed: 112 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,34 @@ variable "region" {
3030
default = "us-central1"
3131
}
3232

33+
variable "labels" {
34+
type = map(string)
35+
description = "The resource labels (a map of key/value pairs) to be applied to the Cloud Composer."
36+
default = {}
37+
}
38+
39+
variable "network" {
40+
type = string
41+
description = "The VPC network to host the composer cluster."
42+
}
43+
44+
variable "network_project_id" {
45+
type = string
46+
description = "The project ID of the shared VPC's host (for shared vpc support)"
47+
default = ""
48+
}
49+
50+
variable "subnetwork" {
51+
type = string
52+
description = "The subnetwork to host the composer cluster."
53+
}
54+
55+
variable "subnetwork_region" {
56+
type = string
57+
description = "The subnetwork region of the shared VPC's host (for shared vpc support)"
58+
default = ""
59+
}
60+
3361
variable "zone" {
3462
description = "Zone where the Cloud Composer nodes are created."
3563
type = string
@@ -48,25 +76,98 @@ variable "machine_type" {
4876
default = "n1-standard-8"
4977
}
5078

51-
variable "network_name" {
52-
description = "Name of network created for Cloud Composer Environment."
79+
variable "composer_service_account" {
80+
description = "Service Account for running Cloud Composer."
5381
type = string
54-
default = "composer-network-01"
82+
default = null
5583
}
5684

57-
variable "subnet_name" {
58-
description = "Name of subnetwork created for Cloud Composer Environment."
85+
variable "disk_size" {
86+
description = "The disk size for nodes."
5987
type = string
60-
default = "composer-subnet-01"
88+
default = "100"
89+
}
90+
91+
variable "oauth_scopes" {
92+
description = "Google API scopes to be made available on all node."
93+
type = set(string)
94+
default = ["https://www.googleapis.com/auth/cloud-platform"]
6195
}
6296

63-
variable "ip_cidr_range" {
64-
description = "CIDR range for the Cloud Composer Subnet."
97+
variable "tags" {
98+
description = "Tags applied to all nodes. Tags are used to identify valid sources or targets for network firewalls."
99+
type = set(string)
100+
default = []
101+
}
102+
103+
variable "use_ip_aliases" {
104+
description = "Enable Alias IPs in the GKE cluster. If true, a VPC-native cluster is created."
105+
type = bool
106+
default = false
107+
}
108+
109+
variable "pod_ip_allocation_range_name" {
110+
description = "The name of the cluster's secondary range used to allocate IP addresses to pods."
65111
type = string
66-
default = "10.0.0.0/14"
112+
default = null
67113
}
68114

69-
variable "composer_service_account" {
70-
description = "Service Account for running Cloud Composer."
115+
variable "service_ip_allocation_range_name" {
71116
type = string
117+
description = "The name of the services' secondary range used to allocate IP addresses to the cluster."
118+
default = null
119+
}
120+
121+
variable "airflow_config_overrides" {
122+
type = map(string)
123+
description = "Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example \"core-dags_are_paused_at_creation\"."
124+
default = {}
125+
}
126+
127+
variable "env_variables" {
128+
type = map(string)
129+
description = "Variables of the airflow environment."
130+
default = {}
131+
}
132+
133+
variable "image_version" {
134+
type = string
135+
description = "The version of the aiflow running in the cloud composer environment."
136+
default = null
137+
}
138+
139+
variable "pypi_packages" {
140+
type = map(string)
141+
description = " Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. \"numpy\")."
142+
default = {}
143+
}
144+
145+
variable "python_version" {
146+
description = "The default version of Python used to run the Airflow scheduler, worker, and webserver processes."
147+
type = string
148+
default = "3"
149+
}
150+
151+
variable "cloud_sql_ipv4_cidr" {
152+
description = "The CIDR block from which IP range in tenant project will be reserved for Cloud SQL."
153+
type = string
154+
default = null
155+
}
156+
157+
variable "web_server_ipv4_cidr" {
158+
description = "The CIDR block from which IP range in tenant project will be reserved for the web server."
159+
type = string
160+
default = null
161+
}
162+
163+
variable "master_ipv4_cidr" {
164+
description = "The CIDR block from which IP range in tenant project will be reserved for the master."
165+
type = string
166+
default = null
167+
}
168+
169+
variable "enable_private_endpoint" {
170+
description = "Configure public access to the cluster endpoint."
171+
type = bool
172+
default = false
72173
}

‎test/fixtures/simple-composer-env/main.tf‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,14 @@
1717
module "simple-composer" {
1818
source = "../../../examples/simple_composer_env"
1919

20-
project_id = var.project_id
21-
composer_env_name = "composer-env-${random_id.random_suffix.hex}"
22-
region = "us-central1"
23-
composer_service_account = var.composer_sa
20+
project_id = var.project_id
21+
composer_env_name = "composer-env-${random_id.random_suffix.hex}"
22+
region = var.region
23+
composer_service_account = var.composer_sa
24+
network = google_compute_network.main.name
25+
subnetwork = google_compute_subnetwork.main.name
26+
pod_ip_allocation_range_name = google_compute_subnetwork.main.secondary_ip_range[0].range_name
27+
service_ip_allocation_range_name = google_compute_subnetwork.main.secondary_ip_range[1].range_name
2428
}
2529

2630
resource "random_id" "random_suffix" {

0 commit comments

Comments
 (0)