fix: duplicate maintenance_window (#100)

imrannayer · web-flow · commit 82e3c041d90c · 2023-11-22T09:19:58.000-06:00
diff --git a/examples/simple_composer_env_v2/README.md b/examples/simple_composer_env_v2/README.md
@@ -1,6 +1,8 @@
 # Simple Cloud Composer Environment (V2) Example
 
-This example illustrates how to use the `composer` module.
+This example illustrates how to use the `composer` V2 module to deploy private composer environment with private GKE cluster, CloudSql instance and use private service connect (PSC) endpoint to connect to CloudSql instance. `use_private_environment` and `enable_private_endpoint` needs to be `true` to deploy private GKE cluster and CloudSql instance. `cloud_composer_connection_subnetwork` value will result in PSC endpoint for CloudSql instance.
+
+This example also creates a Cloud Storage Bucket for scheduled snapshots and assign appropriate permission(s) to Composer Service Account on the bucket.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
@@ -14,7 +16,8 @@ This example illustrates how to use the `composer` module.
 | project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes |
 | region | Region where Cloud Composer Environment is created. | `string` | n/a | yes |
 | service\_ip\_allocation\_range\_name | The name of the services' secondary range used to allocate IP addresses to the cluster. | `string` | n/a | yes |
-| subnetwork | Subetwork where Cloud Composer is created. | `string` | n/a | yes |
+| subnetwork | Name of the Subetwork where Cloud Composer is created. | `string` | n/a | yes |
+| subnetwork\_self\_link | self\_link of the Subetwork where Cloud Composer is created. | `string` | n/a | yes |
 
 ## Outputs
 
diff --git a/examples/simple_composer_env_v2/main.tf b/examples/simple_composer_env_v2/main.tf
@@ -36,31 +36,36 @@ resource "google_storage_bucket_iam_member" "object_admin" {
 }
 
 module "simple-composer-environment" {
-  source                           = "../../modules/create_environment_v2"
-  project_id                       = var.project_id
-  composer_env_name                = var.composer_env_name
-  region                           = var.region
-  composer_service_account         = var.composer_service_account
-  network                          = var.network
-  subnetwork                       = var.subnetwork
-  pod_ip_allocation_range_name     = var.pod_ip_allocation_range_name
-  service_ip_allocation_range_name = var.service_ip_allocation_range_name
-  grant_sa_agent_permission        = false
-  environment_size                 = "ENVIRONMENT_SIZE_SMALL"
-  enable_private_endpoint          = true
-  use_private_environment          = true
+  source                               = "../../modules/create_environment_v2"
+  project_id                           = var.project_id
+  composer_env_name                    = var.composer_env_name
+  region                               = var.region
+  composer_service_account             = var.composer_service_account
+  network                              = var.network
+  subnetwork                           = var.subnetwork
+  pod_ip_allocation_range_name         = var.pod_ip_allocation_range_name
+  service_ip_allocation_range_name     = var.service_ip_allocation_range_name
+  grant_sa_agent_permission            = false
+  environment_size                     = "ENVIRONMENT_SIZE_SMALL"
+  enable_private_endpoint              = true
+  use_private_environment              = true
+  cloud_composer_connection_subnetwork = var.subnetwork_self_link
+  cloud_data_lineage_integration       = true
+  resilience_mode                      = "STANDARD_RESILIENCE"
 
   scheduler = {
     cpu        = 0.5
     memory_gb  = 1.875
     storage_gb = 1
     count      = 2
+
   }
   web_server = {
     cpu        = 0.5
     memory_gb  = 1.875
     storage_gb = 1
   }
+
   worker = {
     cpu        = 0.5
     memory_gb  = 1.875
@@ -69,18 +74,22 @@ module "simple-composer-environment" {
     max_count  = 3
   }
 
+  triggerer = {
+    cpu       = 1
+    memory_gb = 1
+    count     = 2
+  }
+
   scheduled_snapshots_config = {
     enabled                    = true
     snapshot_location          = google_storage_bucket.my_bucket.url
     snapshot_creation_schedule = "0 4 * * *"
     time_zone                  = "UTC+01"
   }
 
-  maintenance_window = {
-    start_time = "2023-01-01T00:00:00Z"
-    end_time   = "2023-01-01T12:00:00Z"
-    recurrence = "FREQ=WEEKLY;BYDAY=SU,WE,SA"
-  }
+  maintenance_start_time = "2023-01-01T00:00:00Z"
+  maintenance_end_time   = "2023-01-01T12:00:00Z"
+  maintenance_recurrence = "FREQ=WEEKLY;BYDAY=SU,SA"
 
   depends_on = [
     google_storage_bucket_iam_member.object_admin,
diff --git a/examples/simple_composer_env_v2/variables.tf b/examples/simple_composer_env_v2/variables.tf
@@ -41,7 +41,12 @@ variable "network" {
 }
 
 variable "subnetwork" {
-  description = "Subetwork where Cloud Composer is created."
+  description = "Name of the Subetwork where Cloud Composer is created."
+  type        = string
+}
+
+variable "subnetwork_self_link" {
+  description = "self_link of the Subetwork where Cloud Composer is created."
   type        = string
 }
 
diff --git a/modules/create_environment_v2/README.md b/modules/create_environment_v2/README.md
@@ -1,57 +1,96 @@
 # Module Cloud Composer Environment ([V2](https://cloud.google.com/composer/docs/composer-2/composer-overview))
 
-This optional module is used to create a Cloud Composer environment.
+This module is used to create a Cloud Composer V2 environment.
+
 
 ```hcl
-module "composer" {
-  source = "terraform-google-modules/composer/google//modules/create_environment_v2"
+module "simple-composer-environment" {
+  source                               = "terraform-google-modules/composer/google//modules/create_environment_v2"
+  version                              = "~> 4.0"
+  project_id                           = var.project_id
+  composer_env_name                    = "test-composer-env"
+  region                               = "us-central1"
+  composer_service_account             = var.composer_service_account
+  network                              = "test-vpc"
+  subnetwork                           = "test-subnet"
+  pod_ip_allocation_range_name         = "test-subnet-pod-ip-name"
+  service_ip_allocation_range_name     = "test-subnet-service-ip-name"
+  grant_sa_agent_permission            = false
+  environment_size                     = "ENVIRONMENT_SIZE_SMALL"
+  enable_private_endpoint              = true
+  use_private_environment              = true
+  cloud_composer_connection_subnetwork = var.subnetwork_self_link
+
+  scheduler = {
+    cpu        = 0.5
+    memory_gb  = 1.875
+    storage_gb = 1
+    count      = 2
+  }
+
+  web_server = {
+    cpu        = 0.5
+    memory_gb  = 1.875
+    storage_gb = 1
+  }
+
+  worker = {
+    cpu        = 0.5
+    memory_gb  = 1.875
+    storage_gb = 1
+    min_count  = 2
+    max_count  = 3
+  }
+
+  triggerer = {
+    cpu       = 1
+    memory_gb = 1
+    count     = 2
+  }
 
-  project = "project-123"
-  name    = "Composer-Prod-Env"
-  region  = "us-central1"
 }
+
 ```
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | airflow\_config\_overrides | Airflow configuration properties to override. Property keys contain the section and property names, separated by a hyphen, for example "core-dags\_are\_paused\_at\_creation". | `map(string)` | `{}` | no |
-| cloud\_composer\_connection\_subnetwork | When specified, the environment will use Private Service Connect instead of VPC peerings to connect to Cloud SQL in the Tenant Project | `string` | `null` | no |
-| cloud\_composer\_network\_ipv4\_cidr\_block | The CIDR block from which IP range in tenant project will be reserved. | `string` | `null` | no |
+| cloud\_composer\_connection\_subnetwork | Subnetwork self-link. When specified, the environment will use Private Service Connect instead of VPC peerings to connect to CloudSQL in the Tenant Project. IP address of psc endpoint is allocated from this subnet | `string` | `null` | no |
+| cloud\_composer\_network\_ipv4\_cidr\_block | The CIDR block from which IP range in tenant project will be reserved. Required if VPC peering is used to connect to CloudSql instead of PSC | `string` | `null` | no |
 | cloud\_data\_lineage\_integration | Whether or not Dataplex data lineage integration is enabled. Cloud Composer environments in versions composer-2.1.2-airflow-..* and newer) | `bool` | `false` | no |
-| cloud\_sql\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for Cloud SQL. | `string` | `null` | no |
+| cloud\_sql\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for Cloud SQL private service access. Required if VPC peering is used to connect to CloudSql instead of PSC | `string` | `null` | no |
 | composer\_env\_name | Name of Cloud Composer Environment | `string` | n/a | yes |
 | composer\_service\_account | Service Account for running Cloud Composer. | `string` | `null` | no |
 | enable\_ip\_masq\_agent | Deploys 'ip-masq-agent' daemon set in the GKE cluster and defines nonMasqueradeCIDRs equals to pod IP range so IP masquerading is used for all destination addresses, except between pods traffic. | `bool` | `false` | no |
-| enable\_private\_endpoint | Configure public access to the cluster endpoint. | `bool` | `false` | no |
+| enable\_private\_endpoint | Configure private access to the cluster endpoint. If true, access to the public endpoint of the GKE cluster is denied | `bool` | `false` | no |
 | env\_variables | Variables of the airflow environment. | `map(string)` | `{}` | no |
-| environment\_size | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. Values for environment size are: ENVIRONMENT\_SIZE\_SMALL, ENVIRONMENT\_SIZE\_MEDIUM, and ENVIRONMENT\_SIZE\_LARGE. | `string` | `"ENVIRONMENT_SIZE_MEDIUM"` | no |
+| environment\_size | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. Values for environment size are: `ENVIRONMENT_SIZE_SMALL`, `ENVIRONMENT_SIZE_MEDIUM`, and `ENVIRONMENT_SIZE_LARGE`. | `string` | `"ENVIRONMENT_SIZE_MEDIUM"` | no |
 | grant\_sa\_agent\_permission | Cloud Composer relies on Workload Identity as Google API authentication mechanism for Airflow. | `bool` | `true` | no |
 | image\_version | The version of the aiflow running in the cloud composer environment. | `string` | `"composer-2.5.0-airflow-2.6.3"` | no |
 | labels | The resource labels (a map of key/value pairs) to be applied to the Cloud Composer. | `map(string)` | `{}` | no |
 | maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `null` | no |
 | maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `null` | no |
 | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no |
-| maintenance\_window | The recovery configuration settings for the Cloud Composer environment | <pre>object({<br>    start_time = string<br>    end_time   = string<br>    recurrence = string<br>  })</pre> | `null` | no |
 | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | <pre>list(object({<br>    cidr_block   = string<br>    display_name = string<br>  }))</pre> | `[]` | no |
-| master\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for the master. | `string` | `null` | no |
+| master\_ipv4\_cidr | The CIDR block from which IP range in tenant project will be reserved for the GKE master. Required when `use_private_environment` and `enable_private_endpoint` is `true` | `string` | `null` | no |
 | network | The VPC network to host the composer cluster. | `string` | n/a | yes |
 | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no |
-| pod\_ip\_allocation\_range\_name | The name of the cluster's secondary range used to allocate IP addresses to pods. | `string` | `null` | no |
+| pod\_ip\_allocation\_range\_name | The name of the subnet secondary range, used to allocate IP addresses for the pods. | `string` | `null` | no |
 | project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes |
 | pypi\_packages | Custom Python Package Index (PyPI) packages to be installed in the environment. Keys refer to the lowercase package name (e.g. "numpy"). | `map(string)` | `{}` | no |
 | region | Region where the Cloud Composer Environment is created. | `string` | `"us-central1"` | no |
 | resilience\_mode | Cloud Composer 2.1.15 or newer only. The resilience mode states whether high resilience is enabled for the environment or not. Values for resilience mode are `HIGH_RESILIENCE` for high resilience and `STANDARD_RESILIENCE` for standard resilience | `string` | `null` | no |
 | scheduled\_snapshots\_config | The recovery configuration settings for the Cloud Composer environment | <pre>object({<br>    enabled                    = optional(bool, false)<br>    snapshot_location          = optional(string)<br>    snapshot_creation_schedule = optional(string)<br>    time_zone                  = optional(string)<br>  })</pre> | `null` | no |
 | scheduler | Configuration for resources used by Airflow schedulers. | <pre>object({<br>    cpu        = string<br>    memory_gb  = number<br>    storage_gb = number<br>    count      = number<br>  })</pre> | <pre>{<br>  "count": 2,<br>  "cpu": 2,<br>  "memory_gb": 7.5,<br>  "storage_gb": 5<br>}</pre> | no |
-| service\_ip\_allocation\_range\_name | The name of the services' secondary range used to allocate IP addresses to the cluster. | `string` | `null` | no |
+| service\_ip\_allocation\_range\_name | The name of the subnet secondary range, used to allocate IP addresses for the Services. | `string` | `null` | no |
 | storage\_bucket | Name of an existing Cloud Storage bucket to be used by the environment | `string` | `null` | no |
-| subnetwork | The subnetwork to host the composer cluster. | `string` | n/a | yes |
+| subnetwork | The name of the subnetwork to host the composer cluster. | `string` | n/a | yes |
 | subnetwork\_region | The subnetwork region of the shared VPC's host (for shared vpc support) | `string` | `""` | no |
 | tags | Tags applied to all nodes. Tags are used to identify valid sources or targets for network firewalls. | `set(string)` | `[]` | no |
 | triggerer | Configuration for resources used by Airflow triggerer | <pre>object({<br>    cpu       = string<br>    memory_gb = number<br>    count     = number<br>  })</pre> | `null` | no |
-| use\_private\_environment | Enable private environment. | `bool` | `false` | no |
+| use\_private\_environment | Create a private environment. | `bool` | `false` | no |
 | web\_server | Configuration for resources used by Airflow web server. | <pre>object({<br>    cpu        = string<br>    memory_gb  = number<br>    storage_gb = number<br>  })</pre> | <pre>{<br>  "cpu": 2,<br>  "memory_gb": 7.5,<br>  "storage_gb": 5<br>}</pre> | no |
 | worker | Configuration for resources used by Airflow workers. | <pre>object({<br>    cpu        = string<br>    memory_gb  = number<br>    storage_gb = number<br>    min_count  = number<br>    max_count  = number<br>  })</pre> | <pre>{<br>  "cpu": 2,<br>  "max_count": 6,<br>  "memory_gb": 7.5,<br>  "min_count": 2,<br>  "storage_gb": 5<br>}</pre> | no |
 
@@ -60,6 +99,7 @@ module "composer" {
 | Name | Description |
 |------|-------------|
 | airflow\_uri | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. |
+| composer\_env | Cloud Composer Environment |
 | composer\_env\_id | ID of Cloud Composer Environment. |
 | composer\_env\_name | Name of the Cloud Composer Environment. |
 | gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. |
diff --git a/modules/create_environment_v2/main.tf b/modules/create_environment_v2/main.tf
@@ -186,16 +186,6 @@ resource "google_composer_environment" "composer_env" {
       }
     }
 
-    dynamic "maintenance_window" {
-      for_each = var.maintenance_window != null ? [var.maintenance_window] : []
-      content {
-        start_time = maintenance_window.value["start_time"]
-        end_time   = maintenance_window.value["end_time"]
-        recurrence = maintenance_window.value["recurrence"]
-      }
-
-    }
-
   }
 
   depends_on = [google_project_iam_member.composer_agent_service_account]
diff --git a/modules/create_environment_v2/outputs.tf b/modules/create_environment_v2/outputs.tf
@@ -38,3 +38,8 @@ output "airflow_uri" {
   value       = google_composer_environment.composer_env.config[0].airflow_uri
   description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment."
 }
+
+output "composer_env" {
+  value       = google_composer_environment.composer_env
+  description = "Cloud Composer Environment"
+}
diff --git a/modules/create_environment_v2/variables.tf b/modules/create_environment_v2/variables.tf
diff --git a/test/fixtures/simple-composer-env-v2/main.tf b/test/fixtures/simple-composer-env-v2/main.tf
diff --git a/test/setup/main.tf b/test/setup/main.tf

Original file line number	Diff line number	Diff line change
`@@ -41,7 +41,12 @@ variable "network" {`
`41`	`41`	`}`
`42`	`42`
`43`	`43`	`variable "subnetwork" {`
`44`		`- description = "Subetwork where Cloud Composer is created."`
	`44`	`+ description = "Name of the Subetwork where Cloud Composer is created."`
	`45`	`+ type = string`
	`46`	`+}`
	`47`	`+`
	`48`	`+variable "subnetwork_self_link" {`
	`49`	`+ description = "self_link of the Subetwork where Cloud Composer is created."`
`45`	`50`	`type = string`
`46`	`51`	`}`
`47`	`52`
Original file line number	Diff line number	Diff line change
`@@ -186,16 +186,6 @@ resource "google_composer_environment" "composer_env" {`
`186`	`186`	`}`
`187`	`187`	`}`
`188`	`188`
`189`		`- dynamic "maintenance_window" {`
`190`		`- for_each = var.maintenance_window != null ? [var.maintenance_window] : []`
`191`		`- content {`
`192`		`- start_time = maintenance_window.value["start_time"]`
`193`		`- end_time = maintenance_window.value["end_time"]`
`194`		`- recurrence = maintenance_window.value["recurrence"]`
`195`		`- }`
`196`		`-`
`197`		`- }`
`198`		`-`
`199`	`189`	`}`
`200`	`190`
`201`	`191`	`depends_on = [google_project_iam_member.composer_agent_service_account]`
Original file line number	Diff line number	Diff line change
`@@ -38,3 +38,8 @@ output "airflow_uri" {`
`38`	`38`	`value = google_composer_environment.composer_env.config[0].airflow_uri`
`39`	`39`	`description = "URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment."`
`40`	`40`	`}`
	`41`	`+`
	`42`	`+output "composer_env" {`
	`43`	`+ value = google_composer_environment.composer_env`
	`44`	`+ description = "Cloud Composer Environment"`
	`45`	`+}`