feat!: Cloud composer v2 (#36)

* refactor: separate packages by Airflow version

* feature: new cloud composer (v2) version

* feature: license

* feature: add roles/composer.ServiceAgentV2Ext to composer api agent

* fix: rename simple composer env dir

* fix: cloud build kitchen-tf test integration

* fix: cloud build kitchen-tf test integration

* fix: cloud build kitchen-tf test integration

* refactor: license year + docs

* fix: lint

* fix: master_authorized_networks_config flag

* update

Author: diegolnasc

README.md

@@ -7,8 +7,14 @@ The resources/services/activations/deletions that this module will create/trigge
 - Create a GCP Composer Environment
 
 ## Usage
+Cloud Composer currently has two versions: V2 has greater flexibility in the Airflow core services
+(scheduler, webserver, worker) and has a more practical and scalable infrastructure. Therefore, we recommend prioritizing the
+use of V2 for new environments.
 
-Basic usage of this module is as follows:
+You can find an overview of the product [here](https://cloud.google.com/composer/docs/composer-2/composer-overview)
+and the [list of major differences](https://cloud.google.com/composer/docs/concepts/versioning/composer-versioning-overview).
+
+Simple usage is as follows:
 
 ```hcl
 module "composer" {

build/int.cloudbuild.yaml

@@ -28,23 +28,41 @@ steps:
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create']
 
-# ----- SUITE simple-composer-env-local
+# ----- SUITE simple-composer-env-v1-local
 
-- id: converge simple-composer-env-local
+- id: converge simple-composer-env-v1-local
   waitFor:
     - create all
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-composer-env-local']
-- id: verify simple-composer-env-local
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-composer-env-v1-local']
+- id: verify simple-composer-env-v1-local
   waitFor:
-    - converge simple-composer-env-local
+    - converge simple-composer-env-v1-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-composer-env-local']
-- id: destroy simple-composer-env-local
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-composer-env-v1-local']
+- id: destroy simple-composer-env-v1-local
   waitFor:
-    - verify simple-composer-env-local
+    - verify simple-composer-env-v1-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-composer-env-local']
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-composer-env-v1-local']
+
+  # ----- SUITE simple-composer-env-v2-local
+
+- id: converge simple-composer-env-v2-local
+  waitFor:
+    - create all
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-composer-env-v2-local']
+- id: verify simple-composer-env-v2-local
+  waitFor:
+    - converge simple-composer-env-v2-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-composer-env-v2-local']
+- id: destroy simple-composer-env-v2-local
+  waitFor:
+    - verify simple-composer-env-v2-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-composer-env-v2-local']
 
 # ----- SUITE airflow-connection-local
 

examples/airflow_connection/main.tf

@@ -30,7 +30,7 @@ provider "external" {
 }
 
 module "simple-composer-environment" {
-  source                           = "../../modules/create_environment"
+  source                           = "../../modules/create_environment_v1"
   project_id                       = var.project_id
   composer_env_name                = var.composer_env_name
   region                           = var.region
@@ -40,6 +40,8 @@ module "simple-composer-environment" {
   use_ip_aliases                   = true
   pod_ip_allocation_range_name     = var.pod_ip_allocation_range_name
   service_ip_allocation_range_name = var.service_ip_allocation_range_name
+  node_count                       = 3
+  machine_type                     = "n1-standard-1"
 }
 
 # Making the k8s master globally available is only to make the integration testing portable and should be removed

examples/airflow_pool/main.tf

@@ -30,7 +30,7 @@ provider "external" {
 }
 
 module "simple-composer-environment" {
-  source                           = "../../modules/create_environment"
+  source                           = "../../modules/create_environment_v1"
   project_id                       = var.project_id
   composer_env_name                = var.composer_env_name
   region                           = var.region
@@ -40,6 +40,8 @@ module "simple-composer-environment" {
   use_ip_aliases                   = true
   pod_ip_allocation_range_name     = var.pod_ip_allocation_range_name
   service_ip_allocation_range_name = var.service_ip_allocation_range_name
+  node_count                       = 3
+  machine_type                     = "n1-standard-1"
 }
 
 # Making the k8s master globally available is only to make the integration testing portable and should be removed

examples/simple_composer_env/README.md renamed to examples/simple_composer_env_v1/README.md

@@ -1,4 +1,4 @@
-# Simple Cloud Composer Environment Example
+# Simple Cloud Composer Environment (V1) Example
 
 This example illustrates how to use the `composer` module.
 

examples/simple_composer_env/main.tf renamed to examples/simple_composer_env_v1/main.tf

@@ -15,7 +15,7 @@
  */
 
 module "simple-composer-environment" {
-  source                           = "../../modules/create_environment"
+  source                           = "../../modules/create_environment_v1"
   project_id                       = var.project_id
   composer_env_name                = var.composer_env_name
   region                           = var.region
@@ -25,4 +25,6 @@ module "simple-composer-environment" {
   use_ip_aliases                   = true
   pod_ip_allocation_range_name     = var.pod_ip_allocation_range_name
   service_ip_allocation_range_name = var.service_ip_allocation_range_name
+  node_count                       = 3
+  machine_type                     = "n1-standard-1"
 }

examples/simple_composer_env/outputs.tf renamed to examples/simple_composer_env_v1/outputs.tf

@@ -0,0 +1,35 @@
+# Simple Cloud Composer Environment (V2) Example
+
+This example illustrates how to use the `composer` module.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| composer\_env\_name | Name of Cloud Composer Environment. | `string` | `"ci-composer"` | no |
+| composer\_service\_account | Service Account to be used for running Cloud Composer Environment. | `string` | n/a | yes |
+| network | Network where Cloud Composer is created. | `string` | n/a | yes |
+| pod\_ip\_allocation\_range\_name | The name of the cluster's secondary range used to allocate IP addresses to pods. | `string` | n/a | yes |
+| project\_id | Project ID where Cloud Composer Environment is created. | `string` | n/a | yes |
+| region | Region where Cloud Composer Environment is created. | `string` | n/a | yes |
+| service\_ip\_allocation\_range\_name | The name of the services' secondary range used to allocate IP addresses to the cluster. | `string` | n/a | yes |
+| subnetwork | Subetwork where Cloud Composer is created. | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| airflow\_uri | URI of the Apache Airflow Web UI hosted within the Cloud Composer Environment. |
+| composer\_env\_id | ID of Cloud Composer Environment. |
+| composer\_env\_name | Name of the Cloud Composer Environment. |
+| gcs\_bucket | Google Cloud Storage bucket which hosts DAGs for the Cloud Composer Environment. |
+| gke\_cluster | Google Kubernetes Engine cluster used to run the Cloud Composer Environment. |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+To provision this example, run the following from within this directory:
+- `terraform init` to get the plugins
+- `terraform plan` to see the infrastructure plan
+- `terraform apply` to apply the infrastructure build
+- `terraform destroy` to destroy the built infrastructure