feat: add submodule to manage storage (#28)

* feat: add storage management module

* fix: rename to make it consistent with others

* feat: added storage module

* fix: cleanup

Author: jond3k

modules/airflow_storage/README.md

@@ -0,0 +1,37 @@
+# Module Cloud Composer Storage
+
+This optional module is used to configure Airflow objects like dags, plugins and data.
+
+Unfortunately this capability is not exposed via the Composer API so we need to use `gcloud`
+
+```hcl
+module "my-dag" {
+  source      = "terraform-google-modules/composer/google//modules/airflow_storage"
+  project_id  = "project-123"
+  environment = "Composer-Prod-Env"
+  location    = "us-central1"
+  type        = "dag"
+  source_path = "${path.root}/dags/my-dag"
+}
+```
+
+See the examples for cloudsql support.
+
+```
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| destination\_path | The optional destination path | `string` | `null` | no |
+| environment | n/a | `string` | n/a | yes |
+| location | n/a | `string` | n/a | yes |
+| project\_id | n/a | `string` | n/a | yes |
+| source\_path | The source on the local file system | `string` | n/a | yes |
+| type | The type of resource to upload. Either dag, plugin or data | `string` | n/a | yes |
+
+## Outputs
+
+No output.
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/airflow_storage/main.tf

@@ -0,0 +1,51 @@
+/**
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  # Emulate the default behaviour when --destination isn't set otherwise we don't know how to run the destroy command
+  implied_destination = coalesce(var.destination_path, basename(var.source_path))
+  create_parameters = {
+    "--project"     = var.project_id
+    "--location"    = var.location
+    "--environment" = var.environment
+    "--source"      = var.source_path
+    "--destination" = var.destination_path
+  }
+  destroy_parameters = {
+    "--project"     = var.project_id
+    "--location"    = var.location
+    "--environment" = var.environment
+  }
+
+  gcloud_cmd_body  = "composer environments storage ${var.type}"
+  create_cmd_body  = "${local.gcloud_cmd_body} import ${join(" ", [for k, v in local.create_parameters : "${k}=${v}" if v != null])}"
+  destroy_cmd_body = "${local.gcloud_cmd_body} delete ${join(" ", [for k, v in local.destroy_parameters : "${k}=${v}"])} ${local.implied_destination}"
+}
+
+
+module "gcloud" {
+  source           = "terraform-google-modules/gcloud/google"
+  version          = "~> 3.1"
+  platform         = "linux"
+  create_cmd_body  = local.create_cmd_body
+  destroy_cmd_body = local.destroy_cmd_body
+
+  create_cmd_triggers = {
+    filesha1 = join(",", [
+      for f in fileset(var.source_path, "**") : "${f}:${filesha1("${var.source_path}/${f}")}"
+    ])
+  }
+}

modules/airflow_storage/outputs.tf

@@ -0,0 +1,16 @@
+/**
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+

modules/airflow_storage/variables.tf

@@ -0,0 +1,43 @@
+/**
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  type = string
+}
+
+variable "location" {
+  type = string
+}
+
+variable "environment" {
+  type = string
+}
+
+variable "type" {
+  description = "The type of resource to upload. Either dag, plugin or data"
+  type        = string
+}
+
+variable "source_path" {
+  description = "The source on the local file system"
+  type        = string
+}
+
+variable "destination_path" {
+  description = "The optional destination path"
+  type        = string
+  default     = null
+}

modules/airflow_storage/versions.tf

@@ -0,0 +1,30 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  required_version = ">= 0.14"
+  required_providers {
+
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 3.53"
+    }
+  }
+
+  provider_meta "google" {
+    module_name = "blueprints/terraform/terraform-google-composer:airflow_storage/v2.1.0"
+  }
+}