feat!: Add connection metadata, output and support for ml integration for Mysql module (#637)

Co-authored-by: Imran Nayer <[email protected]>

Author: q2w

Makefile

@@ -77,6 +77,7 @@ docker_test_lint:
 .PHONY: docker_generate_docs
 docker_generate_docs:
 	docker run --rm -it \
+		-e ENABLE_BPMETADATA \
 		-v "$(CURDIR)":/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs'

examples/mysql-public/README.md

@@ -31,6 +31,7 @@ terraform destroy
 
 | Name | Description |
 |------|-------------|
+| env\_vars | Exported environment variables |
 | mysql\_conn | The connection name of the master instance to be used in connection strings |
 | mysql\_user\_pass | The password for the default user. If not set, a random one will be generated and available in the generated\_user\_password output variable. |
 | name | The name for Cloud SQL instance |

examples/mysql-public/outputs.tf

@@ -45,3 +45,7 @@ output "private_ip_address" {
   value       = module.mysql-db.private_ip_address
 }
 
+output "env_vars" {
+  value       = module.mysql-db.env_vars
+  description = "Exported environment variables"
+}

metadata.yaml

@@ -68,6 +68,8 @@ spec:
         location: examples/postgresql-public-iam
       - name: postgresql-with-cross-region-failover
         location: examples/postgresql-with-cross-region-failover
+      - name: private_service_access
+        location: examples/private_service_access
   interfaces: {}
   requirements:
     roles:

modules/backup/metadata.yaml

@@ -57,6 +57,8 @@ spec:
         location: examples/postgresql-public-iam
       - name: postgresql-with-cross-region-failover
         location: examples/postgresql-with-cross-region-failover
+      - name: private_service_access
+        location: examples/private_service_access
   interfaces:
     variables:
       - name: backup_monitoring_frequency

modules/mssql/metadata.yaml

@@ -58,6 +58,8 @@ spec:
         location: examples/postgresql-public-iam
       - name: postgresql-with-cross-region-failover
         location: examples/postgresql-with-cross-region-failover
+      - name: private_service_access
+        location: examples/private_service_access
   interfaces:
     variables:
       - name: activation_policy

modules/mysql/README.md

@@ -16,6 +16,7 @@ Note: CloudSQL provides [disk autoresize](https://cloud.google.com/sql/docs/mysq
 | create\_timeout | The optional timout that is applied to limit long database creates. | `string` | `"30m"` | no |
 | data\_cache\_enabled | Whether data cache is enabled for the instance. Defaults to false. Feature is only available for ENTERPRISE\_PLUS tier and supported database\_versions | `bool` | `false` | no |
 | database\_flags | List of Cloud SQL flags that are applied to the database server. See [more details](https://cloud.google.com/sql/docs/mysql/flags) | <pre>list(object({<br>    name  = string<br>    value = string<br>  }))</pre> | `[]` | no |
+| database\_integration\_roles | The roles required by default database instance service account for integration with GCP services | `list(string)` | `[]` | no |
 | database\_version | The database version to use | `string` | n/a | yes |
 | db\_charset | The charset for the default database | `string` | `""` | no |
 | db\_collation | The collation for the default database. Example: 'utf8\_general\_ci' | `string` | `""` | no |
@@ -31,6 +32,7 @@ Note: CloudSQL provides [disk autoresize](https://cloud.google.com/sql/docs/mysq
 | edition | The edition of the instance, can be ENTERPRISE or ENTERPRISE\_PLUS. | `string` | `null` | no |
 | enable\_default\_db | Enable or disable the creation of the default database | `bool` | `true` | no |
 | enable\_default\_user | Enable or disable the creation of the default user | `bool` | `true` | no |
+| enable\_google\_ml\_integration | Enable database ML integration | `bool` | `false` | no |
 | enable\_random\_password\_special | Enable special characters in generated random passwords. | `bool` | `false` | no |
 | encryption\_key\_name | The full path to the encryption key used for the CMEK disk encryption | `string` | `null` | no |
 | follow\_gae\_application | A Google App Engine application whose zone to remain in. Must be in the same region as this instance. | `string` | `null` | no |
@@ -70,6 +72,7 @@ Note: CloudSQL provides [disk autoresize](https://cloud.google.com/sql/docs/mysq
 | Name | Description |
 |------|-------------|
 | additional\_users | List of maps of additional users and passwords |
+| env\_vars | Exported environment variables |
 | generated\_user\_password | The auto generated default user password if not input password was provided |
 | iam\_users | The list of the IAM users with access to the CloudSQL instance |
 | instance\_connection\_name | The connection name of the master instance to be used in connection strings |

modules/mysql/main.tf

@@ -41,6 +41,8 @@ locals {
 
   // Force the usage of connector_enforcement
   connector_enforcement = var.connector_enforcement ? "REQUIRED" : "NOT_REQUIRED"
+
+  database_name = var.enable_default_db ? var.db_name : (length(var.additional_databases) > 0 ? var.additional_databases[0].name : "")
 }
 
 resource "random_id" "suffix" {
@@ -62,12 +64,13 @@ resource "google_sql_database_instance" "default" {
   root_password        = var.root_password == "" ? null : var.root_password
 
   settings {
-    tier                        = var.tier
-    edition                     = var.edition
-    activation_policy           = var.activation_policy
-    availability_type           = var.availability_type
-    deletion_protection_enabled = var.deletion_protection_enabled
-    connector_enforcement       = local.connector_enforcement
+    tier                         = var.tier
+    edition                      = var.edition
+    activation_policy            = var.activation_policy
+    availability_type            = var.availability_type
+    deletion_protection_enabled  = var.deletion_protection_enabled
+    connector_enforcement        = local.connector_enforcement
+    enable_google_ml_integration = var.enable_google_ml_integration
 
     dynamic "backup_configuration" {
       for_each = [var.backup_configuration]
@@ -308,6 +311,13 @@ resource "google_sql_user" "iam_account" {
   deletion_policy = var.user_deletion_policy
 }
 
+resource "google_project_iam_member" "database_integration" {
+  for_each = toset(var.database_integration_roles)
+  project  = var.project_id
+  role     = each.value
+  member   = "serviceAccount:${google_sql_database_instance.default.service_account_email_address}"
+}
+
 resource "null_resource" "module_depends_on" {
   triggers = {
     value = length(var.module_depends_on)

modules/mysql/metadata.yaml

@@ -58,6 +58,8 @@ spec:
         location: examples/postgresql-public-iam
       - name: postgresql-with-cross-region-failover
         location: examples/postgresql-with-cross-region-failover
+      - name: private_service_access
+        location: examples/private_service_access
   interfaces:
     variables:
       - name: activation_policy
@@ -122,6 +124,10 @@ spec:
               value = string
             }))
         defaultValue: []
+      - name: database_integration_roles
+        description: The roles required by default database instance service account for integration with GCP services
+        varType: list(string)
+        defaultValue: []
       - name: database_version
         description: The database version to use
         varType: string
@@ -186,6 +192,10 @@ spec:
         description: Enable or disable the creation of the default user
         varType: bool
         defaultValue: true
+      - name: enable_google_ml_integration
+        description: Enable database ML integration
+        varType: bool
+        defaultValue: false
       - name: enable_random_password_special
         description: Enable special characters in generated random passwords.
         varType: bool
@@ -204,6 +214,17 @@ spec:
               email = string
             }))
         defaultValue: []
+        connections:
+          - source:
+              source: github.com/terraform-google-modules/terraform-google-service-accounts//modules/simple-sa
+              version: v4.3.0
+            spec:
+              outputExpr: id
+          - source:
+              source: github.com/GoogleCloudPlatform/terraform-google-cloud-run//modules/v2
+              version: v0.13.0
+            spec:
+              outputExpr: service_account_id
       - name: insights_config
         description: The insights_config settings for the database.
         varType: |-
@@ -231,6 +252,13 @@ spec:
               psc_allowed_consumer_projects                 = optional(list(string), [])
             })
         defaultValue: {}
+        connections:
+          - source:
+              source: github.com/terraform-google-modules/terraform-google-network//modules/vpc
+              version: v9.1.0
+            spec:
+              outputExpr: network_id
+              inputPath: private_network
       - name: maintenance_window_day
         description: The day of week (1-7) for the master instance maintenance.
         varType: number
@@ -377,6 +405,8 @@ spec:
     outputs:
       - name: additional_users
         description: List of maps of additional users and passwords
+      - name: env_vars
+        description: Exported environment variables
       - name: generated_user_password
         description: The auto generated default user password if not input password was provided
       - name: iam_users

modules/mysql/outputs.tf

@@ -145,3 +145,12 @@ output "instances" {
   description = "A list of all `google_sql_database_instance` resources we've created"
   sensitive   = true
 }
+
+output "env_vars" {
+  description = "Exported environment variables"
+  value = {
+    "CLOUD_SQL_DATABASE_HOST" : google_sql_database_instance.default.first_ip_address,
+    "CLOUD_SQL_DATABASE_CONNECTION_NAME" : google_sql_database_instance.default.connection_name,
+    "CLOUD_SQL_DATABASE_NAME" : local.database_name
+  }
+}