feat: Provide an option to compress backups for PostreSQL and MySQL (#335)

* Provide an option to compress backups for PostreSQL and MySQL

* Fix formatting

* Extend API Activation waiting time in order to make more build succeed

* Fix Whitespaces and adapt test for provided backup

Author: gleichda

build/int.cloudbuild.yaml

@@ -192,4 +192,4 @@ tags:
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
   _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.3'
-  _API_ACTIVATION_SECONDS_WAIT: '600'
+  _API_ACTIVATION_SECONDS_WAIT: '900'

examples/mysql-backup-create-service-account/main.tf

@@ -51,4 +51,5 @@ module "backup" {
   backup_retention_time = 1
   backup_schedule       = "5 * * * *"
   export_schedule       = "10 * * * *"
+  compress_export       = true
 }

examples/mysql-backup-create-service-account/outputs.tf

@@ -43,3 +43,8 @@ output "instance_name" {
   value       = module.mysql.instance_name
   description = "The name of the SQL instance"
 }
+
+output "mysql-password" {
+  value     = module.mysql.generated_user_password
+  sensitive = true
+}

modules/backup/README.md

@@ -55,6 +55,7 @@ fetch workflows.googleapis.com/Workflow
 |------|-------------|------|---------|:--------:|
 | backup\_retention\_time | The number of days backups should be kept | `number` | `30` | no |
 | backup\_schedule | The cron schedule to execute the internal backup | `string` | `"45 2 * * *"` | no |
+| compress\_export | Whether or not to compress the export when storing in the bucket; Only valid for MySQL and PostgreSQL | `bool` | `false` | no |
 | enable\_export\_backup | Weather to create exports to GCS Buckets with this module | `bool` | `true` | no |
 | enable\_internal\_backup | Wether to create internal backups with this module | `bool` | `true` | no |
 | export\_databases | The list of databases that should be exported - if is an empty set all databases will be exported | `set(string)` | `[]` | no |

modules/backup/main.tf

@@ -109,6 +109,7 @@ resource "google_workflows_workflow" "sql_export" {
     databases           = jsonencode(var.export_databases)
     gcsBucket           = var.export_uri
     dbType              = split("_", data.google_sql_database_instance.backup_instance.database_version)[0]
+    compressExport      = var.compress_export
   })
 }
 

modules/backup/templates/export.yaml.tftpl

@@ -51,7 +51,7 @@ main:
                 body:
                   exportContext:
                     databases: [$${database}]
-                    uri: $${"${gcsBucket}/${instanceName}-" + database + "-" + backupTime + ".sql"}
+                    uri: $${"${gcsBucket}/${instanceName}-" + database + "-" + backupTime + %{ if compressExport == true }".sql.gz"%{ else }".sql"%{ endif }}
 %{ endif }
 
 %{ if dbType == "SQLSERVER" }
@@ -85,6 +85,6 @@ main:
         body:
           exportContext:
             databases: $${databases}
-            uri: $${"${gcsBucket}/${instanceName}-" + backupTime + ".sql"}
+            uri: $${"${gcsBucket}/${instanceName}-" + backupTime + %{ if compressExport == true }".sql.gz"%{ else }".sql"%{ endif }}
 %{ endif }
 

modules/backup/variables.tf

@@ -84,6 +84,13 @@ variable "export_uri" {
   type        = string
   validation {
     condition     = can(regex("^gs:\\/\\/", var.export_uri))
-    error_message = "Must be a full GCS URI starting with gs://." #TODO: test
+    error_message = "Must be a full GCS URI starting with gs://."
   }
 }
+
+#TODO: gleichda change default to true on next major release (#336)
+variable "compress_export" {
+  description = "Whether or not to compress the export when storing in the bucket; Only valid for MySQL and PostgreSQL"
+  type        = bool
+  default     = false
+}

test/integration/mysql-backup-create-service-account/mysql_backup_create_service_account_test.go

@@ -55,7 +55,7 @@ func TestMySqlBackupModuleCreateServiceAccount(t *testing.T) {
 			fmt.Sprintf("instance: %s", instanceName),
 			fmt.Sprintf("project: %s", projectID),
 			"- backupTime: ${string(int(sys.now()))}",
-			fmt.Sprintf("uri: ${\"gs://%s-backup/%[1]s-\" + backupTime + \".sql\"}", instanceName),
+			fmt.Sprintf("uri: ${\"gs://%s-backup/%[1]s-\" + backupTime + \".sql.gz\"}", instanceName),
 		}
 
 		for _, expected := range backupContainsExpecations {