feat: add support to use a different KMS key for backup encryption

modules/fscloud/README.md

@@ -51,6 +51,7 @@ No resources.
 | <a name="input_resource_tags"></a> [resource\_tags](#input\_resource\_tags) | Optional list of tags to be added to the PostgreSQL instance. | `list(string)` | `[]` | no |
 | <a name="input_service_credential_names"></a> [service\_credential\_names](#input\_service\_credential\_names) | Map of name, role for service credentials that you want to create for the database | `map(string)` | `{}` | no |
 | <a name="input_skip_iam_authorization_policy"></a> [skip\_iam\_authorization\_policy](#input\_skip\_iam\_authorization\_policy) | Set to true to skip the creation of an IAM authorization policy that permits all PostgreSQL database instances in the resource group to read the encryption key from the Hyper Protect Crypto Services instance. The HPCS instance is passed in through the var.existing\_kms\_instance\_guid variable. | `bool` | `false` | no |
+| <a name="input_use_default_backup_encryption_key"></a> [use\_default\_backup\_encryption\_key](#input\_use\_default\_backup\_encryption\_key) | Set to true to use default ICD randomly generated keys. | `bool` | `false` | no |
 | <a name="input_users"></a> [users](#input\_users) | A list of users that you want to create on the database. Multiple blocks are allowed. The user password must be in the range of 10-32 characters. Be warned that in most case using IAM service credentials (via the var.service\_credential\_names) is sufficient to control access to the Postgres instance. This blocks creates native postgres database users, more info on that can be found here https://cloud.ibm.com/docs/databases-for-postgresql?topic=databases-for-postgresql-user-management&interface=ui | <pre>list(object({<br/>    name     = string<br/>    password = string # pragma: allowlist secret<br/>    type     = optional(string)<br/>    role     = optional(string)<br/>  }))</pre> | `[]` | no |
 
 ### Outputs

modules/fscloud/main.tf

@@ -1,27 +1,28 @@
 module "postgresql_db" {
-  source                        = "../../"
-  resource_group_id             = var.resource_group_id
-  name                          = var.name
-  region                        = var.region
-  skip_iam_authorization_policy = var.skip_iam_authorization_policy
-  service_endpoints             = "private"
-  pg_version                    = var.pg_version
-  kms_encryption_enabled        = true
-  existing_kms_instance_guid    = var.existing_kms_instance_guid
-  kms_key_crn                   = var.kms_key_crn
-  backup_encryption_key_crn     = var.backup_encryption_key_crn
-  resource_tags                 = var.resource_tags
-  access_tags                   = var.access_tags
-  cbr_rules                     = var.cbr_rules
-  configuration                 = var.configuration
-  member_memory_mb              = var.member_memory_mb
-  member_disk_mb                = var.member_disk_mb
-  member_cpu_count              = var.member_cpu_count
-  member_host_flavor            = var.member_host_flavor
-  members                       = var.members
-  admin_pass                    = var.admin_pass
-  users                         = var.users
-  service_credential_names      = var.service_credential_names
-  auto_scaling                  = var.auto_scaling
-  backup_crn                    = var.backup_crn
+  source                            = "../../"
+  resource_group_id                 = var.resource_group_id
+  name                              = var.name
+  region                            = var.region
+  skip_iam_authorization_policy     = var.skip_iam_authorization_policy
+  service_endpoints                 = "private"
+  pg_version                        = var.pg_version
+  kms_encryption_enabled            = true
+  existing_kms_instance_guid        = var.existing_kms_instance_guid
+  kms_key_crn                       = var.kms_key_crn
+  backup_encryption_key_crn         = var.backup_encryption_key_crn
+  resource_tags                     = var.resource_tags
+  access_tags                       = var.access_tags
+  cbr_rules                         = var.cbr_rules
+  configuration                     = var.configuration
+  member_memory_mb                  = var.member_memory_mb
+  member_disk_mb                    = var.member_disk_mb
+  member_cpu_count                  = var.member_cpu_count
+  member_host_flavor                = var.member_host_flavor
+  members                           = var.members
+  admin_pass                        = var.admin_pass
+  users                             = var.users
+  service_credential_names          = var.service_credential_names
+  auto_scaling                      = var.auto_scaling
+  backup_crn                        = var.backup_crn
+  use_default_backup_encryption_key = var.use_default_backup_encryption_key
 }

modules/fscloud/variables.tf

@@ -207,3 +207,9 @@ variable "backup_crn" {
   description = "The CRN of a backup resource to restore from. The backup is created by a database deployment with the same service ID. The backup is loaded after provisioning and the new deployment starts up that uses that data. A backup CRN is in the format crn:v1:<…>:backup:. If omitted, the database is provisioned empty."
   default     = null
 }
+
+variable "use_default_backup_encryption_key" {
+  type        = bool
+  description = "Set to true to use default ICD randomly generated keys."
+  default     = false
+}

solutions/standard/main.tf

@@ -3,9 +3,8 @@
 #######################################################################################################################
 
 locals {
-  existing_kms_instance_crn_split = var.existing_kms_instance_crn != null ? split(":", var.existing_kms_instance_crn) : null
-  existing_kms_instance_guid      = var.existing_kms_instance_crn != null ? element(local.existing_kms_instance_crn_split, length(local.existing_kms_instance_crn_split) - 3) : null
-  existing_kms_instance_region    = var.existing_kms_instance_crn != null ? element(local.existing_kms_instance_crn_split, length(local.existing_kms_instance_crn_split) - 5) : null
+  existing_kms_instance_guid   = var.existing_kms_instance_crn != null ? module.kms_instance_crn_parser[0].service_instance : null
+  existing_kms_instance_region = var.existing_kms_instance_crn != null ? module.kms_instance_crn_parser[0].region : null
 
   key_name      = var.prefix != null ? "${var.prefix}-${var.key_name}" : var.key_name
   key_ring_name = var.prefix != null ? "${var.prefix}-${var.key_ring_name}" : var.key_ring_name
@@ -14,9 +13,7 @@ locals {
 
   create_cross_account_auth_policy = !var.skip_iam_authorization_policy && var.ibmcloud_kms_api_key != null
 
-  kms_service_name = local.kms_key_crn != null ? (
-    can(regex(".*kms.*", local.kms_key_crn)) ? "kms" : can(regex(".*hs-crypto.*", local.kms_key_crn)) ? "hs-crypto" : null
-  ) : null
+  kms_service_name = var.existing_kms_instance_crn != null ? module.kms_instance_crn_parser[0].service_name : null
 }
 
 #######################################################################################################################
@@ -37,6 +34,14 @@ data "ibm_iam_account_settings" "iam_account_settings" {
   count = local.create_cross_account_auth_policy ? 1 : 0
 }
 
+# If existing KMS intance CRN passed, parse details from it
+module "kms_instance_crn_parser" {
+  count   = var.existing_kms_instance_crn != null ? 1 : 0
+  source  = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser"
+  version = "1.1.0"
+  crn     = var.existing_kms_instance_crn
+}
+
 resource "ibm_iam_authorization_policy" "kms_policy" {
   count                       = local.create_cross_account_auth_policy ? 1 : 0
   provider                    = ibm.kms
@@ -85,30 +90,108 @@ module "kms" {
   ]
 }
 
+#######################################################################################################################
+# KMS backup encryption key for Postgresql
+#######################################################################################################################
+
+locals {
+  existing_backup_kms_instance_guid   = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].service_instance : null
+  existing_backup_kms_instance_region = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].region : null
+
+  backup_key_name         = var.prefix != null ? "${var.prefix}-${var.key_name}" : var.backup_key_name
+  backup_key_ring_name    = var.prefix != null ? "${var.prefix}-${var.key_ring_name}" : var.backup_key_ring_name
+  backup_kms_key_crn      = var.use_default_backup_encryption_key ? null : var.existing_backup_kms_key_crn != null ? var.existing_backup_kms_key_crn : var.existing_backup_kms_instance_crn != null ? module.backup_kms[0].keys[format("%s.%s", local.key_ring_name, local.key_name)].crn : null
+  backup_kms_service_name = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].service_name : null
+}
+
+data "ibm_iam_account_settings" "backup_iam_account_settings" {
+  count = var.ibmcloud_backup_kms_api_key != null ? 1 : 0
+}
+
+
+# If existing KMS intance CRN passed, parse details from it
+module "backup_kms_instance_crn_parser" {
+  count   = var.existing_backup_kms_instance_crn != null ? 1 : 0
+  source  = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser"
+  version = "1.1.0"
+  crn     = var.existing_backup_kms_instance_crn
+}
+
+resource "ibm_iam_authorization_policy" "backup_kms_policy" {
+  count                       = var.use_default_backup_encryption_key ? 0 : var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? !var.skip_backup_kms_iam_authorization_policy ? 1 : 0 : 0
+  provider                    = ibm.backup-kms
+  source_service_account      = var.ibmcloud_backup_kms_api_key != null ? data.ibm_iam_account_settings.backup_iam_account_settings[0].account_id : null
+  source_service_name         = "databases-for-postgresql"
+  source_resource_group_id    = module.resource_group.resource_group_id
+  target_service_name         = local.backup_kms_service_name
+  target_resource_instance_id = local.existing_backup_kms_instance_guid
+  roles                       = ["Reader"]
+  description                 = "Allow all Postgresql instances in the resource group ${module.resource_group.resource_group_id} to read from the ${local.backup_kms_service_name} instance GUID ${local.existing_backup_kms_instance_guid}"
+}
+
+# workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478
+resource "time_sleep" "wait_for_backup_kms_authorization_policy" {
+  depends_on      = [ibm_iam_authorization_policy.backup_kms_policy]
+  create_duration = "30s"
+}
+
+module "backup_kms" {
+  providers = {
+    ibm = ibm.backup-kms
+  }
+  count                       = var.use_default_backup_encryption_key ? 0 : var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? 1 : 0
+  source                      = "terraform-ibm-modules/kms-all-inclusive/ibm"
+  version                     = "4.15.13"
+  create_key_protect_instance = false
+  region                      = local.existing_backup_kms_instance_region
+  existing_kms_instance_crn   = var.existing_backup_kms_instance_crn
+  key_ring_endpoint_type      = var.backup_kms_endpoint_type
+  key_endpoint_type           = var.backup_kms_endpoint_type
+  keys = [
+    {
+      key_ring_name         = local.backup_key_ring_name
+      existing_key_ring     = false
+      force_delete_key_ring = true
+      keys = [
+        {
+          key_name                 = local.backup_key_name
+          standard_key             = false
+          rotation_interval_month  = 3
+          dual_auth_delete_enabled = false
+          force_delete             = true
+        }
+      ]
+    }
+  ]
+}
+
 #######################################################################################################################
 # Postgresql
 #######################################################################################################################
 
 module "postgresql_db" {
-  source                        = "../../modules/fscloud"
-  depends_on                    = [time_sleep.wait_for_authorization_policy]
-  resource_group_id             = module.resource_group.resource_group_id
-  name                          = var.prefix != null ? "${var.prefix}-${var.name}" : var.name
-  region                        = var.region
-  skip_iam_authorization_policy = var.skip_iam_authorization_policy
-  pg_version                    = var.pg_version
-  existing_kms_instance_guid    = local.existing_kms_instance_guid
-  kms_key_crn                   = local.kms_key_crn
-  access_tags                   = var.access_tags
-  resource_tags                 = var.resource_tags
-  admin_pass                    = var.admin_pass
-  users                         = var.users
-  members                       = var.members
-  member_host_flavor            = var.member_host_flavor
-  member_memory_mb              = var.member_memory_mb
-  member_disk_mb                = var.member_disk_mb
-  member_cpu_count              = var.member_cpu_count
-  auto_scaling                  = var.auto_scaling
-  configuration                 = var.configuration
-  service_credential_names      = var.service_credential_names
+  source                            = "../../modules/fscloud"
+  depends_on                        = [time_sleep.wait_for_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy]
+  resource_group_id                 = module.resource_group.resource_group_id
+  name                              = var.prefix != null ? "${var.prefix}-${var.name}" : var.name
+  region                            = var.region
+  skip_iam_authorization_policy     = var.skip_iam_authorization_policy
+  pg_version                        = var.pg_version
+  existing_kms_instance_guid        = local.existing_kms_instance_guid
+  kms_key_crn                       = local.kms_key_crn
+  access_tags                       = var.access_tags
+  resource_tags                     = var.resource_tags
+  admin_pass                        = var.admin_pass
+  users                             = var.users
+  members                           = var.members
+  member_host_flavor                = var.member_host_flavor
+  member_memory_mb                  = var.member_memory_mb
+  member_disk_mb                    = var.member_disk_mb
+  member_cpu_count                  = var.member_cpu_count
+  auto_scaling                      = var.auto_scaling
+  configuration                     = var.configuration
+  service_credential_names          = var.service_credential_names
+  backup_encryption_key_crn         = local.backup_kms_key_crn
+  backup_crn                        = var.backup_crn
+  use_default_backup_encryption_key = var.use_default_backup_encryption_key
 }

solutions/standard/provider.tf

@@ -9,3 +9,9 @@ provider "ibm" {
   ibmcloud_api_key = var.ibmcloud_kms_api_key != null ? var.ibmcloud_kms_api_key : var.ibmcloud_api_key
   region           = local.existing_kms_instance_region
 }
+
+provider "ibm" {
+  alias            = "backup-kms"
+  ibmcloud_api_key = var.ibmcloud_backup_kms_api_key != null ? var.ibmcloud_backup_kms_api_key : var.ibmcloud_api_key
+  region           = local.existing_kms_instance_region
+}

solutions/standard/variables.tf

@@ -15,6 +15,13 @@ variable "ibmcloud_kms_api_key" {
   default     = null
 }
 
+variable "ibmcloud_backup_kms_api_key" {
+  type        = string
+  description = "The IBM Cloud API key that can create a root key and key ring in the key management service (KMS) instance. If not specified, the 'ibmcloud_api_key' variable is used. Specify this key if the instance in `existing_kms_instance_crn` is in an account that's different from the PostgreSQL instance. Leave this input empty if the same account owns both instances."
+  sensitive   = true
+  default     = null
+}
+
 variable "use_existing_resource_group" {
   type        = bool
   description = "Whether to use an existing resource group."
@@ -50,6 +57,12 @@ variable "pg_version" {
   default     = null
 }
 
+variable "backup_crn" {
+  type        = string
+  description = "The CRN of a backup resource to restore from. The backup is created by a database deployment with the same service ID. The backup is loaded after provisioning and the new deployment starts up that uses that data. A backup CRN is in the format crn:v1:<…>:backup:. If omitted, the database is provisioned empty."
+  default     = null
+}
+
 ##############################################################################
 # ICD hosting model properties
 ##############################################################################
@@ -237,3 +250,52 @@ variable "skip_iam_authorization_policy" {
   description = "Whether to create an IAM authorization policy that permits all PostgreSQL instances in the resource group to read the encryption key from the Hyper Protect Crypto Services instance specified in the `existing_kms_instance_crn` variable."
   default     = false
 }
+
+##############################################################
+# Backup Encryption
+##############################################################
+variable "backup_key_name" {
+  type        = string
+  default     = "postgresql-backup-key"
+  description = "The name for the key created for the PostgreSQL key. Applies only if not specifying an existing key. If a prefix input variable is specified, the prefix is added to the name in the `<prefix>-<name>` format."
+}
+
+variable "backup_key_ring_name" {
+  type        = string
+  default     = "postgresql-backup-key-ring"
+  description = "The name for the key ring created for the PostgreSQL key. Applies only if not specifying an existing key. If a prefix input variable is specified, the prefix is added to the name in the `<prefix>-<name>` format."
+}
+
+variable "backup_kms_endpoint_type" {
+  type        = string
+  description = "The type of endpoint to use for communicating with the Key Protect or Hyper Protect Crypto Services instance. Possible values: `public`, `private`."
+  default     = "private"
+  validation {
+    condition     = can(regex("public|private", var.backup_kms_endpoint_type))
+    error_message = "The backup_kms_endpoint_type value must be 'public' or 'private'."
+  }
+}
+
+variable "existing_backup_kms_key_crn" {
+  type        = string
+  description = "The CRN of a Hyper Protect Crypto Services or Key Protect root key to use for disk encryption. If not specified, a root key is created in the KMS instance."
+  default     = null
+}
+
+variable "existing_backup_kms_instance_crn" {
+  description = "The CRN of a Hyper Protect Crypto Services or Key Protect instance in the same account as the PostgreSQL instance. This value is used to create an authorization policy if `skip_iam_authorization_policy` is false. If not specified, a root key is created."
+  type        = string
+  default     = null
+}
+
+variable "skip_backup_kms_iam_authorization_policy" {
+  type        = bool
+  description = "Whether to create an IAM authorization policy that permits all PostgreSQL instances in the resource group to read the encryption key from the Hyper Protect Crypto Services instance specified in the `existing_kms_instance_crn` variable."
+  default     = false
+}
+
+variable "use_default_backup_encryption_key" {
+  type        = bool
+  description = "Set to true to use default ICD randomly generated keys."
+  default     = false
+}

tests/pr_test.go

@@ -109,10 +109,12 @@ func TestRunStandardSolution(t *testing.T) {
 	})
 
 	options.TerraformVars = map[string]interface{}{
-		"pg_version":                "16", // Always lock this test into the latest supported PostgreSQL version
-		"existing_kms_instance_crn": permanentResources["hpcs_south_crn"],
-		"kms_endpoint_type":         "public",
-		"resource_group_name":       options.Prefix,
+		"pg_version":                               "16", // Always lock this test into the latest supported PostgreSQL version
+		"existing_kms_instance_crn":                permanentResources["hpcs_south_crn"],
+		"kms_endpoint_type":                        "public",
+		"existing_backup_kms_key_crn":              permanentResources["hpcs_south_root_key_crn"],
+		"skip_backup_kms_iam_authorization_policy": true,
+		"resource_group_name":                      options.Prefix,
 	}
 
 	output, err := options.RunTestConsistency()