feat: add support to use a different KMS key for backup encryption

README.md

@@ -75,12 +75,14 @@ You need the following permissions to run this module.
 | Name | Type |
 |------|------|
 | [ibm_database.elasticsearch](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/database) | resource |
+| [ibm_iam_authorization_policy.backup_kms_policy](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/iam_authorization_policy) | resource |
 | [ibm_iam_authorization_policy.policy](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/iam_authorization_policy) | resource |
 | [ibm_resource_key.service_credentials](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/resource_key) | resource |
 | [ibm_resource_tag.elasticsearch_tag](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/resource_tag) | resource |
 | [null_resource.put_vectordb_model](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [null_resource.start_vectordb_model](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [time_sleep.wait_for_authorization_policy](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource |
+| [time_sleep.wait_for_backup_kms_authorization_policy](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource |
 | [ibm_database_connection.database_connection](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/data-sources/database_connection) | data source |
 
 ### Inputs

cra-config.yaml

@@ -6,7 +6,7 @@ CRA_TARGETS:
     PROFILE_ID: "bfacb71d-4b84-41ac-9825-e8a3a3eb7405" # SCC profile ID (currently set to IBM Cloud Framework for Financial Services 1.6.0 profile).
     CRA_ENVIRONMENT_VARIABLES:
       TF_VAR_existing_kms_instance_crn: "crn:v1:bluemix:public:hs-crypto:us-south:a/abac0df06b644a9cabc6e44f55b3880e:e6dce284-e80f-46e1-a3c1-830f7adff7a9::"
-      TF_VAR_kms_key_crn: "crn:v1:bluemix:public:hs-crypto:us-south:a/abac0df06b644a9cabc6e44f55b3880e:e6dce284-e80f-46e1-a3c1-830f7adff7a9:key:76170fae-4e0c-48c3-8ebe-326059ebb533"
+      TF_VAR_existing_kms_key_crn: "crn:v1:bluemix:public:hs-crypto:us-south:a/abac0df06b644a9cabc6e44f55b3880e:e6dce284-e80f-46e1-a3c1-830f7adff7a9:key:76170fae-4e0c-48c3-8ebe-326059ebb533"
       TF_VAR_provider_visibility: "public"
       TF_VAR_resource_group_name: "test-es-cra"
       TF_VAR_use_existing_resource_group: false

ibm_catalog.json

@@ -42,6 +42,10 @@
           {
             "title": "Attaches access tags",
             "description": "Attaches access tags to the Elasticsearch instance."
+          },
+          {
+            "title": "Supports backup restoration",
+            "description": "Provides database restoration using a backup created by a deployment with the same service ID."
           }
         ],
         "flavors": [
@@ -278,6 +282,15 @@
                 {
                   "key": "auto_scaling"
                 },
+                {
+                  "key": "backup_crn"
+                },
+                {
+                  "key": "existing_backup_kms_key_crn"
+                },
+                {
+                  "key": "existing_backup_kms_instance_crn"
+                },
                 {
                   "key": "enable_elser_model"
                 },

main.tf

@@ -17,7 +17,11 @@ locals {
 
   # For more info, see https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok and https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups"
 
-  backup_encryption_key_crn = var.use_default_backup_encryption_key == true ? null : (var.backup_encryption_key_crn != null ? var.backup_encryption_key_crn : var.kms_key_crn)
+  backup_encryption_key_crn        = var.use_default_backup_encryption_key == true ? null : (var.backup_encryption_key_crn != null ? var.backup_encryption_key_crn : var.kms_key_crn)
+  parsed_backup_encryption_key_crn = local.backup_encryption_key_crn != null ? split(":", local.backup_encryption_key_crn) : []
+  backup_kms_key_id                = length(local.parsed_backup_encryption_key_crn) > 0 ? local.parsed_backup_encryption_key_crn[9] : null
+
+  create_backup_kms_policy = local.create_kp_auth_policy == 1 && local.backup_encryption_key_crn != null && var.backup_encryption_key_crn != null
 
   # Determine if auto scaling is enabled
   auto_scaling_enabled = var.auto_scaling == null ? [] : [1]
@@ -81,9 +85,52 @@ resource "time_sleep" "wait_for_authorization_policy" {
   create_duration = "30s"
 }
 
+resource "ibm_iam_authorization_policy" "backup_kms_policy" {
+  count                    = local.create_backup_kms_policy ? 1 : 0
+  source_service_name      = "databases-for-elasticsearch"
+  source_resource_group_id = var.resource_group_id
+  roles                    = ["Reader"]
+  description              = "Allow all Elastic Search instances in the Resource Group ${var.resource_group_id} to read the ${local.kms_service} key ${local.backup_kms_key_id} from the instance GUID ${var.existing_kms_instance_guid}"
+  resource_attributes {
+    name     = "serviceName"
+    operator = "stringEquals"
+    value    = local.kms_service
+  }
+  resource_attributes {
+    name     = "accountId"
+    operator = "stringEquals"
+    value    = local.kms_account_id
+  }
+  resource_attributes {
+    name     = "serviceInstance"
+    operator = "stringEquals"
+    value    = var.existing_kms_instance_guid
+  }
+  resource_attributes {
+    name     = "resourceType"
+    operator = "stringEquals"
+    value    = "key"
+  }
+  resource_attributes {
+    name     = "resource"
+    operator = "stringEquals"
+    value    = local.backup_kms_key_id
+  }
+  # Scope of policy now includes the key, so ensure to create new policy before
+  # destroying old one to prevent any disruption to every day services.
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+# workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478
+resource "time_sleep" "wait_for_backup_kms_authorization_policy" {
+  depends_on      = [ibm_iam_authorization_policy.backup_kms_policy]
+  create_duration = "30s"
+}
 
 resource "ibm_database" "elasticsearch" {
-  depends_on                = [time_sleep.wait_for_authorization_policy]
+  depends_on                = [time_sleep.wait_for_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy]
   name                      = var.name
   plan                      = var.plan
   location                  = var.region

solutions/standard/main.tf

@@ -122,14 +122,85 @@ module "kms" {
   ]
 }
 
+
+#######################################################################################################################
+# KMS backup encryption key for Elasticsearch
+#######################################################################################################################
+
+locals {
+  existing_backup_kms_instance_guid   = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].service_instance : null
+  existing_backup_kms_instance_region = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].region : null
+
+  backup_key_name         = var.prefix != null ? "${var.prefix}-backup-encryption-${var.elasticsearch_key_name}" : "backup-encryption-${var.elasticsearch_key_name}"
+  backup_key_ring_name    = var.prefix != null ? "${var.prefix}-backup-encryption-${var.elasticsearch_key_ring_name}" : "backup-encryption-${var.elasticsearch_key_ring_name}"
+  backup_kms_key_crn      = var.existing_backup_kms_key_crn != null ? var.existing_backup_kms_key_crn : var.existing_backup_kms_instance_crn != null ? module.backup_kms[0].keys[format("%s.%s", local.backup_key_ring_name, local.backup_key_name)].crn : null
+  backup_kms_service_name = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].service_name : null
+}
+
+# If existing KMS intance CRN passed, parse details from it
+module "backup_kms_instance_crn_parser" {
+  count   = var.existing_backup_kms_instance_crn != null ? 1 : 0
+  source  = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser"
+  version = "1.1.0"
+  crn     = var.existing_backup_kms_instance_crn
+}
+
+resource "ibm_iam_authorization_policy" "backup_kms_policy" {
+  count                       = local.existing_backup_kms_instance_guid == local.existing_kms_instance_guid ? 0 : var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? !var.skip_iam_authorization_policy ? 1 : 0 : 0
+  provider                    = ibm.kms
+  source_service_account      = local.create_cross_account_auth_policy ? data.ibm_iam_account_settings.iam_account_settings[0].account_id : null
+  source_service_name         = "databases-for-elasticsearch"
+  source_resource_group_id    = module.resource_group.resource_group_id
+  target_service_name         = local.backup_kms_service_name
+  target_resource_instance_id = local.existing_backup_kms_instance_guid
+  roles                       = ["Reader"]
+  description                 = "Allow all Elasticsearch instances in the resource group ${module.resource_group.resource_group_id} to read from the ${local.backup_kms_service_name} instance GUID ${local.existing_backup_kms_instance_guid}"
+}
+
+# workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478
+resource "time_sleep" "wait_for_backup_kms_authorization_policy" {
+  depends_on      = [ibm_iam_authorization_policy.backup_kms_policy]
+  create_duration = "30s"
+}
+
+module "backup_kms" {
+  providers = {
+    ibm = ibm.kms
+  }
+  count                       = var.use_ibm_owned_encryption_key ? 0 : var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? 1 : 0
+  source                      = "terraform-ibm-modules/kms-all-inclusive/ibm"
+  version                     = "4.16.8"
+  create_key_protect_instance = false
+  region                      = local.existing_backup_kms_instance_region
+  existing_kms_instance_crn   = var.existing_backup_kms_instance_crn
+  key_ring_endpoint_type      = var.kms_endpoint_type
+  key_endpoint_type           = var.kms_endpoint_type
+  keys = [
+    {
+      key_ring_name         = local.backup_key_ring_name
+      existing_key_ring     = false
+      force_delete_key_ring = true
+      keys = [
+        {
+          key_name                 = local.backup_key_name
+          standard_key             = false
+          rotation_interval_month  = 3
+          dual_auth_delete_enabled = false
+          force_delete             = true
+        }
+      ]
+    }
+  ]
+}
+
 #######################################################################################################################
 # Elasticsearch
 #######################################################################################################################
 
 module "elasticsearch" {
   count                         = local.use_existing_db_instance ? 0 : 1
   source                        = "../../modules/fscloud"
-  depends_on                    = [time_sleep.wait_for_authorization_policy]
+  depends_on                    = [time_sleep.wait_for_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy]
   resource_group_id             = module.resource_group.resource_group_id
   name                          = var.prefix != null ? "${var.prefix}-${var.name}" : var.name
   region                        = var.region
@@ -139,6 +210,8 @@ module "elasticsearch" {
   existing_kms_instance_guid    = local.existing_kms_instance_guid
   use_ibm_owned_encryption_key  = var.use_ibm_owned_encryption_key
   kms_key_crn                   = local.kms_key_crn
+  backup_encryption_key_crn     = local.backup_kms_key_crn
+  backup_crn                    = var.backup_crn
   access_tags                   = var.access_tags
   tags                          = var.tags
   admin_pass                    = local.admin_pass

solutions/standard/variables.tf

@@ -59,6 +59,12 @@ variable "elasticsearch_version" {
   default     = null
 }
 
+variable "backup_crn" {
+  type        = string
+  description = "The CRN of a backup resource to restore from. The backup is created by a database deployment with the same service ID. The backup is loaded after provisioning and the new deployment starts up that uses that data. A backup CRN is in the format crn:v1:<…>:backup:. If omitted, the database is provisioned empty."
+  default     = null
+}
+
 variable "region" {
   type        = string
   description = "The region where you want to deploy your instance."
@@ -208,7 +214,7 @@ variable "auto_scaling" {
 
 variable "existing_kms_key_crn" {
   type        = string
-  description = "The CRN of a Hyper Protect Crypto Services or Key Protect root key to use for disk encryption. If not specified, a root key is created in the KMS instance specified in the `existing_kms_instance_crn` input."
+  description = "The CRN of a Hyper Protect Crypto Services or Key Protect root key to use for disk encryption. If not specified, a root key is created in the KMS instance specified in the `existing_kms_instance_crn` input. Backup encryption is only supported is some regions ([learn more](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok)), so if you need to use a key from a different region for backup encryption, use the `existing_backup_kms_key_crn` input."
   default     = null
 }
 
@@ -230,7 +236,7 @@ variable "kms_endpoint_type" {
 
 variable "existing_kms_instance_crn" {
   type        = string
-  description = "The CRN of a Hyper Protect Crypto Services or Key Protect instance. Required to create a new root key if no value is passed with the `existing_kms_key_crn` input. Also required to create an authorization policy if `skip_iam_authorization_policy` is false."
+  description = "The CRN of a Hyper Protect Crypto Services or Key Protect instance. Required to create a new root key if no value is passed with the `existing_kms_key_crn` input. Also required to create an authorization policy if `skip_iam_authorization_policy` is false. Backup encryption is only supported is some regions ([learn more](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok)), so if you need to use a different instance for backup encryption from a supported region, use the `existing_backup_kms_instance_crn` input."
   default     = null
 }
 
@@ -294,7 +300,7 @@ variable "service_credential_secrets" {
     }))
   }))
   default     = []
-  description = "Service credential secrets configuration for Databases for Elasticsearch. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-elasticsearch/tree/main/solutions/instance/DA-types.md#service-credential-secrets)."
+  description = "Service credential secrets configuration for Databases for Elasticsearch. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/solutions/standard/DA-types.md#service-credential-secrets)."
 
   validation {
     condition = alltrue([
@@ -354,3 +360,18 @@ variable "elasticsearch_full_version" {
   type        = string
   default     = null
 }
+
+##############################################################
+# Backup Encryption
+##############################################################
+variable "existing_backup_kms_key_crn" {
+  type        = string
+  description = "The CRN of an Hyper Protect Crypto Services or Key Protect encryption key that you want to use to encrypt database backups. If no value is passed, the value of `existing_kms_key_crn` is used. If no value is passed for that, a new key will be created in the provided KMS instance and used for both disk encryption, and backup encryption."
+  default     = null
+}
+
+variable "existing_backup_kms_instance_crn" {
+  description = "The CRN of an Hyper Protect Crypto Services or Key Protect instance that you want to use to encrypt database backups. If no value is passed, the value of the `existing_kms_instance_crn` input will be used, however backup encryption is only supported in certain regions so you need to ensure the KMS for backup is coming from one of the supported regions. [Learn more](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok)"
+  type        = string
+  default     = null
+}

tests/pr_test.go

@@ -121,6 +121,7 @@ func TestRunStandardSolutionSchematics(t *testing.T) {
 		{Name: "ibmcloud_api_key", Value: options.RequiredEnvironmentVars["TF_VAR_ibmcloud_api_key"], DataType: "string", Secure: true},
 		{Name: "access_tags", Value: permanentResources["accessTags"], DataType: "list(string)"},
 		{Name: "existing_kms_instance_crn", Value: permanentResources["hpcs_south_crn"], DataType: "string"},
+		{Name: "existing_backup_kms_key_crn", Value: permanentResources["hpcs_south_root_key_crn"], DataType: "string"},
 		{Name: "kms_endpoint_type", Value: "public", DataType: "string"},
 		{Name: "resource_group_name", Value: options.Prefix, DataType: "string"},
 		{Name: "plan", Value: "platinum", DataType: "string"},