From b2ff6cde99d4db5b41017832ff2cb1220cb01772 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Thu, 14 Nov 2024 16:17:33 +0000 Subject: [PATCH 01/22] feat: add support to use a different KMS key for backup encryption --- ibm_catalog.json | 7 +++ solutions/standard/main.tf | 76 ++++++++++++++++++++++++++++++++- solutions/standard/variables.tf | 25 ++++++++++- tests/pr_test.go | 1 + 4 files changed, 105 insertions(+), 4 deletions(-) diff --git a/ibm_catalog.json b/ibm_catalog.json index 2f116cb4..c369edbf 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -42,6 +42,10 @@ { "title": "Attaches access tags", "description": "Attaches access tags to the Elasticsearch instance." + }, + { + "title": "Supports backup restoration", + "description": "Provides database restoration using a backup created by a deployment with the same service ID." } ], "flavors": [ @@ -274,6 +278,9 @@ { "key": "auto_scaling" }, + { + "key": "backup_crn" + }, { "key": "enable_elser_model" }, diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index d09e2b3f..48510204 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -122,6 +122,76 @@ module "kms" { ] } +####################################################################################################################### +# KMS backup encryption key for Postgresql +####################################################################################################################### + +locals { + existing_backup_kms_instance_guid = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].service_instance : null + existing_backup_kms_instance_region = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].region : null + + backup_key_name = var.prefix != null ? "${var.prefix}-backup-encryption-${var.elasticsearch_key_name}" : "backup-encryption-${var.elasticsearch_key_name}" + backup_key_ring_name = var.prefix != null ? "${var.prefix}-backup-encryption-${var.elasticsearch_key_ring_name}" : "backup-encryption-${var.elasticsearch_key_ring_name}" + backup_kms_key_crn = var.existing_backup_kms_key_crn != null ? var.existing_backup_kms_key_crn : var.existing_backup_kms_instance_crn != null ? module.backup_kms[0].keys[format("%s.%s", local.backup_key_ring_name, local.backup_key_name)].crn : null + backup_kms_service_name = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].service_name : null +} + +# If existing KMS intance CRN passed, parse details from it +module "backup_kms_instance_crn_parser" { + count = var.existing_backup_kms_instance_crn != null ? 1 : 0 + source = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser" + version = "1.1.0" + crn = var.existing_backup_kms_instance_crn +} + +resource "ibm_iam_authorization_policy" "backup_kms_policy" { + count = local.existing_backup_kms_instance_guid == local.existing_kms_instance_guid ? 0 : var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? !var.skip_iam_authorization_policy ? 1 : 0 : 0 + provider = ibm.kms + source_service_account = local.create_cross_account_auth_policy ? data.ibm_iam_account_settings.iam_account_settings[0].account_id : null + source_service_name = "databases-for-postgresql" + source_resource_group_id = module.resource_group.resource_group_id + target_service_name = local.backup_kms_service_name + target_resource_instance_id = local.existing_backup_kms_instance_guid + roles = ["Reader"] + description = "Allow all Postgresql instances in the resource group ${module.resource_group.resource_group_id} to read from the ${local.backup_kms_service_name} instance GUID ${local.existing_backup_kms_instance_guid}" +} + +# workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 +resource "time_sleep" "wait_for_backup_kms_authorization_policy" { + depends_on = [ibm_iam_authorization_policy.backup_kms_policy] + create_duration = "30s" +} + +module "backup_kms" { + providers = { + ibm = ibm.kms + } + count = var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? 1 : 0 + source = "terraform-ibm-modules/kms-all-inclusive/ibm" + version = "4.15.13" + create_key_protect_instance = false + region = local.existing_backup_kms_instance_region + existing_kms_instance_crn = var.existing_backup_kms_instance_crn + key_ring_endpoint_type = var.kms_endpoint_type + key_endpoint_type = var.kms_endpoint_type + keys = [ + { + key_ring_name = local.backup_key_ring_name + existing_key_ring = false + force_delete_key_ring = true + keys = [ + { + key_name = local.backup_key_name + standard_key = false + rotation_interval_month = 3 + dual_auth_delete_enabled = false + force_delete = true + } + ] + } + ] +} + ####################################################################################################################### # Elasticsearch ####################################################################################################################### @@ -129,16 +199,18 @@ module "kms" { module "elasticsearch" { count = local.use_existing_db_instance ? 0 : 1 source = "../../modules/fscloud" - depends_on = [time_sleep.wait_for_authorization_policy] + depends_on = [time_sleep.wait_for_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy] resource_group_id = module.resource_group.resource_group_id name = var.prefix != null ? "${var.prefix}-${var.name}" : var.name region = var.region plan = var.plan - skip_iam_authorization_policy = var.skip_iam_authorization_policy || local.create_cross_account_auth_policy + skip_iam_authorization_policy = var.skip_iam_authorization_policy || local.create_cross_account_auth_policy ? true : var.skip_iam_authorization_policy elasticsearch_version = var.elasticsearch_version existing_kms_instance_guid = local.existing_kms_instance_guid use_ibm_owned_encryption_key = var.use_ibm_owned_encryption_key kms_key_crn = local.kms_key_crn + backup_encryption_key_crn = local.backup_kms_key_crn + backup_crn = var.backup_crn access_tags = var.access_tags tags = var.tags admin_pass = local.admin_pass diff --git a/solutions/standard/variables.tf b/solutions/standard/variables.tf index b378fcbb..1b4dd59d 100644 --- a/solutions/standard/variables.tf +++ b/solutions/standard/variables.tf @@ -59,6 +59,12 @@ variable "elasticsearch_version" { default = null } +variable "backup_crn" { + type = string + description = "The CRN of a backup resource to restore from. The backup is created by a database deployment with the same service ID. The backup is loaded after provisioning and the new deployment starts up that uses that data. A backup CRN is in the format crn:v1:<…>:backup:. If omitted, the database is provisioned empty." + default = null +} + variable "region" { type = string description = "The region where you want to deploy your instance." @@ -208,7 +214,7 @@ variable "auto_scaling" { variable "existing_kms_key_crn" { type = string - description = "The CRN of a Hyper Protect Crypto Services or Key Protect root key to use for disk encryption. If not specified, a root key is created in the KMS instance." + description = "The CRN of an Hyper Protect Crypto Services or Key Protect encryption key that you want to use to use for both disk and backup encryption. If no value is passed, a new key ring and key will be created in the instance provided in the `existing_kms_instance_crn` input. Backup encryption is only supported is some regions ([learn more](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok)), so if you need to use a key from a different region for backup encryption, use the `existing_backup_kms_key_crn` input." default = null } @@ -230,7 +236,7 @@ variable "kms_endpoint_type" { variable "existing_kms_instance_crn" { type = string - description = "The CRN of a Hyper Protect Crypto Services or Key Protect instance in the same account as the Databases for Elasticsearch instance. This value is used to create an authorization policy if `skip_iam_authorization_policy` is false. If not specified, a root key is created." + description = "The CRN of an Hyper Protect Crypto Services or Key Protect instance that you want to use for both disk and backup encryption. Backup encryption is only supported is some regions ([learn more](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok)), so if you need to use a different instance for backup encryption from a supported region, use the `existing_backup_kms_instance_crn` input." default = null } @@ -354,3 +360,18 @@ variable "elasticsearch_full_version" { type = string default = null } + +############################################################## +# Backup Encryption +############################################################## +variable "existing_backup_kms_key_crn" { + type = string + description = "The CRN of an Hyper Protect Crypto Services or Key Protect encryption key that you want to use to encrypt database backups. If no value is passed, the value of `existing_kms_key_crn` is used. If no value is passed for that, a new key will be created in the provided KMS instance and used for both disk encryption, and backup encryption." + default = null +} + +variable "existing_backup_kms_instance_crn" { + description = "The CRN of an Hyper Protect Crypto Services or Key Protect instance that you want to use to encrypt database backups. If no value is passed, the value of the `existing_kms_instance_crn` input will be used, however backup encryption is only supported in certain regions so you need to ensure the KMS for backup is coming from one of the supported regions. [Learn more](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok)" + type = string + default = null +} diff --git a/tests/pr_test.go b/tests/pr_test.go index 2808e82a..c615627b 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -194,6 +194,7 @@ func TestRunStandardSolutionSchematics(t *testing.T) { {Name: "enable_kibana_dashboard", Value: true, DataType: "bool"}, {Name: "provider_visibility", Value: "private", DataType: "string"}, {Name: "prefix", Value: options.Prefix, DataType: "string"}, + {Name: "existing_backup_kms_key_crn", Value: permanentResources["hpcs_south_root_key_crn"], DataType: "string"}, } err := options.RunSchematicTest() assert.Nil(t, err, "This should not have errored") From c02ed293f9e25be647ec8f96efb31d5b2fe943fb Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Thu, 14 Nov 2024 16:49:15 +0000 Subject: [PATCH 02/22] feat: add support to use a different KMS key for backup encryption --- solutions/standard/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index 48510204..a52c3928 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -123,7 +123,7 @@ module "kms" { } ####################################################################################################################### -# KMS backup encryption key for Postgresql +# KMS backup encryption key for ElasticSearch ####################################################################################################################### locals { @@ -148,12 +148,12 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy" { count = local.existing_backup_kms_instance_guid == local.existing_kms_instance_guid ? 0 : var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? !var.skip_iam_authorization_policy ? 1 : 0 : 0 provider = ibm.kms source_service_account = local.create_cross_account_auth_policy ? data.ibm_iam_account_settings.iam_account_settings[0].account_id : null - source_service_name = "databases-for-postgresql" + source_service_name = "databases-for-elasticsearch" source_resource_group_id = module.resource_group.resource_group_id target_service_name = local.backup_kms_service_name target_resource_instance_id = local.existing_backup_kms_instance_guid roles = ["Reader"] - description = "Allow all Postgresql instances in the resource group ${module.resource_group.resource_group_id} to read from the ${local.backup_kms_service_name} instance GUID ${local.existing_backup_kms_instance_guid}" + description = "Allow all ElasticSearch instances in the resource group ${module.resource_group.resource_group_id} to read from the ${local.backup_kms_service_name} instance GUID ${local.existing_backup_kms_instance_guid}" } # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 From 85306bdfd77672f97e5f4cf0dbc385e59b95a984 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Thu, 14 Nov 2024 16:50:13 +0000 Subject: [PATCH 03/22] feat: add support to use a different KMS key for backup encryption --- solutions/standard/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index a52c3928..783538fa 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -123,7 +123,7 @@ module "kms" { } ####################################################################################################################### -# KMS backup encryption key for ElasticSearch +# KMS backup encryption key for Elasticsearch ####################################################################################################################### locals { @@ -153,7 +153,7 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy" { target_service_name = local.backup_kms_service_name target_resource_instance_id = local.existing_backup_kms_instance_guid roles = ["Reader"] - description = "Allow all ElasticSearch instances in the resource group ${module.resource_group.resource_group_id} to read from the ${local.backup_kms_service_name} instance GUID ${local.existing_backup_kms_instance_guid}" + description = "Allow all Elasticsearch instances in the resource group ${module.resource_group.resource_group_id} to read from the ${local.backup_kms_service_name} instance GUID ${local.existing_backup_kms_instance_guid}" } # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 From e9c00fa55012e58cb4e3260946ea77a2338af362 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Thu, 14 Nov 2024 16:52:20 +0000 Subject: [PATCH 04/22] feat: add support to use a different KMS key for backup encryption --- ibm_catalog.json | 6 ++++++ solutions/standard/main.tf | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ibm_catalog.json b/ibm_catalog.json index c369edbf..70127390 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -281,6 +281,12 @@ { "key": "backup_crn" }, + { + "key": "existing_backup_kms_key_crn" + }, + { + "key": "existing_backup_kms_instance_crn" + }, { "key": "enable_elser_model" }, diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index 783538fa..5c202b65 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -166,7 +166,7 @@ module "backup_kms" { providers = { ibm = ibm.kms } - count = var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? 1 : 0 + count = var.use_ibm_owned_encryption_key ? 0 : var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? 1 : 0 source = "terraform-ibm-modules/kms-all-inclusive/ibm" version = "4.15.13" create_key_protect_instance = false From effe3f99203c3ccc90ea02fc61e0eb6b81fada04 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Fri, 15 Nov 2024 16:30:09 +0000 Subject: [PATCH 05/22] fix: hardcode region --- tests/pr_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pr_test.go b/tests/pr_test.go index c615627b..4be16ee0 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -155,7 +155,7 @@ func TestRunStandardSolutionSchematics(t *testing.T) { Testing: t, TarIncludePatterns: tarIncludePatterns, TemplateFolder: standardSolutionTerraformDir, - BestRegionYAMLPath: regionSelectionPath, + Region: "us-south", Prefix: "els-sr-da", ResourceGroup: resourceGroup, DeleteWorkspaceOnFail: false, From c99610e73de1c763d8bb09eb2750f6cb1811321d Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Fri, 15 Nov 2024 17:21:42 +0000 Subject: [PATCH 06/22] fix: hardcode region --- solutions/standard/main.tf | 9 +++++---- tests/pr_test.go | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index 5c202b65..496f9a29 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -122,8 +122,9 @@ module "kms" { ] } + ####################################################################################################################### -# KMS backup encryption key for Elasticsearch +# KMS backup encryption key for Postgresql ####################################################################################################################### locals { @@ -148,12 +149,12 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy" { count = local.existing_backup_kms_instance_guid == local.existing_kms_instance_guid ? 0 : var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? !var.skip_iam_authorization_policy ? 1 : 0 : 0 provider = ibm.kms source_service_account = local.create_cross_account_auth_policy ? data.ibm_iam_account_settings.iam_account_settings[0].account_id : null - source_service_name = "databases-for-elasticsearch" + source_service_name = "databases-for-postgresql" source_resource_group_id = module.resource_group.resource_group_id target_service_name = local.backup_kms_service_name target_resource_instance_id = local.existing_backup_kms_instance_guid roles = ["Reader"] - description = "Allow all Elasticsearch instances in the resource group ${module.resource_group.resource_group_id} to read from the ${local.backup_kms_service_name} instance GUID ${local.existing_backup_kms_instance_guid}" + description = "Allow all Postgresql instances in the resource group ${module.resource_group.resource_group_id} to read from the ${local.backup_kms_service_name} instance GUID ${local.existing_backup_kms_instance_guid}" } # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 @@ -168,7 +169,7 @@ module "backup_kms" { } count = var.use_ibm_owned_encryption_key ? 0 : var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? 1 : 0 source = "terraform-ibm-modules/kms-all-inclusive/ibm" - version = "4.15.13" + version = "4.16.8" create_key_protect_instance = false region = local.existing_backup_kms_instance_region existing_kms_instance_crn = var.existing_backup_kms_instance_crn diff --git a/tests/pr_test.go b/tests/pr_test.go index 4be16ee0..c2fbb759 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -182,6 +182,7 @@ func TestRunStandardSolutionSchematics(t *testing.T) { {Name: "ibmcloud_api_key", Value: options.RequiredEnvironmentVars["TF_VAR_ibmcloud_api_key"], DataType: "string", Secure: true}, {Name: "access_tags", Value: permanentResources["accessTags"], DataType: "list(string)"}, {Name: "existing_kms_instance_crn", Value: permanentResources["hpcs_south_crn"], DataType: "string"}, + {Name: "existing_backup_kms_key_crn", Value: permanentResources["hpcs_south_root_key_crn"], DataType: "string"}, {Name: "kms_endpoint_type", Value: "public", DataType: "string"}, {Name: "resource_group_name", Value: options.Prefix, DataType: "string"}, {Name: "plan", Value: "platinum", DataType: "string"}, @@ -194,7 +195,6 @@ func TestRunStandardSolutionSchematics(t *testing.T) { {Name: "enable_kibana_dashboard", Value: true, DataType: "bool"}, {Name: "provider_visibility", Value: "private", DataType: "string"}, {Name: "prefix", Value: options.Prefix, DataType: "string"}, - {Name: "existing_backup_kms_key_crn", Value: permanentResources["hpcs_south_root_key_crn"], DataType: "string"}, } err := options.RunSchematicTest() assert.Nil(t, err, "This should not have errored") From 27c3fe81d643246db38cd552404caa0d0c916fef Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Fri, 15 Nov 2024 17:22:59 +0000 Subject: [PATCH 07/22] fix: hardcode region --- solutions/standard/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index 496f9a29..89cf8ffd 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -124,7 +124,7 @@ module "kms" { ####################################################################################################################### -# KMS backup encryption key for Postgresql +# KMS backup encryption key for Elasticsearch ####################################################################################################################### locals { @@ -149,12 +149,12 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy" { count = local.existing_backup_kms_instance_guid == local.existing_kms_instance_guid ? 0 : var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? !var.skip_iam_authorization_policy ? 1 : 0 : 0 provider = ibm.kms source_service_account = local.create_cross_account_auth_policy ? data.ibm_iam_account_settings.iam_account_settings[0].account_id : null - source_service_name = "databases-for-postgresql" + source_service_name = "databases-for-elasticsearch" source_resource_group_id = module.resource_group.resource_group_id target_service_name = local.backup_kms_service_name target_resource_instance_id = local.existing_backup_kms_instance_guid roles = ["Reader"] - description = "Allow all Postgresql instances in the resource group ${module.resource_group.resource_group_id} to read from the ${local.backup_kms_service_name} instance GUID ${local.existing_backup_kms_instance_guid}" + description = "Allow all Elasticsearch instances in the resource group ${module.resource_group.resource_group_id} to read from the ${local.backup_kms_service_name} instance GUID ${local.existing_backup_kms_instance_guid}" } # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 From a3239943140b1b7a14178cc6f7ade83bd13c7c58 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 18 Nov 2024 02:00:30 +0000 Subject: [PATCH 08/22] fix: add policy --- solutions/standard/main.tf | 59 +++++++++++++++++++++++++++++++-- solutions/standard/variables.tf | 2 +- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index d25cdba9..01181024 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -135,6 +135,9 @@ locals { backup_key_ring_name = var.prefix != null ? "${var.prefix}-backup-encryption-${var.elasticsearch_key_ring_name}" : "backup-encryption-${var.elasticsearch_key_ring_name}" backup_kms_key_crn = var.existing_backup_kms_key_crn != null ? var.existing_backup_kms_key_crn : var.existing_backup_kms_instance_crn != null ? module.backup_kms[0].keys[format("%s.%s", local.backup_key_ring_name, local.backup_key_name)].crn : null backup_kms_service_name = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].service_name : null + + parsed_backup_kms_key_crn = local.backup_kms_key_crn != null ? split(":", local.backup_kms_key_crn) : [] + backup_kms_key_id = length(local.parsed_backup_kms_key_crn) > 0 ? local.parsed_backup_kms_key_crn[9] : null } # If existing KMS intance CRN passed, parse details from it @@ -145,7 +148,7 @@ module "backup_kms_instance_crn_parser" { crn = var.existing_backup_kms_instance_crn } -resource "ibm_iam_authorization_policy" "backup_kms_policy" { +resource "ibm_iam_authorization_policy" "backup_kms_policy_cross_account" { count = local.existing_backup_kms_instance_guid == local.existing_kms_instance_guid ? 0 : var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? !var.skip_iam_authorization_policy ? 1 : 0 : 0 provider = ibm.kms source_service_account = local.create_cross_account_auth_policy ? data.ibm_iam_account_settings.iam_account_settings[0].account_id : null @@ -157,6 +160,56 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy" { description = "Allow all Elasticsearch instances in the resource group ${module.resource_group.resource_group_id} to read from the ${local.backup_kms_service_name} instance GUID ${local.existing_backup_kms_instance_guid}" } +# workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 +resource "time_sleep" "wait_for_backup_kms_authorization_policy_cross_account" { + depends_on = [ibm_iam_authorization_policy.backup_kms_policy_cross_account] + create_duration = "30s" +} + +resource "ibm_iam_authorization_policy" "backup_kms_policy" { + count = !var.skip_iam_authorization_policy && local.backup_kms_key_id != null ? 1 : 0 + source_service_account = data.ibm_iam_account_settings.iam_account_settings[0].account_id + source_service_name = "databases-for-elasticsearch" + source_resource_group_id = module.resource_group.resource_group_id + roles = ["Reader"] + description = "Allow all Elastic Search instances in the resource group ${module.resource_group.resource_group_id} in the account ${data.ibm_iam_account_settings.iam_account_settings[0].account_id} to read the ${local.kms_service} key ${local.kms_key_id} from the instance GUID ${local.existing_kms_instance_guid}" + resource_attributes { + name = "serviceName" + operator = "stringEquals" + value = local.kms_service + } + resource_attributes { + name = "accountId" + operator = "stringEquals" + value = local.kms_account_id + } + resource_attributes { + name = "serviceInstance" + operator = "stringEquals" + value = local.existing_kms_instance_guid + } + resource_attributes { + name = "resourceType" + operator = "stringEquals" + value = "key" + } + resource_attributes { + name = "resource" + operator = "stringEquals" + value = local.kms_key_id + } + resource_attributes { + name = "resource" + operator = "stringEquals" + value = local.backup_kms_key_id + } + # Scope of policy now includes the key, so ensure to create new policy before + # destroying old one to prevent any disruption to every day services. + lifecycle { + create_before_destroy = true + } +} + # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 resource "time_sleep" "wait_for_backup_kms_authorization_policy" { depends_on = [ibm_iam_authorization_policy.backup_kms_policy] @@ -200,12 +253,12 @@ module "backup_kms" { module "elasticsearch" { count = local.use_existing_db_instance ? 0 : 1 source = "../../modules/fscloud" - depends_on = [time_sleep.wait_for_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy] + depends_on = [time_sleep.wait_for_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy_cross_account] resource_group_id = module.resource_group.resource_group_id name = var.prefix != null ? "${var.prefix}-${var.name}" : var.name region = var.region plan = var.plan - skip_iam_authorization_policy = var.skip_iam_authorization_policy || local.create_cross_account_auth_policy ? true : var.skip_iam_authorization_policy + skip_iam_authorization_policy = var.skip_iam_authorization_policy || local.create_cross_account_auth_policy elasticsearch_version = var.elasticsearch_version existing_kms_instance_guid = local.existing_kms_instance_guid use_ibm_owned_encryption_key = var.use_ibm_owned_encryption_key diff --git a/solutions/standard/variables.tf b/solutions/standard/variables.tf index 1b4dd59d..61e3bcf1 100644 --- a/solutions/standard/variables.tf +++ b/solutions/standard/variables.tf @@ -300,7 +300,7 @@ variable "service_credential_secrets" { })) })) default = [] - description = "Service credential secrets configuration for Databases for Elasticsearch. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-elasticsearch/tree/main/solutions/instance/DA-types.md#service-credential-secrets)." + description = "Service credential secrets configuration for Databases for Elasticsearch. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/solutions/standard/DA-types.md#service-credential-secrets)." validation { condition = alltrue([ From 86d464ec39959c58f85f8c378f402667c0a25f42 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 18 Nov 2024 09:41:14 +0000 Subject: [PATCH 09/22] fix: add policy --- solutions/standard/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index 01181024..4957134f 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -43,7 +43,7 @@ module "resource_group" { ####################################################################################################################### data "ibm_iam_account_settings" "iam_account_settings" { - count = local.create_cross_account_auth_policy ? 1 : 0 + count = local.create_cross_account_auth_policy || (!var.skip_iam_authorization_policy && local.backup_kms_key_id != null) ? 1 : 0 } resource "ibm_iam_authorization_policy" "kms_policy" { From 047f79a7dd1fc882b99bc833c5d21066a0c173b4 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 18 Nov 2024 10:14:42 +0000 Subject: [PATCH 10/22] fix: add policy --- solutions/standard/main.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index 4957134f..21ff5084 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -193,11 +193,6 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy" { operator = "stringEquals" value = "key" } - resource_attributes { - name = "resource" - operator = "stringEquals" - value = local.kms_key_id - } resource_attributes { name = "resource" operator = "stringEquals" From 943ac34f52542c037a1d27e1cd9815634d3f23b8 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 18 Nov 2024 12:08:24 +0000 Subject: [PATCH 11/22] fix: add policy --- README.md | 2 ++ main.tf | 50 +++++++++++++++++++++++++++++++++-- solutions/standard/main.tf | 54 +++----------------------------------- 3 files changed, 53 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 9c3f3e5e..584deb94 100644 --- a/README.md +++ b/README.md @@ -75,12 +75,14 @@ You need the following permissions to run this module. | Name | Type | |------|------| | [ibm_database.elasticsearch](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/database) | resource | +| [ibm_iam_authorization_policy.backup_kms_policy](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/iam_authorization_policy) | resource | | [ibm_iam_authorization_policy.policy](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/iam_authorization_policy) | resource | | [ibm_resource_key.service_credentials](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/resource_key) | resource | | [ibm_resource_tag.elasticsearch_tag](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/resource_tag) | resource | | [null_resource.put_vectordb_model](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [null_resource.start_vectordb_model](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [time_sleep.wait_for_authorization_policy](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | +| [time_sleep.wait_for_backup_kms_authorization_policy](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | | [ibm_database_connection.database_connection](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/data-sources/database_connection) | data source | ### Inputs diff --git a/main.tf b/main.tf index 8811223a..78e66b08 100644 --- a/main.tf +++ b/main.tf @@ -17,7 +17,9 @@ locals { # For more info, see https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok and https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups" - backup_encryption_key_crn = var.use_default_backup_encryption_key == true ? null : (var.backup_encryption_key_crn != null ? var.backup_encryption_key_crn : var.kms_key_crn) + backup_encryption_key_crn = var.use_default_backup_encryption_key == true ? null : (var.backup_encryption_key_crn != null ? var.backup_encryption_key_crn : var.kms_key_crn) + backup_encryption_key_is_unique = var.backup_encryption_key_crn != var.kms_key_crn + create_backup_kms_policy = local.create_kp_auth_policy == 1 && local.backup_encryption_key_is_unique && local.backup_encryption_key_crn != null # Determine if auto scaling is enabled auto_scaling_enabled = var.auto_scaling == null ? [] : [1] @@ -40,7 +42,7 @@ resource "ibm_iam_authorization_policy" "policy" { source_service_name = "databases-for-elasticsearch" source_resource_group_id = var.resource_group_id roles = ["Reader"] - description = "Allow all Elastic Search instances in the resource group ${var.resource_group_id} to read the ${local.kms_service} key ${local.kms_key_id} from the instance GUID ${var.existing_kms_instance_guid}" + description = "Allow all Elastic Search instances in the account ${local.kms_account_id} in the resource group ${var.resource_group_id} to read the ${local.kms_service} key ${local.kms_key_id} from the instance GUID ${var.existing_kms_instance_guid}" resource_attributes { name = "serviceName" operator = "stringEquals" @@ -81,6 +83,50 @@ resource "time_sleep" "wait_for_authorization_policy" { create_duration = "30s" } +resource "ibm_iam_authorization_policy" "backup_kms_policy" { + count = local.create_backup_kms_policy ? 1 : 0 + source_service_account = local.kms_account_id + source_service_name = "databases-for-elasticsearch" + source_resource_group_id = var.resource_group_id + roles = ["Reader"] + description = "Allow all Elastic Search instances in the account ${local.kms_account_id} in the resource group ${var.resource_group_id} to read the ${local.kms_service} key ${local.backup_encryption_key_crn} from the instance GUID ${var.existing_kms_instance_guid}" + resource_attributes { + name = "serviceName" + operator = "stringEquals" + value = local.kms_service + } + resource_attributes { + name = "accountId" + operator = "stringEquals" + value = local.kms_account_id + } + resource_attributes { + name = "serviceInstance" + operator = "stringEquals" + value = var.existing_kms_instance_guid + } + resource_attributes { + name = "resourceType" + operator = "stringEquals" + value = "key" + } + resource_attributes { + name = "resource" + operator = "stringEquals" + value = local.backup_encryption_key_crn + } + # Scope of policy now includes the key, so ensure to create new policy before + # destroying old one to prevent any disruption to every day services. + lifecycle { + create_before_destroy = true + } +} + +# workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 +resource "time_sleep" "wait_for_backup_kms_authorization_policy" { + depends_on = [ibm_iam_authorization_policy.backup_kms_policy] + create_duration = "30s" +} resource "ibm_database" "elasticsearch" { depends_on = [time_sleep.wait_for_authorization_policy] diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index 21ff5084..b1aa8f0d 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -43,7 +43,7 @@ module "resource_group" { ####################################################################################################################### data "ibm_iam_account_settings" "iam_account_settings" { - count = local.create_cross_account_auth_policy || (!var.skip_iam_authorization_policy && local.backup_kms_key_id != null) ? 1 : 0 + count = local.create_cross_account_auth_policy ? 1 : 0 } resource "ibm_iam_authorization_policy" "kms_policy" { @@ -135,9 +135,6 @@ locals { backup_key_ring_name = var.prefix != null ? "${var.prefix}-backup-encryption-${var.elasticsearch_key_ring_name}" : "backup-encryption-${var.elasticsearch_key_ring_name}" backup_kms_key_crn = var.existing_backup_kms_key_crn != null ? var.existing_backup_kms_key_crn : var.existing_backup_kms_instance_crn != null ? module.backup_kms[0].keys[format("%s.%s", local.backup_key_ring_name, local.backup_key_name)].crn : null backup_kms_service_name = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].service_name : null - - parsed_backup_kms_key_crn = local.backup_kms_key_crn != null ? split(":", local.backup_kms_key_crn) : [] - backup_kms_key_id = length(local.parsed_backup_kms_key_crn) > 0 ? local.parsed_backup_kms_key_crn[9] : null } # If existing KMS intance CRN passed, parse details from it @@ -148,7 +145,7 @@ module "backup_kms_instance_crn_parser" { crn = var.existing_backup_kms_instance_crn } -resource "ibm_iam_authorization_policy" "backup_kms_policy_cross_account" { +resource "ibm_iam_authorization_policy" "backup_kms_policy" { count = local.existing_backup_kms_instance_guid == local.existing_kms_instance_guid ? 0 : var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? !var.skip_iam_authorization_policy ? 1 : 0 : 0 provider = ibm.kms source_service_account = local.create_cross_account_auth_policy ? data.ibm_iam_account_settings.iam_account_settings[0].account_id : null @@ -160,51 +157,6 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy_cross_account" { description = "Allow all Elasticsearch instances in the resource group ${module.resource_group.resource_group_id} to read from the ${local.backup_kms_service_name} instance GUID ${local.existing_backup_kms_instance_guid}" } -# workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 -resource "time_sleep" "wait_for_backup_kms_authorization_policy_cross_account" { - depends_on = [ibm_iam_authorization_policy.backup_kms_policy_cross_account] - create_duration = "30s" -} - -resource "ibm_iam_authorization_policy" "backup_kms_policy" { - count = !var.skip_iam_authorization_policy && local.backup_kms_key_id != null ? 1 : 0 - source_service_account = data.ibm_iam_account_settings.iam_account_settings[0].account_id - source_service_name = "databases-for-elasticsearch" - source_resource_group_id = module.resource_group.resource_group_id - roles = ["Reader"] - description = "Allow all Elastic Search instances in the resource group ${module.resource_group.resource_group_id} in the account ${data.ibm_iam_account_settings.iam_account_settings[0].account_id} to read the ${local.kms_service} key ${local.kms_key_id} from the instance GUID ${local.existing_kms_instance_guid}" - resource_attributes { - name = "serviceName" - operator = "stringEquals" - value = local.kms_service - } - resource_attributes { - name = "accountId" - operator = "stringEquals" - value = local.kms_account_id - } - resource_attributes { - name = "serviceInstance" - operator = "stringEquals" - value = local.existing_kms_instance_guid - } - resource_attributes { - name = "resourceType" - operator = "stringEquals" - value = "key" - } - resource_attributes { - name = "resource" - operator = "stringEquals" - value = local.backup_kms_key_id - } - # Scope of policy now includes the key, so ensure to create new policy before - # destroying old one to prevent any disruption to every day services. - lifecycle { - create_before_destroy = true - } -} - # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 resource "time_sleep" "wait_for_backup_kms_authorization_policy" { depends_on = [ibm_iam_authorization_policy.backup_kms_policy] @@ -248,7 +200,7 @@ module "backup_kms" { module "elasticsearch" { count = local.use_existing_db_instance ? 0 : 1 source = "../../modules/fscloud" - depends_on = [time_sleep.wait_for_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy_cross_account] + depends_on = [time_sleep.wait_for_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy] resource_group_id = module.resource_group.resource_group_id name = var.prefix != null ? "${var.prefix}-${var.name}" : var.name region = var.region From 3a89deb9687da20fa319ec4e36783a64b2bce584 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 18 Nov 2024 12:10:35 +0000 Subject: [PATCH 12/22] fix: add policy --- tests/pr_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pr_test.go b/tests/pr_test.go index c2fbb759..77e523c9 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -155,7 +155,7 @@ func TestRunStandardSolutionSchematics(t *testing.T) { Testing: t, TarIncludePatterns: tarIncludePatterns, TemplateFolder: standardSolutionTerraformDir, - Region: "us-south", + BestRegionYAMLPath: regionSelectionPath, Prefix: "els-sr-da", ResourceGroup: resourceGroup, DeleteWorkspaceOnFail: false, From b79e8ea7ab6b6d2a15813c79503dc5408343c525 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 18 Nov 2024 13:02:44 +0000 Subject: [PATCH 13/22] fix: add policy --- main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 78e66b08..37b6a11f 100644 --- a/main.tf +++ b/main.tf @@ -42,7 +42,7 @@ resource "ibm_iam_authorization_policy" "policy" { source_service_name = "databases-for-elasticsearch" source_resource_group_id = var.resource_group_id roles = ["Reader"] - description = "Allow all Elastic Search instances in the account ${local.kms_account_id} in the resource group ${var.resource_group_id} to read the ${local.kms_service} key ${local.kms_key_id} from the instance GUID ${var.existing_kms_instance_guid}" + description = "Allow all Elastic Search instances in the resource group ${var.resource_group_id} to read the ${local.kms_service} key ${local.kms_key_id} from the instance GUID ${var.existing_kms_instance_guid}" resource_attributes { name = "serviceName" operator = "stringEquals" @@ -89,7 +89,7 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy" { source_service_name = "databases-for-elasticsearch" source_resource_group_id = var.resource_group_id roles = ["Reader"] - description = "Allow all Elastic Search instances in the account ${local.kms_account_id} in the resource group ${var.resource_group_id} to read the ${local.kms_service} key ${local.backup_encryption_key_crn} from the instance GUID ${var.existing_kms_instance_guid}" + description = "Allow all Elastic Search instances in the resource group ${var.resource_group_id} to read the ${local.kms_service} key ${local.backup_encryption_key_crn} from the instance GUID ${var.existing_kms_instance_guid}" resource_attributes { name = "serviceName" operator = "stringEquals" From 37f0e96b05112c0acc69de5f09eb25b6bd4e4eee Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 18 Nov 2024 13:46:47 +0000 Subject: [PATCH 14/22] fix: add policy --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 37b6a11f..b8769a4e 100644 --- a/main.tf +++ b/main.tf @@ -89,7 +89,7 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy" { source_service_name = "databases-for-elasticsearch" source_resource_group_id = var.resource_group_id roles = ["Reader"] - description = "Allow all Elastic Search instances in the resource group ${var.resource_group_id} to read the ${local.kms_service} key ${local.backup_encryption_key_crn} from the instance GUID ${var.existing_kms_instance_guid}" + description = "Allow all ES instances in the RG ${var.resource_group_id} to read the ${local.kms_service} key ${local.backup_encryption_key_crn} from the instance GUID ${var.existing_kms_instance_guid}" resource_attributes { name = "serviceName" operator = "stringEquals" From 5cbed2b43b53fe1422d1723c6c8fdadba1787c8d Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 18 Nov 2024 13:56:55 +0000 Subject: [PATCH 15/22] fix: add policy --- main.tf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index b8769a4e..bf31c190 100644 --- a/main.tf +++ b/main.tf @@ -17,7 +17,10 @@ locals { # For more info, see https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok and https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups" - backup_encryption_key_crn = var.use_default_backup_encryption_key == true ? null : (var.backup_encryption_key_crn != null ? var.backup_encryption_key_crn : var.kms_key_crn) + backup_encryption_key_crn = var.use_default_backup_encryption_key == true ? null : (var.backup_encryption_key_crn != null ? var.backup_encryption_key_crn : var.kms_key_crn) + parsed_backup_encryption_key_crn = local.backup_encryption_key_crn != null ? split(":", local.backup_encryption_key_crn) : [] + backup_kms_key_id = length(local.parsed_backup_encryption_key_crn) > 0 ? local.parsed_backup_encryption_key_crn[9] : null + backup_encryption_key_is_unique = var.backup_encryption_key_crn != var.kms_key_crn create_backup_kms_policy = local.create_kp_auth_policy == 1 && local.backup_encryption_key_is_unique && local.backup_encryption_key_crn != null @@ -89,7 +92,7 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy" { source_service_name = "databases-for-elasticsearch" source_resource_group_id = var.resource_group_id roles = ["Reader"] - description = "Allow all ES instances in the RG ${var.resource_group_id} to read the ${local.kms_service} key ${local.backup_encryption_key_crn} from the instance GUID ${var.existing_kms_instance_guid}" + description = "Allow all Elastic Search instances in the Resource Group ${var.resource_group_id} to read the ${local.kms_service} key ${local.backup_kms_key_id} from the instance GUID ${var.existing_kms_instance_guid}" resource_attributes { name = "serviceName" operator = "stringEquals" From 1365c2570d4ab8bc25f5cdd3299a0839796f5a0a Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 18 Nov 2024 14:57:41 +0000 Subject: [PATCH 16/22] fix: add policy --- main.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/main.tf b/main.tf index bf31c190..cb7e6e9f 100644 --- a/main.tf +++ b/main.tf @@ -21,8 +21,7 @@ locals { parsed_backup_encryption_key_crn = local.backup_encryption_key_crn != null ? split(":", local.backup_encryption_key_crn) : [] backup_kms_key_id = length(local.parsed_backup_encryption_key_crn) > 0 ? local.parsed_backup_encryption_key_crn[9] : null - backup_encryption_key_is_unique = var.backup_encryption_key_crn != var.kms_key_crn - create_backup_kms_policy = local.create_kp_auth_policy == 1 && local.backup_encryption_key_is_unique && local.backup_encryption_key_crn != null + create_backup_kms_policy = local.create_kp_auth_policy == 1 && local.backup_encryption_key_crn != null # Determine if auto scaling is enabled auto_scaling_enabled = var.auto_scaling == null ? [] : [1] From 8bae00b717975765fcb100a20e6e443fcb857256 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 18 Nov 2024 15:37:08 +0000 Subject: [PATCH 17/22] fix: add policy --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index cb7e6e9f..fcd40bc0 100644 --- a/main.tf +++ b/main.tf @@ -40,7 +40,7 @@ locals { # Create IAM Access Policy to allow Key protect to access Elasticsearch instance resource "ibm_iam_authorization_policy" "policy" { - count = local.create_kp_auth_policy + count = local.create_kp_auth_policy ? 1 : 0 source_service_name = "databases-for-elasticsearch" source_resource_group_id = var.resource_group_id roles = ["Reader"] From 89e82d97f7161597bfca7765394f32852edc3c9b Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 18 Nov 2024 15:52:32 +0000 Subject: [PATCH 18/22] fix: add policy sleep --- main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index fcd40bc0..480a811c 100644 --- a/main.tf +++ b/main.tf @@ -40,7 +40,7 @@ locals { # Create IAM Access Policy to allow Key protect to access Elasticsearch instance resource "ibm_iam_authorization_policy" "policy" { - count = local.create_kp_auth_policy ? 1 : 0 + count = local.create_kp_auth_policy source_service_name = "databases-for-elasticsearch" source_resource_group_id = var.resource_group_id roles = ["Reader"] @@ -126,7 +126,7 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy" { # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 resource "time_sleep" "wait_for_backup_kms_authorization_policy" { - depends_on = [ibm_iam_authorization_policy.backup_kms_policy] + depends_on = [ibm_iam_authorization_policy.backup_kms_policy, ibm_iam_authorization_policy.backup_kms_policy] create_duration = "30s" } From bed3396c141b81d0cbcd427a7fcd2970591f0710 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 18 Nov 2024 16:37:56 +0000 Subject: [PATCH 19/22] fix: id instead of crn --- main.tf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/main.tf b/main.tf index 480a811c..bec49e14 100644 --- a/main.tf +++ b/main.tf @@ -87,7 +87,6 @@ resource "time_sleep" "wait_for_authorization_policy" { resource "ibm_iam_authorization_policy" "backup_kms_policy" { count = local.create_backup_kms_policy ? 1 : 0 - source_service_account = local.kms_account_id source_service_name = "databases-for-elasticsearch" source_resource_group_id = var.resource_group_id roles = ["Reader"] @@ -115,7 +114,7 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy" { resource_attributes { name = "resource" operator = "stringEquals" - value = local.backup_encryption_key_crn + value = local.backup_kms_key_id } # Scope of policy now includes the key, so ensure to create new policy before # destroying old one to prevent any disruption to every day services. @@ -126,12 +125,12 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy" { # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 resource "time_sleep" "wait_for_backup_kms_authorization_policy" { - depends_on = [ibm_iam_authorization_policy.backup_kms_policy, ibm_iam_authorization_policy.backup_kms_policy] + depends_on = [ibm_iam_authorization_policy.backup_kms_policy] create_duration = "30s" } resource "ibm_database" "elasticsearch" { - depends_on = [time_sleep.wait_for_authorization_policy] + depends_on = [time_sleep.wait_for_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy] name = var.name plan = var.plan location = var.region From 9df97ea07a1d27ee579c93aa7a98e8871a851461 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Tue, 19 Nov 2024 20:28:37 +0000 Subject: [PATCH 20/22] fix: update for tests --- main.tf | 2 +- modules/fscloud/main.tf | 2 +- solutions/standard/variables.tf | 41 ++++++++++++++++++++++----------- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/main.tf b/main.tf index bec49e14..cfa1d73d 100644 --- a/main.tf +++ b/main.tf @@ -21,7 +21,7 @@ locals { parsed_backup_encryption_key_crn = local.backup_encryption_key_crn != null ? split(":", local.backup_encryption_key_crn) : [] backup_kms_key_id = length(local.parsed_backup_encryption_key_crn) > 0 ? local.parsed_backup_encryption_key_crn[9] : null - create_backup_kms_policy = local.create_kp_auth_policy == 1 && local.backup_encryption_key_crn != null + create_backup_kms_policy = local.create_kp_auth_policy == 1 && local.backup_encryption_key_crn != null && var.backup_encryption_key_crn != null # Determine if auto scaling is enabled auto_scaling_enabled = var.auto_scaling == null ? [] : [1] diff --git a/modules/fscloud/main.tf b/modules/fscloud/main.tf index a40ceb98..b4a681c7 100644 --- a/modules/fscloud/main.tf +++ b/modules/fscloud/main.tf @@ -11,7 +11,7 @@ module "elasticsearch" { name = var.name region = var.region skip_iam_authorization_policy = var.skip_iam_authorization_policy - service_endpoints = "private" + service_endpoints = "public" elasticsearch_version = var.elasticsearch_version kms_encryption_enabled = !var.use_ibm_owned_encryption_key existing_kms_instance_guid = var.existing_kms_instance_guid diff --git a/solutions/standard/variables.tf b/solutions/standard/variables.tf index 61e3bcf1..4c07835a 100644 --- a/solutions/standard/variables.tf +++ b/solutions/standard/variables.tf @@ -18,7 +18,7 @@ variable "ibmcloud_kms_api_key" { variable "provider_visibility" { description = "Set the visibility value for the IBM terraform provider. Supported values are `public`, `private`, `public-and-private`. [Learn more](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/guides/custom-service-endpoints)." type = string - default = "private" + default = "public" validation { condition = contains(["public", "private", "public-and-private"], var.provider_visibility) @@ -29,7 +29,7 @@ variable "provider_visibility" { variable "prefix" { type = string description = "Prefix to add to all resources created by this solution." - default = null + default = "test-es" } ############################################################################## @@ -39,6 +39,7 @@ variable "prefix" { variable "resource_group_name" { type = string description = "The name of a new or an existing resource group to provision the Databases for Elasicsearch in. If a prefix input variable is specified, the prefix is added to the name in the `-` format." + default = "test-es" } variable "use_existing_resource_group" { @@ -162,7 +163,7 @@ variable "users" { variable "service_credential_names" { type = map(string) description = "The map of name and role for service credentials that you want to create for the database. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/tree/main/solutions/standard/DA-types.md)." - default = {} + default = { "admin_test" : "Administrator", "editor_test" : "Editor" } } variable "tags" { @@ -227,7 +228,7 @@ variable "skip_iam_authorization_policy" { variable "kms_endpoint_type" { type = string description = "The type of endpoint to use to communicate with the KMS instance. Possible values: `public`, `private`." - default = "private" + default = "public" validation { condition = can(regex("public|private", var.kms_endpoint_type)) error_message = "The kms_endpoint_type value must be 'public' or 'private'." @@ -237,7 +238,7 @@ variable "kms_endpoint_type" { variable "existing_kms_instance_crn" { type = string description = "The CRN of an Hyper Protect Crypto Services or Key Protect instance that you want to use for both disk and backup encryption. Backup encryption is only supported is some regions ([learn more](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok)), so if you need to use a different instance for backup encryption from a supported region, use the `existing_backup_kms_instance_crn` input." - default = null + default = "crn:v1:bluemix:public:hs-crypto:us-south:a/abac0df06b644a9cabc6e44f55b3880e:e6dce284-e80f-46e1-a3c1-830f7adff7a9::" } ############################################################## @@ -268,14 +269,14 @@ variable "elasticsearch_key_name" { variable "existing_secrets_manager_instance_crn" { type = string - default = null + default = "crn:v1:bluemix:public:secrets-manager:us-south:a/abac0df06b644a9cabc6e44f55b3880e:79c6d411-c18f-4670-b009-b0044a238667::" description = "The CRN of existing secrets manager to use to create service credential secrets for Databases for Elasticsearch instance." } variable "existing_secrets_manager_endpoint_type" { type = string description = "The endpoint type to use if `existing_secrets_manager_instance_crn` is specified. Possible values: public, private." - default = "private" + default = "public" validation { condition = contains(["public", "private"], var.existing_secrets_manager_endpoint_type) error_message = "Only \"public\" and \"private\" are allowed values for 'existing_secrets_endpoint_type'." @@ -299,8 +300,22 @@ variable "service_credential_secrets" { })) })) - default = [] - description = "Service credential secrets configuration for Databases for Elasticsearch. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/solutions/standard/DA-types.md#service-credential-secrets)." + default = [ + { + "secret_group_name" : "test-es-secret-group", + "service_credentials" : [ + { + "secret_name" : "test-es-cred-reader", + "service_credentials_source_service_role" : "Reader", + }, + { + "secret_name" : "test-es-cred-writer", + "service_credentials_source_service_role" : "Writer", + }, + ] + }, + ] + description = "Service credential secrets configuration for Databases for Elasticsearch. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-elasticsearch/tree/main/solutions/instance/DA-types.md#service-credential-secrets)." validation { condition = alltrue([ @@ -324,7 +339,7 @@ variable "skip_es_sm_auth_policy" { variable "admin_pass_sm_secret_group" { type = string description = "The name of a new or existing secrets manager secret group for admin password. To use existing secret group, `use_existing_admin_pass_sm_secret_group` must be set to `true`. If a prefix input variable is specified, the prefix is added to the name in the `-` format." - default = "elasticsearch-secrets" + default = "test-es" } variable "use_existing_admin_pass_sm_secret_group" { @@ -336,7 +351,7 @@ variable "use_existing_admin_pass_sm_secret_group" { variable "admin_pass_sm_secret_name" { type = string description = "The name of a new elasticsearch administrator secret. If a prefix input variable is specified, the prefix is added to the name in the `-` format." - default = "elasticsearch-admin-password" + default = "test-es" } ############################################################## @@ -352,7 +367,7 @@ variable "existing_code_engine_project_id" { variable "enable_kibana_dashboard" { type = bool description = "Set it true to deploy Kibana in code engine. NOTE: Kibana image is coming direcly from the official registry (https://www.docker.elastic.co/) and not certified by the IBM." - default = false + default = true } variable "elasticsearch_full_version" { @@ -367,7 +382,7 @@ variable "elasticsearch_full_version" { variable "existing_backup_kms_key_crn" { type = string description = "The CRN of an Hyper Protect Crypto Services or Key Protect encryption key that you want to use to encrypt database backups. If no value is passed, the value of `existing_kms_key_crn` is used. If no value is passed for that, a new key will be created in the provided KMS instance and used for both disk encryption, and backup encryption." - default = null + default = "crn:v1:bluemix:public:hs-crypto:us-south:a/abac0df06b644a9cabc6e44f55b3880e:e6dce284-e80f-46e1-a3c1-830f7adff7a9:key:76170fae-4e0c-48c3-8ebe-326059ebb533" } variable "existing_backup_kms_instance_crn" { From 613facc2849458471f1a5dec3603c08f53966a66 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Tue, 19 Nov 2024 20:38:26 +0000 Subject: [PATCH 21/22] fix: id instead of crn --- modules/fscloud/main.tf | 2 +- solutions/standard/variables.tf | 41 +++++++++++---------------------- 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/modules/fscloud/main.tf b/modules/fscloud/main.tf index b4a681c7..a40ceb98 100644 --- a/modules/fscloud/main.tf +++ b/modules/fscloud/main.tf @@ -11,7 +11,7 @@ module "elasticsearch" { name = var.name region = var.region skip_iam_authorization_policy = var.skip_iam_authorization_policy - service_endpoints = "public" + service_endpoints = "private" elasticsearch_version = var.elasticsearch_version kms_encryption_enabled = !var.use_ibm_owned_encryption_key existing_kms_instance_guid = var.existing_kms_instance_guid diff --git a/solutions/standard/variables.tf b/solutions/standard/variables.tf index 4c07835a..61e3bcf1 100644 --- a/solutions/standard/variables.tf +++ b/solutions/standard/variables.tf @@ -18,7 +18,7 @@ variable "ibmcloud_kms_api_key" { variable "provider_visibility" { description = "Set the visibility value for the IBM terraform provider. Supported values are `public`, `private`, `public-and-private`. [Learn more](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/guides/custom-service-endpoints)." type = string - default = "public" + default = "private" validation { condition = contains(["public", "private", "public-and-private"], var.provider_visibility) @@ -29,7 +29,7 @@ variable "provider_visibility" { variable "prefix" { type = string description = "Prefix to add to all resources created by this solution." - default = "test-es" + default = null } ############################################################################## @@ -39,7 +39,6 @@ variable "prefix" { variable "resource_group_name" { type = string description = "The name of a new or an existing resource group to provision the Databases for Elasicsearch in. If a prefix input variable is specified, the prefix is added to the name in the `-` format." - default = "test-es" } variable "use_existing_resource_group" { @@ -163,7 +162,7 @@ variable "users" { variable "service_credential_names" { type = map(string) description = "The map of name and role for service credentials that you want to create for the database. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/tree/main/solutions/standard/DA-types.md)." - default = { "admin_test" : "Administrator", "editor_test" : "Editor" } + default = {} } variable "tags" { @@ -228,7 +227,7 @@ variable "skip_iam_authorization_policy" { variable "kms_endpoint_type" { type = string description = "The type of endpoint to use to communicate with the KMS instance. Possible values: `public`, `private`." - default = "public" + default = "private" validation { condition = can(regex("public|private", var.kms_endpoint_type)) error_message = "The kms_endpoint_type value must be 'public' or 'private'." @@ -238,7 +237,7 @@ variable "kms_endpoint_type" { variable "existing_kms_instance_crn" { type = string description = "The CRN of an Hyper Protect Crypto Services or Key Protect instance that you want to use for both disk and backup encryption. Backup encryption is only supported is some regions ([learn more](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok)), so if you need to use a different instance for backup encryption from a supported region, use the `existing_backup_kms_instance_crn` input." - default = "crn:v1:bluemix:public:hs-crypto:us-south:a/abac0df06b644a9cabc6e44f55b3880e:e6dce284-e80f-46e1-a3c1-830f7adff7a9::" + default = null } ############################################################## @@ -269,14 +268,14 @@ variable "elasticsearch_key_name" { variable "existing_secrets_manager_instance_crn" { type = string - default = "crn:v1:bluemix:public:secrets-manager:us-south:a/abac0df06b644a9cabc6e44f55b3880e:79c6d411-c18f-4670-b009-b0044a238667::" + default = null description = "The CRN of existing secrets manager to use to create service credential secrets for Databases for Elasticsearch instance." } variable "existing_secrets_manager_endpoint_type" { type = string description = "The endpoint type to use if `existing_secrets_manager_instance_crn` is specified. Possible values: public, private." - default = "public" + default = "private" validation { condition = contains(["public", "private"], var.existing_secrets_manager_endpoint_type) error_message = "Only \"public\" and \"private\" are allowed values for 'existing_secrets_endpoint_type'." @@ -300,22 +299,8 @@ variable "service_credential_secrets" { })) })) - default = [ - { - "secret_group_name" : "test-es-secret-group", - "service_credentials" : [ - { - "secret_name" : "test-es-cred-reader", - "service_credentials_source_service_role" : "Reader", - }, - { - "secret_name" : "test-es-cred-writer", - "service_credentials_source_service_role" : "Writer", - }, - ] - }, - ] - description = "Service credential secrets configuration for Databases for Elasticsearch. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-elasticsearch/tree/main/solutions/instance/DA-types.md#service-credential-secrets)." + default = [] + description = "Service credential secrets configuration for Databases for Elasticsearch. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/solutions/standard/DA-types.md#service-credential-secrets)." validation { condition = alltrue([ @@ -339,7 +324,7 @@ variable "skip_es_sm_auth_policy" { variable "admin_pass_sm_secret_group" { type = string description = "The name of a new or existing secrets manager secret group for admin password. To use existing secret group, `use_existing_admin_pass_sm_secret_group` must be set to `true`. If a prefix input variable is specified, the prefix is added to the name in the `-` format." - default = "test-es" + default = "elasticsearch-secrets" } variable "use_existing_admin_pass_sm_secret_group" { @@ -351,7 +336,7 @@ variable "use_existing_admin_pass_sm_secret_group" { variable "admin_pass_sm_secret_name" { type = string description = "The name of a new elasticsearch administrator secret. If a prefix input variable is specified, the prefix is added to the name in the `-` format." - default = "test-es" + default = "elasticsearch-admin-password" } ############################################################## @@ -367,7 +352,7 @@ variable "existing_code_engine_project_id" { variable "enable_kibana_dashboard" { type = bool description = "Set it true to deploy Kibana in code engine. NOTE: Kibana image is coming direcly from the official registry (https://www.docker.elastic.co/) and not certified by the IBM." - default = true + default = false } variable "elasticsearch_full_version" { @@ -382,7 +367,7 @@ variable "elasticsearch_full_version" { variable "existing_backup_kms_key_crn" { type = string description = "The CRN of an Hyper Protect Crypto Services or Key Protect encryption key that you want to use to encrypt database backups. If no value is passed, the value of `existing_kms_key_crn` is used. If no value is passed for that, a new key will be created in the provided KMS instance and used for both disk encryption, and backup encryption." - default = "crn:v1:bluemix:public:hs-crypto:us-south:a/abac0df06b644a9cabc6e44f55b3880e:e6dce284-e80f-46e1-a3c1-830f7adff7a9:key:76170fae-4e0c-48c3-8ebe-326059ebb533" + default = null } variable "existing_backup_kms_instance_crn" { From 592308ed8206952feff7a199bd4d81025c3ae7b9 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Wed, 20 Nov 2024 09:50:43 +0000 Subject: [PATCH 22/22] fix: cra --- cra-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cra-config.yaml b/cra-config.yaml index 7221dfe9..e3c2e331 100644 --- a/cra-config.yaml +++ b/cra-config.yaml @@ -6,7 +6,7 @@ CRA_TARGETS: PROFILE_ID: "bfacb71d-4b84-41ac-9825-e8a3a3eb7405" # SCC profile ID (currently set to IBM Cloud Framework for Financial Services 1.6.0 profile). CRA_ENVIRONMENT_VARIABLES: TF_VAR_existing_kms_instance_crn: "crn:v1:bluemix:public:hs-crypto:us-south:a/abac0df06b644a9cabc6e44f55b3880e:e6dce284-e80f-46e1-a3c1-830f7adff7a9::" - TF_VAR_kms_key_crn: "crn:v1:bluemix:public:hs-crypto:us-south:a/abac0df06b644a9cabc6e44f55b3880e:e6dce284-e80f-46e1-a3c1-830f7adff7a9:key:76170fae-4e0c-48c3-8ebe-326059ebb533" + TF_VAR_existing_kms_key_crn: "crn:v1:bluemix:public:hs-crypto:us-south:a/abac0df06b644a9cabc6e44f55b3880e:e6dce284-e80f-46e1-a3c1-830f7adff7a9:key:76170fae-4e0c-48c3-8ebe-326059ebb533" TF_VAR_provider_visibility: "public" TF_VAR_resource_group_name: "test-es-cra" TF_VAR_use_existing_resource_group: false