diff --git a/README.md b/README.md index 721dddca..e34369d4 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,7 @@ You need the following permissions to run this module. | [service\_endpoints](#input\_service\_endpoints) | The type of endpoint of the database instance. Possible values: `public`, `private`, `public-and-private`. | `string` | `"public"` | no | | [skip\_iam\_authorization\_policy](#input\_skip\_iam\_authorization\_policy) | Whether to create an IAM authorization policy that permits all Databases for Elasticsearch instances in the resource group to read the encryption key from the Hyper Protect Crypto Services instance specified in the `existing_kms_instance_guid` variable. If set to `false`, specify a value for the KMS instance in the `existing_kms_instance_guid` variable. No policy is created if `kms_encryption_enabled` is false. | `bool` | `false` | no | | [tags](#input\_tags) | The list of tags to be added to the Databases for Elasticsearch instance. | `list(string)` | `[]` | no | +| [use\_custom\_backup\_encryption\_key](#input\_use\_custom\_backup\_encryption\_key) | Whether to use a custom IBM Cloud Databases generated key for backup encryption. | `bool` | `false` | no | | [use\_default\_backup\_encryption\_key](#input\_use\_default\_backup\_encryption\_key) | Whether to use the IBM Cloud Databases generated keys for backup encryption. | `bool` | `false` | no | | [users](#input\_users) | The list of users that have access to the database. Multiple blocks are allowed. The user password must be 10-32 characters. In most cases, you can use IAM service credentials (by specifying `service_credential_names`) to control access to the database instance. This block creates native database users. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-user-management&interface=ui). |
list(object({
name = string
password = string # pragma: allowlist secret
type = optional(string)
role = optional(string)
}))
| `[]` | no | diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 913c54cd..369eb636 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -34,6 +34,10 @@ module "key_protect_all_inclusive" { { key_name = "${var.prefix}-elasticsearch" force_delete = true + }, + { + key_name = "backup-${var.prefix}-elasticsearch" + force_delete = true } ] } @@ -45,23 +49,25 @@ module "key_protect_all_inclusive" { ############################################################################## module "icd_elasticsearch" { - source = "../../" - resource_group_id = module.resource_group.resource_group_id - name = "${var.prefix}-elasticsearch" - region = var.region - plan = var.plan - kms_encryption_enabled = true - access_tags = var.access_tags - admin_pass = var.admin_pass - users = var.users - existing_kms_instance_guid = module.key_protect_all_inclusive.kms_guid - service_credential_names = var.service_credential_names - elasticsearch_version = var.elasticsearch_version - kms_key_crn = module.key_protect_all_inclusive.keys["icd.${var.prefix}-elasticsearch"].crn - tags = var.resource_tags - auto_scaling = var.auto_scaling - member_host_flavor = "multitenant" - member_memory_mb = 4096 + source = "../../" + resource_group_id = module.resource_group.resource_group_id + name = "${var.prefix}-elasticsearch" + region = var.region + plan = var.plan + kms_encryption_enabled = true + access_tags = var.access_tags + admin_pass = var.admin_pass + users = var.users + existing_kms_instance_guid = module.key_protect_all_inclusive.kms_guid + service_credential_names = var.service_credential_names + elasticsearch_version = var.elasticsearch_version + kms_key_crn = module.key_protect_all_inclusive.keys["icd.${var.prefix}-elasticsearch"].crn + backup_encryption_key_crn = module.key_protect_all_inclusive.keys["icd.backup-${var.prefix}-elasticsearch"].crn + use_custom_backup_encryption_key = true + tags = var.resource_tags + auto_scaling = var.auto_scaling + member_host_flavor = "multitenant" + member_memory_mb = 4096 } diff --git a/examples/fscloud/main.tf b/examples/fscloud/main.tf index 888bf044..a1cdeb25 100644 --- a/examples/fscloud/main.tf +++ b/examples/fscloud/main.tf @@ -49,26 +49,57 @@ module "cbr_zone" { }] } +############################################################################## +# Key Protect All Inclusive +############################################################################## + +module "key_protect_all_inclusive" { + source = "terraform-ibm-modules/kms-all-inclusive/ibm" + version = "4.17.1" + resource_group_id = module.resource_group.resource_group_id + # Only us-south, eu-de backup encryption keys are supported. See https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok for details. + # Note: Database instance and Key Protect must be created on the same region. + region = var.region + key_protect_instance_name = "${var.prefix}-kp" + resource_tags = var.resource_tags + keys = [ + { + key_ring_name = "icd" + keys = [ + { + key_name = "${var.prefix}-elasticsearch" + force_delete = true + }, + { + key_name = "backup-${var.prefix}-elasticsearch" + force_delete = true + } + ] + } + ] +} + ############################################################################## # ICD elasticsearch database ############################################################################## module "elasticsearch" { - source = "../../modules/fscloud" - resource_group_id = module.resource_group.resource_group_id - name = "${var.prefix}-elasticsearch" - region = var.region - tags = var.resource_tags - access_tags = var.access_tags - kms_key_crn = var.kms_key_crn - existing_kms_instance_guid = var.existing_kms_instance_guid - elasticsearch_version = var.elasticsearch_version - service_credential_names = var.service_credential_names - auto_scaling = var.auto_scaling - member_host_flavor = "b3c.4x16.encrypted" - backup_encryption_key_crn = var.backup_encryption_key_crn - backup_crn = var.backup_crn - enable_elser_model = var.enable_elser_model + source = "../../modules/fscloud" + resource_group_id = module.resource_group.resource_group_id + name = "${var.prefix}-elasticsearch" + region = var.region + tags = var.resource_tags + access_tags = var.access_tags + existing_kms_instance_guid = module.key_protect_all_inclusive.kms_guid + kms_key_crn = module.key_protect_all_inclusive.keys["icd.${var.prefix}-elasticsearch"].crn + backup_encryption_key_crn = module.key_protect_all_inclusive.keys["icd.backup-${var.prefix}-elasticsearch"].crn + use_custom_backup_encryption_key = true + elasticsearch_version = var.elasticsearch_version + service_credential_names = var.service_credential_names + auto_scaling = var.auto_scaling + member_host_flavor = "b3c.4x16.encrypted" + backup_crn = var.backup_crn + enable_elser_model = var.enable_elser_model cbr_rules = [ { description = "${var.prefix}-elasticsearch access only from vpc" diff --git a/examples/fscloud/variables.tf b/examples/fscloud/variables.tf index 664994c7..3accdea5 100644 --- a/examples/fscloud/variables.tf +++ b/examples/fscloud/variables.tf @@ -34,20 +34,9 @@ variable "access_tags" { default = [] } -variable "existing_kms_instance_guid" { - description = "The GUID of the Hyper Protect Crypto services in which the key specified in var.kms_key_crn is coming from" - type = string -} - -variable "kms_key_crn" { - type = string - description = "The root key CRN of a Hyper Protect Crypto Services (HPCS) that you want to use for disk encryption. See https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs&interface=ui for more information on integrating HPCS with Elasticsearch instance." -} - variable "elasticsearch_version" { type = string description = "Version of the Elasticsearch instance. If no value is passed, the current preferred version of IBM Cloud Databases is used." - default = null } variable "service_credential_names" { @@ -102,13 +91,6 @@ variable "backup_crn" { default = null } -variable "backup_encryption_key_crn" { - type = string - description = "The CRN of a Hyper Protect Crypto Services use for encrypting the disk that holds deployment backups. There are limitation per region on the Hyper Protect Crypto Services and region for those services. See https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups" - default = null - # Validation happens in the root module -} - variable "enable_elser_model" { type = bool description = "Set it to true to install and start the Elastic's Natural Language Processing model. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-elser-embeddings-elasticsearch)" diff --git a/ibm_catalog.json b/ibm_catalog.json index bfc04fd4..05870c44 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -291,6 +291,9 @@ { "key": "existing_backup_kms_instance_crn" }, + { + "key": "use_custom_backup_encryption_key" + }, { "key": "enable_elser_model" }, diff --git a/main.tf b/main.tf index 6903f08f..b6e7700e 100644 --- a/main.tf +++ b/main.tf @@ -7,7 +7,13 @@ locals { # tflint-ignore: terraform_unused_declarations validate_auth_policy = var.kms_encryption_enabled && var.skip_iam_authorization_policy == false && var.existing_kms_instance_guid == null ? tobool("When var.skip_iam_authorization_policy is set to false, and var.kms_encryption_enabled to true, a value must be passed for var.existing_kms_instance_guid in order to create the auth policy.") : true # tflint-ignore: terraform_unused_declarations - validate_backup_key = var.backup_encryption_key_crn != null && var.use_default_backup_encryption_key == true ? tobool("When passing a value for 'backup_encryption_key_crn' you cannot set 'use_default_backup_encryption_key' to 'true'") : true + validate_backup_key_default = var.backup_encryption_key_crn != null && var.use_default_backup_encryption_key == true ? tobool("When passing a value for 'backup_encryption_key_crn' you cannot set 'use_default_backup_encryption_key' to 'true'") : true + # tflint-ignore: terraform_unused_declarations + validate_backup_key_custom = var.backup_encryption_key_crn == null && var.use_custom_backup_encryption_key == true ? tobool("When setting 'use_custom_backup_encryption_key' to 'true' you must also pass a value for 'backup_encryption_key_crn'") : true + # tflint-ignore: terraform_unused_declarations + validate_backup_key_custom_flag = var.backup_encryption_key_crn != null && var.use_custom_backup_encryption_key == false ? tobool("When passing a value for 'backup_encryption_key_crn', 'use_custom_backup_encryption_key' must also be set to 'true'") : true + # tflint-ignore: terraform_unused_declarations + validate_backup_key_custom_or_default = var.use_default_backup_encryption_key == true && var.use_custom_backup_encryption_key == true ? tobool("You cannot set 'use_default_backup_encryption_key' and 'use_custom_backup_encryption_key' simultaneously. You must choose one or the other.") : true # tflint-ignore: terraform_unused_declarations validate_plan = var.enable_elser_model && var.plan != "platinum" ? tobool("When var.enable_elser_model is set to true, a value for var.plan must be 'platinum' in order to enable ELSER model.") : true # tflint-ignore: terraform_unused_declarations @@ -21,7 +27,7 @@ locals { parsed_backup_encryption_key_crn = local.backup_encryption_key_crn != null ? split(":", local.backup_encryption_key_crn) : [] backup_kms_key_id = length(local.parsed_backup_encryption_key_crn) > 0 ? local.parsed_backup_encryption_key_crn[9] : null - create_backup_kms_policy = local.create_kp_auth_policy == 1 && local.backup_encryption_key_crn != null && var.backup_encryption_key_crn != null + create_backup_kms_policy = local.create_kp_auth_policy == 1 && var.use_custom_backup_encryption_key # Determine if auto scaling is enabled auto_scaling_enabled = var.auto_scaling == null ? [] : [1] @@ -125,6 +131,7 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy" { # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 resource "time_sleep" "wait_for_backup_kms_authorization_policy" { + count = local.create_backup_kms_policy ? 1 : 0 depends_on = [ibm_iam_authorization_policy.backup_kms_policy] create_duration = "30s" } diff --git a/modules/fscloud/README.md b/modules/fscloud/README.md index d89eaa74..fc4165d8 100644 --- a/modules/fscloud/README.md +++ b/modules/fscloud/README.md @@ -53,6 +53,7 @@ No resources. | [service\_credential\_names](#input\_service\_credential\_names) | Map of name, role for service credentials that you want to create for the database | `map(string)` | `{}` | no | | [skip\_iam\_authorization\_policy](#input\_skip\_iam\_authorization\_policy) | Set to true to skip the creation of an IAM authorization policy that permits all ElasticSearch database instances in the resource group to read the encryption key from the Hyper Protect Crypto Services or Key Protect instance. The instance is passed in through the var.existing\_kms\_instance\_guid variable. | `bool` | `false` | no | | [tags](#input\_tags) | Optional list of tags to be added to the Elasticsearch instance. | `list(any)` | `[]` | no | +| [use\_custom\_backup\_encryption\_key](#input\_use\_custom\_backup\_encryption\_key) | Whether to use a custom IBM Cloud Databases generated key for backup encryption. | `bool` | `false` | no | | [use\_ibm\_owned\_encryption\_key](#input\_use\_ibm\_owned\_encryption\_key) | Set to true to use the default IBM Cloud® Databases randomly generated keys for disk and backups encryption. To control the encryption keys, use the `kms_key_crn` and `backup_encryption_key_crn` inputs. | `bool` | `false` | no | | [users](#input\_users) | A list of users that you want to create on the database. Multiple blocks are allowed. The user password must be in the range of 10-32 characters. Be warned that in most case using IAM service credentials (via the var.service\_credential\_names) is sufficient to control access to the Elasticsearch instance. This blocks creates native Elasticsearch database users, more info on that can be found here https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-user-management&interface=ui |
list(object({
name = string
password = string # pragma: allowlist secret
type = optional(string)
role = optional(string)
}))
| `[]` | no | diff --git a/modules/fscloud/main.tf b/modules/fscloud/main.tf index a40ceb98..a2865935 100644 --- a/modules/fscloud/main.tf +++ b/modules/fscloud/main.tf @@ -6,31 +6,32 @@ locals { } module "elasticsearch" { - source = "../../" - resource_group_id = var.resource_group_id - name = var.name - region = var.region - skip_iam_authorization_policy = var.skip_iam_authorization_policy - service_endpoints = "private" - elasticsearch_version = var.elasticsearch_version - kms_encryption_enabled = !var.use_ibm_owned_encryption_key - existing_kms_instance_guid = var.existing_kms_instance_guid - kms_key_crn = var.kms_key_crn - backup_crn = var.backup_crn - backup_encryption_key_crn = var.backup_encryption_key_crn - cbr_rules = var.cbr_rules - access_tags = var.access_tags - tags = var.tags - plan = var.plan - members = var.members - member_memory_mb = var.member_memory_mb - admin_pass = var.admin_pass - users = var.users - member_disk_mb = var.member_disk_mb - member_cpu_count = var.member_cpu_count - member_host_flavor = var.member_host_flavor - auto_scaling = var.auto_scaling - service_credential_names = var.service_credential_names - enable_elser_model = var.enable_elser_model - elser_model_type = var.elser_model_type + source = "../../" + resource_group_id = var.resource_group_id + name = var.name + region = var.region + skip_iam_authorization_policy = var.skip_iam_authorization_policy + service_endpoints = "private" + elasticsearch_version = var.elasticsearch_version + kms_encryption_enabled = !var.use_ibm_owned_encryption_key + existing_kms_instance_guid = var.existing_kms_instance_guid + kms_key_crn = var.kms_key_crn + backup_crn = var.backup_crn + backup_encryption_key_crn = var.backup_encryption_key_crn + use_custom_backup_encryption_key = var.use_custom_backup_encryption_key + cbr_rules = var.cbr_rules + access_tags = var.access_tags + tags = var.tags + plan = var.plan + members = var.members + member_memory_mb = var.member_memory_mb + admin_pass = var.admin_pass + users = var.users + member_disk_mb = var.member_disk_mb + member_cpu_count = var.member_cpu_count + member_host_flavor = var.member_host_flavor + auto_scaling = var.auto_scaling + service_credential_names = var.service_credential_names + enable_elser_model = var.enable_elser_model + elser_model_type = var.elser_model_type } diff --git a/modules/fscloud/variables.tf b/modules/fscloud/variables.tf index d6d0bc3c..87d81f67 100644 --- a/modules/fscloud/variables.tf +++ b/modules/fscloud/variables.tf @@ -148,6 +148,12 @@ variable "kms_key_crn" { default = null } +variable "use_custom_backup_encryption_key" { + type = bool + description = "Whether to use a custom IBM Cloud Databases generated key for backup encryption." + default = false +} + variable "backup_encryption_key_crn" { type = string description = "The Hyper Protect Crypto Services (HPCS) or Key Protect root key CRN to use for encrypting the disk that holds deployment backups. There are region limitations for backup encryption. See https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups (HPCS) and https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok (Key Protect)." diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index 5af0ec1f..cf158f1d 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -198,33 +198,34 @@ module "backup_kms" { ####################################################################################################################### module "elasticsearch" { - count = local.use_existing_db_instance ? 0 : 1 - source = "../../modules/fscloud" - depends_on = [time_sleep.wait_for_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy] - resource_group_id = module.resource_group.resource_group_id - name = var.prefix != null ? "${var.prefix}-${var.name}" : var.name - region = var.region - plan = var.plan - skip_iam_authorization_policy = var.skip_iam_authorization_policy || local.create_cross_account_auth_policy - elasticsearch_version = var.elasticsearch_version - existing_kms_instance_guid = local.existing_kms_instance_guid - use_ibm_owned_encryption_key = var.use_ibm_owned_encryption_key - kms_key_crn = local.kms_key_crn - backup_encryption_key_crn = local.backup_kms_key_crn - backup_crn = var.backup_crn - access_tags = var.access_tags - tags = var.tags - admin_pass = local.admin_pass - users = var.users - members = var.members - member_host_flavor = var.member_host_flavor - member_memory_mb = var.member_memory_mb - member_disk_mb = var.member_disk_mb - member_cpu_count = var.member_cpu_count - auto_scaling = var.auto_scaling - service_credential_names = var.service_credential_names - enable_elser_model = var.enable_elser_model - elser_model_type = var.elser_model_type + count = local.use_existing_db_instance ? 0 : 1 + source = "../../modules/fscloud" + depends_on = [time_sleep.wait_for_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy] + resource_group_id = module.resource_group.resource_group_id + name = var.prefix != null ? "${var.prefix}-${var.name}" : var.name + region = var.region + plan = var.plan + skip_iam_authorization_policy = var.skip_iam_authorization_policy || local.create_cross_account_auth_policy + elasticsearch_version = var.elasticsearch_version + existing_kms_instance_guid = local.existing_kms_instance_guid + use_ibm_owned_encryption_key = var.use_ibm_owned_encryption_key + kms_key_crn = local.kms_key_crn + backup_encryption_key_crn = local.backup_kms_key_crn + use_custom_backup_encryption_key = var.use_custom_backup_encryption_key + backup_crn = var.backup_crn + access_tags = var.access_tags + tags = var.tags + admin_pass = local.admin_pass + users = var.users + members = var.members + member_host_flavor = var.member_host_flavor + member_memory_mb = var.member_memory_mb + member_disk_mb = var.member_disk_mb + member_cpu_count = var.member_cpu_count + auto_scaling = var.auto_scaling + service_credential_names = var.service_credential_names + enable_elser_model = var.enable_elser_model + elser_model_type = var.elser_model_type } resource "random_password" "admin_password" { diff --git a/solutions/standard/variables.tf b/solutions/standard/variables.tf index b169b2f0..bb44b33d 100644 --- a/solutions/standard/variables.tf +++ b/solutions/standard/variables.tf @@ -375,3 +375,9 @@ variable "existing_backup_kms_instance_crn" { type = string default = null } + +variable "use_custom_backup_encryption_key" { + type = bool + description = "Whether to use a custom IBM Cloud Databases generated key for backup encryption." + default = false +} diff --git a/tests/pr_test.go b/tests/pr_test.go index 727e210f..6b546ea9 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -61,10 +61,8 @@ func TestRunFSCloudExample(t *testing.T) { */ //ResourceGroup: resourceGroup, TerraformVars: map[string]interface{}{ - "elasticsearch_version": latestVersion, // Always lock this test into the latest supported elasticsearch version - "access_tags": permanentResources["accessTags"], - "existing_kms_instance_guid": permanentResources["hpcs_south"], - "kms_key_crn": permanentResources["hpcs_south_root_key_crn"], + "elasticsearch_version": latestVersion, // Always lock this test into the latest supported elasticsearch version + "access_tags": permanentResources["accessTags"], }, CloudInfoService: sharedInfoSvc, }) @@ -122,6 +120,7 @@ func TestRunStandardSolutionSchematics(t *testing.T) { {Name: "access_tags", Value: permanentResources["accessTags"], DataType: "list(string)"}, {Name: "existing_kms_instance_crn", Value: permanentResources["hpcs_south_crn"], DataType: "string"}, {Name: "existing_backup_kms_key_crn", Value: permanentResources["hpcs_south_root_key_crn"], DataType: "string"}, + {Name: "use_custom_backup_encryption_key", Value: true, DataType: "bool"}, {Name: "kms_endpoint_type", Value: "public", DataType: "string"}, {Name: "resource_group_name", Value: options.Prefix, DataType: "string"}, {Name: "plan", Value: "platinum", DataType: "string"}, diff --git a/variables.tf b/variables.tf index 7a6e3184..2385edf7 100644 --- a/variables.tf +++ b/variables.tf @@ -188,6 +188,12 @@ variable "use_default_backup_encryption_key" { default = false } +variable "use_custom_backup_encryption_key" { + type = bool + description = "Whether to use a custom IBM Cloud Databases generated key for backup encryption." + default = false +} + variable "kms_key_crn" { type = string description = "The root key CRN of the Key Protect or Hyper Protect Crypto Services instance to use for disk encryption. Applies only if `kms_encryption_enabled` is true."