diff --git a/.secrets.baseline b/.secrets.baseline index b317401d..e3ea8dfc 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "go.sum|^.secrets.baseline$", "lines": null }, - "generated_at": "2024-08-07T07:25:45Z", + "generated_at": "2024-12-09T17:04:45Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -82,7 +82,7 @@ "hashed_secret": "33da8d0e8af2efc260f01d8e5edfcc5c5aba44ad", "is_secret": true, "is_verified": false, - "line_number": 36, + "line_number": 35, "type": "Secret Keyword", "verified_result": null } diff --git a/README.md b/README.md index 721dddca..b396e37d 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,6 @@ * [Contributing](#contributing) - This module implements an instance of the IBM Cloud Databases for Elasticsearch service. ### Usage @@ -68,7 +67,9 @@ You need the following permissions to run this module. | Name | Source | Version | |------|--------|---------| +| [backup\_key\_crn\_parser](#module\_backup\_key\_crn\_parser) | terraform-ibm-modules/common-utilities/ibm//modules/crn-parser | 1.1.0 | | [cbr\_rule](#module\_cbr\_rule) | terraform-ibm-modules/cbr/ibm//modules/cbr-rule-module | 1.29.0 | +| [kms\_key\_crn\_parser](#module\_kms\_key\_crn\_parser) | terraform-ibm-modules/common-utilities/ibm//modules/crn-parser | 1.1.0 | ### Resources @@ -93,14 +94,12 @@ You need the following permissions to run this module. | [admin\_pass](#input\_admin\_pass) | The password for the database administrator. If the admin password is null, the admin user ID cannot be accessed. You can specify more users in a user block. | `string` | `null` | no | | [auto\_scaling](#input\_auto\_scaling) | The rules to allow the database to increase resources in response to usage. Only a single autoscaling block is allowed. Make sure you understand the effects of autoscaling, especially for production environments. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-autoscaling&interface=cli#autoscaling-considerations). | 
object({
    disk = object({
      capacity_enabled             = optional(bool, false)
      free_space_less_than_percent = optional(number, 10)
      io_above_percent             = optional(number, 90)
      io_enabled                   = optional(bool, false)
      io_over_period               = optional(string, "15m")
      rate_increase_percent        = optional(number, 10)
      rate_limit_mb_per_member     = optional(number, 3670016)
      rate_period_seconds          = optional(number, 900)
      rate_units                   = optional(string, "mb")
    })
    memory = object({
      io_above_percent         = optional(number, 90)
      io_enabled               = optional(bool, false)
      io_over_period           = optional(string, "15m")
      rate_increase_percent    = optional(number, 10)
      rate_limit_mb_per_member = optional(number, 114688)
      rate_period_seconds      = optional(number, 900)
      rate_units               = optional(string, "mb")
    })
  })
| `null` | no | | [backup\_crn](#input\_backup\_crn) | The CRN of a backup resource to restore from. The backup is created by a database deployment with the same service ID. The backup is loaded after both provisioning is complete and the new deployment that uses that data starts. Specify a backup CRN is in the format `crn:v1:<...>:backup:`. If not specified, the database is provisioned empty. | `string` | `null` | no | -| [backup\_encryption\_key\_crn](#input\_backup\_encryption\_key\_crn) | The CRN of a Hyper Protect Crypto Services use for encrypting the disk that holds deployment backups. There are limitation per region on the Hyper Protect Crypto Services and region for those services. See https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups | `string` | `null` | no | +| [backup\_encryption\_key\_crn](#input\_backup\_encryption\_key\_crn) | The CRN of a Key Protect or Hyper Protect Crypto Services encryption key that you want to use for encrypting the disk that holds deployment backups. Applies only if `use_ibm_owned_encryption_key` is false and `use_same_kms_key_for_backups` is false. If no value is passed, and `use_same_kms_key_for_backups` is true, the value of `kms_key_crn` is used. Alternatively set `use_default_backup_encryption_key` to true to use the IBM Cloud Databases default encryption. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups). | `string` | `null` | no | | [cbr\_rules](#input\_cbr\_rules) | The list of context-based restriction rules to create. | 
list(object({
    description = string
    account_id  = string
    rule_contexts = list(object({
      attributes = optional(list(object({
        name  = string
        value = string
    }))) }))
    enforcement_mode = string
  }))
| `[]` | no | | [elasticsearch\_version](#input\_elasticsearch\_version) | The version of Databases for Elasticsearch to deploy. Possible values: `8.7`, `8.10`, `8.12`, `8.15` which requires an Enterprise Platinum pricing plan. If no value is specified, the current preferred version for IBM Cloud Databases is used. | `string` | `null` | no | | [elser\_model\_type](#input\_elser\_model\_type) | Trained ELSER model to be used for Elastic's Natural Language Processing. Possible values: `.elser_model_1`, `.elser_model_2` and `.elser_model_2_linux-x86_64`. [Learn more](https://www.elastic.co/guide/en/machine-learning/current/ml-nlp-elser.html) | `string` | `".elser_model_2_linux-x86_64"` | no | | [enable\_elser\_model](#input\_enable\_elser\_model) | Set it to true to install and start the Elastic's Natural Language Processing model. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-elser-embeddings-elasticsearch) | `bool` | `false` | no | -| [existing\_kms\_instance\_guid](#input\_existing\_kms\_instance\_guid) | The GUID of a Hyper Protect Crypto Services or Key Protect instance for the CRN specified in `kms_key_crn` and `backup_encryption_key_crn`. Applies only if `kms_encryption_enabled` is true, `skip_iam_authorization_policy` is false, and you specify values for `kms_key_crn` or `backup_encryption_key_crn`. | `string` | `null` | no | -| [kms\_encryption\_enabled](#input\_kms\_encryption\_enabled) | Whether to specify the keys used to encrypt data in the database. Specify `true` to identify the encryption keys. If set to `false`, the data is encrypted with randomly generated keys. [Learn more about Key Protect integration](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect). [Learn more about HPCS integration](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs). | `bool` | `false` | no | -| [kms\_key\_crn](#input\_kms\_key\_crn) | The root key CRN of the Key Protect or Hyper Protect Crypto Services instance to use for disk encryption. Applies only if `kms_encryption_enabled` is true. | `string` | `null` | no | +| [kms\_key\_crn](#input\_kms\_key\_crn) | The CRN of a Key Protect or Hyper Protect Crypto Services encryption key to encrypt your data. Applies only if `use_ibm_owned_encryption_key` is false. By default this key is used for both deployment data and backups, but this behaviour can be altered using the `use_same_kms_key_for_backups` and `backup_encryption_key_crn` inputs. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups). | `string` | `null` | no | | [member\_cpu\_count](#input\_member\_cpu\_count) | The dedicated CPU per member that is allocated. For shared CPU, set to 0. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling). | `number` | `0` | no | | [member\_disk\_mb](#input\_member\_disk\_mb) | The disk that is allocated per member. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling). | `number` | `5120` | no | | [member\_host\_flavor](#input\_member\_host\_flavor) | The host flavor per member. [Learn more](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/database#host_flavor). | `string` | `null` | no | @@ -112,9 +111,11 @@ You need the following permissions to run this module. | [resource\_group\_id](#input\_resource\_group\_id) | The resource group ID where the Databases for Elasticsearch instance is created. | `string` | n/a | yes | | [service\_credential\_names](#input\_service\_credential\_names) | The map of name and role for service credentials that you want to create for the database. | `map(string)` | `{}` | no | | [service\_endpoints](#input\_service\_endpoints) | The type of endpoint of the database instance. Possible values: `public`, `private`, `public-and-private`. | `string` | `"public"` | no | -| [skip\_iam\_authorization\_policy](#input\_skip\_iam\_authorization\_policy) | Whether to create an IAM authorization policy that permits all Databases for Elasticsearch instances in the resource group to read the encryption key from the Hyper Protect Crypto Services instance specified in the `existing_kms_instance_guid` variable. If set to `false`, specify a value for the KMS instance in the `existing_kms_instance_guid` variable. No policy is created if `kms_encryption_enabled` is false. | `bool` | `false` | no | +| [skip\_iam\_authorization\_policy](#input\_skip\_iam\_authorization\_policy) | Set to true to skip the creation of IAM authorization policies that permits all Databases for Elasticsearch instances in the given resource group 'Reader' access to the Key Protect or Hyper Protect Crypto Services key that was provided in the `kms_key_crn` and `backup_encryption_key_crn` inputs. This policy is required in order to enable KMS encryption, so only skip creation if there is one already present in your account. No policy is created if `use_ibm_owned_encryption_key` is true. | `bool` | `false` | no | | [tags](#input\_tags) | The list of tags to be added to the Databases for Elasticsearch instance. | `list(string)` | `[]` | no | -| [use\_default\_backup\_encryption\_key](#input\_use\_default\_backup\_encryption\_key) | Whether to use the IBM Cloud Databases generated keys for backup encryption. | `bool` | `false` | no | +| [use\_default\_backup\_encryption\_key](#input\_use\_default\_backup\_encryption\_key) | When `use_ibm_owned_encryption_key` is set to false, backups will be encrypted with either the key specified in `kms_key_crn`, or in `backup_encryption_key_crn` if a value is passed. If you do not want to use your own key for backups encryption, you can set this to `true` to use the IBM Cloud Databases default encryption for backups. Alternatively set `use_ibm_owned_encryption_key` to true to use the default encryption for both backups and deployment data. | `bool` | `false` | no | +| [use\_ibm\_owned\_encryption\_key](#input\_use\_ibm\_owned\_encryption\_key) | IBM Cloud Databases will secure your deployment's data at rest automatically with an encryption key that IBM hold. Alternatively, you may select your own Key Management System instance and encryption key (Key Protect or Hyper Protect Crypto Services) by setting this to false. If setting to false, a value must be passed for the `kms_key_crn` input. | `bool` | `true` | no | +| [use\_same\_kms\_key\_for\_backups](#input\_use\_same\_kms\_key\_for\_backups) | Set this to false if you wan't to use a different key that you own to encrypt backups. When set to false, a value is required for the `backup_encryption_key_crn` input. Alternatiely set `use_default_backup_encryption_key` to true to use the IBM Cloud Databases default encryption. Applies only if `use_ibm_owned_encryption_key` is false. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups). | `bool` | `true` | no | | [users](#input\_users) | The list of users that have access to the database. Multiple blocks are allowed. The user password must be 10-32 characters. In most cases, you can use IAM service credentials (by specifying `service_credential_names`) to control access to the database instance. This block creates native database users. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-user-management&interface=ui). | 
list(object({
    name     = string
    password = string # pragma: allowlist secret
    type     = optional(string)
    role     = optional(string)
  }))
| `[]` | no | ### Outputs diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 70fadd72..b77c2b42 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -14,35 +14,38 @@ module "resource_group" { ############################################################################## module "icd_elasticsearch" { - source = "../../" - resource_group_id = module.resource_group.resource_group_id - name = "${var.prefix}-elasticsearch" - region = var.region - elasticsearch_version = var.elasticsearch_version - tags = var.resource_tags - access_tags = var.access_tags - service_credential_names = var.service_credential_names + source = "../../" + resource_group_id = module.resource_group.resource_group_id + name = "${var.prefix}-elasticsearch" + region = var.region + elasticsearch_version = var.elasticsearch_version + tags = var.resource_tags + access_tags = var.access_tags + service_credential_names = { + "elasticsearch_admin" : "Administrator", + "elasticsearch_operator" : "Operator", + "elasticsearch_viewer" : "Viewer", + "elasticsearch_editor" : "Editor", + } } -# wait 15 secs to allow IAM credential access to kick in before configuring instance +# wait 60 secs to allow IAM credential access to kick in before configuring instance # without the wait, you can intermittently get "Error 401 (Unauthorized)" resource "time_sleep" "wait" { depends_on = [module.icd_elasticsearch] - create_duration = "15s" + create_duration = "60s" } -# Commenting below code to this issue https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/issues/317 - -# resource "elasticsearch_index" "test" { -# depends_on = [time_sleep.wait] -# name = "terraform-test" -# number_of_shards = 1 -# number_of_replicas = 1 -# force_destroy = true -# } +resource "elasticsearch_index" "test" { + depends_on = [time_sleep.wait] + name = "terraform-test" + number_of_shards = 1 + number_of_replicas = 1 + force_destroy = true +} -# resource "elasticsearch_cluster_settings" "global" { -# depends_on = [time_sleep.wait] -# cluster_max_shards_per_node = 10 -# action_auto_create_index = "my-index-000001,index10,-index1*,+ind*" -# } +resource "elasticsearch_cluster_settings" "global" { + depends_on = [time_sleep.wait] + cluster_max_shards_per_node = 10 + action_auto_create_index = "my-index-000001,index10,-index1*,+ind*" +} diff --git a/examples/basic/variables.tf b/examples/basic/variables.tf index 30cf2b15..dd829012 100644 --- a/examples/basic/variables.tf +++ b/examples/basic/variables.tf @@ -39,14 +39,3 @@ variable "resource_tags" { description = "Optional list of tags to be added to created resources" default = [] } - -variable "service_credential_names" { - description = "Map of name, role for service credentials that you want to create for the database" - type = map(string) - default = { - "elasticsearch_admin" : "Administrator", - "elasticsearch_operator" : "Operator", - "elasticsearch_viewer" : "Viewer", - "elasticsearch_editor" : "Editor", - } -} diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 913c54cd..71612679 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -1,6 +1,16 @@ +############################################################################## +# Locals +############################################################################## + locals { - sm_guid = var.existing_sm_instance_guid == null ? module.secrets_manager.secrets_manager_guid : var.existing_sm_instance_guid + sm_guid = var.existing_sm_instance_guid == null ? module.secrets_manager[0].secrets_manager_guid : var.existing_sm_instance_guid sm_region = var.existing_sm_instance_region == null ? var.region : var.existing_sm_instance_region + service_credential_names = { + "es_admin" : "Administrator", + "es_operator" : "Operator", + "es_viewer" : "Viewer", + "es_editor" : "Editor", + } } ############################################################################## @@ -18,11 +28,16 @@ module "resource_group" { # Key Protect All Inclusive ############################################################################## +locals { + data_key_name = "${var.prefix}-elasticsearch" + backups_key_name = "${var.prefix}-elasticsearch-backups" +} + module "key_protect_all_inclusive" { source = "terraform-ibm-modules/kms-all-inclusive/ibm" version = "4.17.1" resource_group_id = module.resource_group.resource_group_id - # Only us-south, eu-de backup encryption keys are supported. See https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok for details. + # Only us-south, us-east and eu-de backup encryption keys are supported. See https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok for details. # Note: Database instance and Key Protect must be created on the same region. region = var.region key_protect_instance_name = "${var.prefix}-kp" @@ -32,7 +47,11 @@ module "key_protect_all_inclusive" { key_ring_name = "icd" keys = [ { - key_name = "${var.prefix}-elasticsearch" + key_name = local.data_key_name + force_delete = true + }, + { + key_name = local.backups_key_name force_delete = true } ] @@ -45,23 +64,26 @@ module "key_protect_all_inclusive" { ############################################################################## module "icd_elasticsearch" { - source = "../../" - resource_group_id = module.resource_group.resource_group_id - name = "${var.prefix}-elasticsearch" - region = var.region - plan = var.plan - kms_encryption_enabled = true - access_tags = var.access_tags - admin_pass = var.admin_pass - users = var.users - existing_kms_instance_guid = module.key_protect_all_inclusive.kms_guid - service_credential_names = var.service_credential_names - elasticsearch_version = var.elasticsearch_version - kms_key_crn = module.key_protect_all_inclusive.keys["icd.${var.prefix}-elasticsearch"].crn - tags = var.resource_tags - auto_scaling = var.auto_scaling - member_host_flavor = "multitenant" - member_memory_mb = 4096 + source = "../../" + resource_group_id = module.resource_group.resource_group_id + name = "${var.prefix}-elasticsearch" + region = var.region + plan = var.plan + access_tags = var.access_tags + admin_pass = var.admin_pass + users = var.users + service_credential_names = local.service_credential_names + elasticsearch_version = var.elasticsearch_version + tags = var.resource_tags + auto_scaling = var.auto_scaling + member_host_flavor = "multitenant" + member_memory_mb = 4096 + + # Example of how to use different KMS keys for data and backups + use_ibm_owned_encryption_key = false + use_same_kms_key_for_backups = false + kms_key_crn = module.key_protect_all_inclusive.keys["icd.${local.backups_key_name}"].crn + backup_encryption_key_crn = module.key_protect_all_inclusive.keys["icd.${local.data_key_name}"].crn } @@ -71,6 +93,7 @@ module "icd_elasticsearch" { # Create Secrets Manager Instance (if not using existing one) module "secrets_manager" { + count = var.existing_sm_instance_guid == null ? 1 : 0 source = "terraform-ibm-modules/secrets-manager/ibm" version = "1.19.2" resource_group_id = module.resource_group.resource_group_id @@ -96,7 +119,7 @@ module "secrets_manager_secrets_group" { module "secrets_manager_service_credentials_user_pass" { source = "terraform-ibm-modules/secrets-manager-secret/ibm" version = "1.4.0" - for_each = var.service_credential_names + for_each = local.service_credential_names region = local.sm_region secrets_manager_guid = local.sm_guid secret_group_id = module.secrets_manager_secrets_group.secret_group_id @@ -107,7 +130,7 @@ module "secrets_manager_service_credentials_user_pass" { secret_type = "username_password" #checkov:skip=CKV_SECRET_6 } -# Add secrets manager certificate to secret manager as a certificate secret type in the created secret group +# Add Elasticsearch certificate to secret manager as a certificate secret type in the created secret group module "secrets_manager_service_credentials_cert" { source = "terraform-ibm-modules/secrets-manager-secret/ibm" version = "1.4.0" diff --git a/examples/complete/variables.tf b/examples/complete/variables.tf index cd2f2283..779688d8 100644 --- a/examples/complete/variables.tf +++ b/examples/complete/variables.tf @@ -58,17 +58,6 @@ variable "existing_sm_instance_region" { default = null } -variable "service_credential_names" { - description = "Map of name, role for service credentials that you want to create for the database" - type = map(string) - default = { - "es_admin" : "Administrator", - "es_operator" : "Operator", - "es_viewer" : "Viewer", - "es_editor" : "Editor", - } -} - variable "admin_pass" { type = string default = null diff --git a/examples/fscloud/main.tf b/examples/fscloud/main.tf index 888bf044..5f5e4e56 100644 --- a/examples/fscloud/main.tf +++ b/examples/fscloud/main.tf @@ -54,21 +54,25 @@ module "cbr_zone" { ############################################################################## module "elasticsearch" { - source = "../../modules/fscloud" - resource_group_id = module.resource_group.resource_group_id - name = "${var.prefix}-elasticsearch" - region = var.region - tags = var.resource_tags - access_tags = var.access_tags - kms_key_crn = var.kms_key_crn - existing_kms_instance_guid = var.existing_kms_instance_guid - elasticsearch_version = var.elasticsearch_version - service_credential_names = var.service_credential_names - auto_scaling = var.auto_scaling - member_host_flavor = "b3c.4x16.encrypted" - backup_encryption_key_crn = var.backup_encryption_key_crn - backup_crn = var.backup_crn - enable_elser_model = var.enable_elser_model + source = "../../modules/fscloud" + resource_group_id = module.resource_group.resource_group_id + name = "${var.prefix}-elasticsearch" + region = var.region + tags = var.resource_tags + access_tags = var.access_tags + kms_key_crn = var.kms_key_crn + elasticsearch_version = var.elasticsearch_version + service_credential_names = { + "elasticsearch_admin" : "Administrator", + "elasticsearch_operator" : "Operator", + "elasticsearch_viewer" : "Viewer", + "elasticsearch_editor" : "Editor", + } + auto_scaling = var.auto_scaling + member_host_flavor = "b3c.4x16.encrypted" + backup_encryption_key_crn = var.backup_encryption_key_crn + backup_crn = var.backup_crn + enable_elser_model = var.enable_elser_model cbr_rules = [ { description = "${var.prefix}-elasticsearch access only from vpc" diff --git a/examples/fscloud/variables.tf b/examples/fscloud/variables.tf index 664994c7..ddebbb75 100644 --- a/examples/fscloud/variables.tf +++ b/examples/fscloud/variables.tf @@ -34,11 +34,6 @@ variable "access_tags" { default = [] } -variable "existing_kms_instance_guid" { - description = "The GUID of the Hyper Protect Crypto services in which the key specified in var.kms_key_crn is coming from" - type = string -} - variable "kms_key_crn" { type = string description = "The root key CRN of a Hyper Protect Crypto Services (HPCS) that you want to use for disk encryption. See https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs&interface=ui for more information on integrating HPCS with Elasticsearch instance." @@ -50,17 +45,6 @@ variable "elasticsearch_version" { default = null } -variable "service_credential_names" { - description = "Map of name, role for service credentials that you want to create for the database" - type = map(string) - default = { - "elasticsearch_admin" : "Administrator", - "elasticsearch_operator" : "Operator", - "elasticsearch_viewer" : "Viewer", - "elasticsearch_editor" : "Editor", - } -} - variable "auto_scaling" { type = object({ disk = object({ diff --git a/ibm_catalog.json b/ibm_catalog.json index bfc04fd4..19c0f4c5 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -1,319 +1,356 @@ { - "products": [ - { - "name": "deploy-arch-ibm-icd-elasticsearch", - "label": "Cloud automation for Databases for Elasticsearch", - "product_kind": "solution", - "tags": [ - "ibm_created", - "target_terraform", - "terraform", - "data_management", - "solution" - ], - "keywords": [ - "elasticsearch", - "IaC", - "infrastructure as code", - "terraform", - "solution", - "elasticsearch standard", - "database", - "nosql" - ], - "short_description": "Creates and configures an instance of IBM Cloud Databases for Elasticsearch.", - "long_description": "This architecture supports creating and configuring an instance of Databases for Elasticsearch with KMS encryption.", - "offering_docs_url": "https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/README.md", - "offering_icon_url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/main/images/elasticsearch_icon.svg", - "provider_name": "IBM", - "features": [ - { - "title": "Creates an instance of Databases for Elasticsearch", - "description": "Creates and configures an IBM Cloud Databases for Elasticsearch instance." - }, - { - "title": "Supports KMS encryption", - "description": "Provides KMS encryption for the data that you store in the database." - }, - { - "title": "Supports autoscaling", - "description": "Provides the autoscaling to allow the database to increase resources in response to usage." - }, - { - "title": "Attaches access tags", - "description": "Attaches access tags to the Elasticsearch instance." + "products": [ + { + "name": "deploy-arch-ibm-icd-elasticsearch", + "label": "Cloud automation for Databases for Elasticsearch", + "product_kind": "solution", + "tags": [ + "ibm_created", + "target_terraform", + "terraform", + "data_management", + "solution" + ], + "keywords": [ + "elasticsearch", + "IaC", + "infrastructure as code", + "terraform", + "solution", + "elasticsearch standard", + "database", + "nosql" + ], + "short_description": "Creates and configures an instance of IBM Cloud Databases for Elasticsearch.", + "long_description": "This architecture supports creating and configuring an instance of Databases for Elasticsearch with KMS encryption.", + "offering_docs_url": "https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/README.md", + "offering_icon_url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/main/images/elasticsearch_icon.svg", + "provider_name": "IBM", + "features": [ + { + "title": "Creates an instance of Databases for Elasticsearch", + "description": "Creates and configures an IBM Cloud Databases for Elasticsearch instance." + }, + { + "title": "Supports KMS encryption", + "description": "Provides KMS encryption for the data that you store in the database." + }, + { + "title": "Supports autoscaling", + "description": "Provides the autoscaling to allow the database to increase resources in response to usage." + }, + { + "title": "Attaches access tags", + "description": "Attaches access tags to the Elasticsearch instance." + }, + { + "title": "Supports backup restoration", + "description": "Provides database restoration using a backup created by a deployment with the same service ID." + } + ], + "flavors": [ + { + "label": "Standard", + "name": "standard", + "install_type": "fullstack", + "working_directory": "solutions/standard", + "compliance": { + "authority": "scc-v3", + "profiles": [ + { + "profile_name": "IBM Cloud Framework for Financial Services", + "profile_version": "1.7.0" + } + ] }, - { - "title": "Supports backup restoration", - "description": "Provides database restoration using a backup created by a deployment with the same service ID." - } - ], - "flavors": [ - { - "label": "Standard", - "name": "standard", - "install_type": "fullstack", - "working_directory": "solutions/standard", - "compliance": { - "authority": "scc-v3", - "profiles": [ - { - "profile_name": "IBM Cloud Framework for Financial Services", - "profile_version": "1.7.0" - } - ]}, - "iam_permissions": [ + "iam_permissions": [ + { + "role_crns": [ + "crn:v1:bluemix:public:iam::::role:Editor" + ], + "service_name": "databases-for-elasticsearch" + } + ], + "architecture": { + "descriptions": "This architecture creates an instance of IBM CLoud Databases for Elasticsearch instance with KMS encryption. Supports autoscaling.", + "features": [ { - "role_crns": [ - "crn:v1:bluemix:public:iam::::role:Editor" - ], - "service_name": "databases-for-elasticsearch" + "title": " Creates an instance of Databases for Elasticsearch", + "description": "This architecture creates an instance of IBM Cloud Databases for Elasticsearch with KMS encryption. It accepts or creates a resource group, and provides autoscaling rules." } ], - "architecture": { - "descriptions": "This architecture creates an instance of IBM CLoud Databases for Elasticsearch instance with KMS encryption. Supports autoscaling.", - "features": [ - { - "title": " Creates an instance of Databases for Elasticsearch", - "description": "This architecture creates an instance of IBM Cloud Databases for Elasticsearch with KMS encryption. It accepts or creates a resource group, and provides autoscaling rules." - } - ], - "diagrams": [ - { - "diagram": { - "caption": "Databases for Elasticsearch instance on IBM Cloud", - "url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/main/reference-architecture/deployable-architecture-elasticsearch.svg", - "type": "image/svg+xml" - }, - "description": "This architecture supports creating and configuring an instance of Databases for Elasticsearch instance with KMS encryption." - } - ] - }, - "configuration": [ - { - "key": "ibmcloud_api_key" - }, - { - "key": "provider_visibility", - "options": [ - { - "displayname": "private", - "value": "private" - }, - { - "displayname": "public", - "value": "public" - }, - { - "displayname": "public-and-private", - "value": "public-and-private" - } - ] - }, - { - "key": "use_existing_resource_group" - }, - { - "key": "resource_group_name" - }, - { - "key": "prefix" - }, - { - "key": "region", - "required": true, - "default_value": "", - "options": [ - { - "displayname": "Chennai (che01)", - "value": "che01" - }, - { - "displayname": "Dallas (us-south)", - "value": "us-south" - }, - { - "displayname": "Frankfurt (eu-de)", - "value": "eu-de" - }, - { - "displayname": "London (eu-gb)", - "value": "eu-gb" - }, - { - "displayname": "Madrid (eu-es)", - "value": "eu-es" - }, - { - "displayname": "Osaka (jp-osa)", - "value": "jp-osa" - }, - { - "displayname": "Paris (par01)", - "value": "par01" - }, - { - "displayname": "Sao Paulo (br-sao)", - "value": "br-sao" - }, - { - "displayname": "Sydney (au-syd)", - "value": "au-syd" - }, - { - "displayname": "Toronto (ca-tor)", - "value": "ca-tor" - }, - { - "displayname": "Tokyo (jp-tok)", - "value": "jp-tok" - }, - { - "displayname": "Washington (us-east)", - "value": "us-east" - } - ] - }, - { - "key": "name" - }, - { - "key": "plan", - "options": [ - { - "displayname": "enterprise", - "value": "enterprise" - }, - { - "displayname": "platinum", - "value": "platinum" - } - ] + "diagrams": [ + { + "diagram": { + "caption": "Databases for Elasticsearch instance on IBM Cloud", + "url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/main/reference-architecture/deployable-architecture-elasticsearch.svg", + "type": "image/svg+xml" }, + "description": "This architecture supports creating and configuring an instance of Databases for Elasticsearch instance with KMS encryption." + } + ] + }, + "configuration": [ + { + "key": "ibmcloud_api_key" + }, + { + "key": "provider_visibility", + "options": [ { - "key": "elasticsearch_version", - "required": true, - "options": [ - { - "displayname": "8.15", - "value": "8.15" - }, - { - "displayname": "8.12", - "value": "8.12" - }, - { - "displayname": "8.10", - "value": "8.10" - } - ] + "displayname": "private", + "value": "private" }, { - "key": "access_tags" + "displayname": "public", + "value": "public" }, { - "key": "use_ibm_owned_encryption_key" - }, + "displayname": "public-and-private", + "value": "public-and-private" + } + ] + }, + { + "key": "use_existing_resource_group" + }, + { + "key": "resource_group_name" + }, + { + "key": "prefix" + }, + { + "key": "region", + "required": true, + "options": [ { - "key": "tags" + "displayname": "Chennai (che01)", + "value": "che01" }, { - "key": "users" + "displayname": "Dallas (us-south)", + "value": "us-south" }, { - "key": "members" + "displayname": "Frankfurt (eu-de)", + "value": "eu-de" }, { - "key": "member_memory_mb" + "displayname": "London (eu-gb)", + "value": "eu-gb" }, { - "key": "member_cpu_count" + "displayname": "Madrid (eu-es)", + "value": "eu-es" }, { - "key": "member_disk_mb" + "displayname": "Osaka (jp-osa)", + "value": "jp-osa" }, { - "key": "member_host_flavor" + "displayname": "Paris (par01)", + "value": "par01" }, { - "key": "service_credential_names" + "displayname": "Sao Paulo (br-sao)", + "value": "br-sao" }, { - "key": "admin_pass" + "displayname": "Sydney (au-syd)", + "value": "au-syd" }, { - "key": "admin_pass_sm_secret_group" + "displayname": "Toronto (ca-tor)", + "value": "ca-tor" }, { - "key": "use_existing_admin_pass_sm_secret_group" + "displayname": "Tokyo (jp-tok)", + "value": "jp-tok" }, { - "key": "admin_pass_sm_secret_name" - }, + "displayname": "Washington (us-east)", + "value": "us-east" + } + ] + }, + { + "key": "name" + }, + { + "key": "existing_db_instance_crn" + }, + { + "key": "plan", + "options": [ { - "key": "skip_iam_authorization_policy" + "displayname": "enterprise", + "value": "enterprise" }, { - "key": "kms_endpoint_type", - "options": [ - { - "displayname": "public", - "value": "public" - }, - { - "displayname": "private", - "value": "private" - } - ] - }, + "displayname": "platinum", + "value": "platinum" + } + ] + }, + { + "key": "elasticsearch_version", + "required": true, + "options": [ { - "key": "existing_kms_key_crn" + "displayname": "8.15", + "value": "8.15" }, { - "key": "existing_kms_instance_crn", - "required": true + "displayname": "8.12", + "value": "8.12" }, { - "key": "elasticsearch_key_ring_name" - }, + "displayname": "8.10", + "value": "8.10" + } + ] + }, + { + "key": "tags" + }, + { + "key": "access_tags" + }, + { + "key": "auto_scaling" + }, + { + "key": "members" + }, + { + "key": "member_memory_mb" + }, + { + "key": "member_cpu_count" + }, + { + "key": "member_disk_mb" + }, + { + "key": "member_host_flavor" + }, + { + "key": "admin_pass" + }, + { + "key": "users" + }, + { + "key": "service_credential_names" + }, + { + "key": "existing_secrets_manager_instance_crn" + }, + { + "key": "existing_secrets_manager_endpoint_type", + "options": [ { - "key": "elasticsearch_key_name" + "displayname": "public", + "value": "public" }, { - "key": "auto_scaling" - }, + "displayname": "private", + "value": "private" + } + ] + }, + { + "key": "service_credential_secrets" + }, + { + "key": "admin_pass_sm_secret_group" + }, + { + "key": "use_existing_admin_pass_sm_secret_group" + }, + { + "key": "admin_pass_sm_secret_name" + }, + { + "key": "skip_es_sm_auth_policy" + }, + { + "key": "ibmcloud_kms_api_key" + }, + { + "key": "kms_endpoint_type", + "options": [ { - "key": "backup_crn" + "displayname": "public", + "value": "public" }, { - "key": "existing_backup_kms_key_crn" - }, + "displayname": "private", + "value": "private" + } + ] + }, + { + "key": "use_ibm_owned_encryption_key" + }, + { + "key": "existing_kms_instance_crn", + "required": true + }, + { + "key": "existing_kms_key_crn" + }, + { + "key": "existing_backup_kms_key_crn" + }, + { + "key": "use_default_backup_encryption_key" + }, + { + "key": "elasticsearch_key_ring_name" + }, + { + "key": "elasticsearch_key_name" + }, + { + "key": "skip_es_kms_auth_policy" + }, + { + "key": "backup_crn" + }, + { + "key": "enable_elser_model" + }, + { + "key": "elser_model_type", + "options": [ { - "key": "existing_backup_kms_instance_crn" + "displayname": ".elser_model_1", + "value": ".elser_model_1" }, { - "key": "enable_elser_model" + "displayname": ".elser_model_2", + "value": ".elser_model_2" }, { - "key": "elser_model_type", - "options": [ - { - "displayname": ".elser_model_1", - "value": ".elser_model_1" - }, - { - "displayname": ".elser_model_2", - "value": ".elser_model_2" - }, - { - "displayname": ".elser_model_2_linux-x86_64", - "value": ".elser_model_2_linux-x86_64" - } - ] + "displayname": ".elser_model_2_linux-x86_64", + "value": ".elser_model_2_linux-x86_64" } ] - } - ] - } - ] - } + }, + { + "key": "enable_kibana_dashboard" + }, + { + "key": "existing_code_engine_project_id" + }, + { + "key": "elasticsearch_full_version" + } + ] + } + ] + } + ] +} diff --git a/main.tf b/main.tf index 6903f08f..af0482f5 100644 --- a/main.tf +++ b/main.tf @@ -1,27 +1,33 @@ +######################################################################################################################## +# Input variable validation +# (approach based on https://github.com/hashicorp/terraform/issues/25609#issuecomment-1057614400) +# +# TODO: Replace with terraform cross variable validation: https://github.ibm.com/GoldenEye/issues/issues/10836 +######################################################################################################################## + locals { - # Validation (approach based on https://github.com/hashicorp/terraform/issues/25609#issuecomment-1057614400) # tflint-ignore: terraform_unused_declarations - validate_kms_values = !var.kms_encryption_enabled && (var.kms_key_crn != null || var.backup_encryption_key_crn != null) ? tobool("When passing values for var.backup_encryption_key_crn or var.kms_key_crn, you must set var.kms_encryption_enabled to true. Otherwise unset them to use default encryption") : true + validate_kms_values = var.use_ibm_owned_encryption_key && (var.kms_key_crn != null || var.backup_encryption_key_crn != null) ? tobool("When passing values for 'kms_key_crn' or 'backup_encryption_key_crn', you must set 'use_ibm_owned_encryption_key' to false. Otherwise unset them to use default encryption.") : true # tflint-ignore: terraform_unused_declarations - validate_kms_vars = var.kms_encryption_enabled && var.kms_key_crn == null && var.backup_encryption_key_crn == null ? tobool("When setting var.kms_encryption_enabled to true, a value must be passed for var.kms_key_crn and/or var.backup_encryption_key_crn") : true + validate_kms_vars = !var.use_ibm_owned_encryption_key && var.kms_key_crn == null ? tobool("When setting 'use_ibm_owned_encryption_key' to false, a value must be passed for 'kms_key_crn'.") : true # tflint-ignore: terraform_unused_declarations - validate_auth_policy = var.kms_encryption_enabled && var.skip_iam_authorization_policy == false && var.existing_kms_instance_guid == null ? tobool("When var.skip_iam_authorization_policy is set to false, and var.kms_encryption_enabled to true, a value must be passed for var.existing_kms_instance_guid in order to create the auth policy.") : true + validate_backup_key = !var.use_ibm_owned_encryption_key && var.backup_encryption_key_crn != null && (var.use_default_backup_encryption_key || var.use_same_kms_key_for_backups) ? tobool("When passing a value for 'backup_encryption_key_crn' you cannot set 'use_default_backup_encryption_key' to true or 'use_ibm_owned_encryption_key' to false.") : true # tflint-ignore: terraform_unused_declarations - validate_backup_key = var.backup_encryption_key_crn != null && var.use_default_backup_encryption_key == true ? tobool("When passing a value for 'backup_encryption_key_crn' you cannot set 'use_default_backup_encryption_key' to 'true'") : true + validate_backup_key_2 = !var.use_ibm_owned_encryption_key && var.backup_encryption_key_crn == null && !var.use_same_kms_key_for_backups ? tobool("When 'use_same_kms_key_for_backups' is set to false, a value needs to be passed for 'backup_encryption_key_crn'.") : true # tflint-ignore: terraform_unused_declarations - validate_plan = var.enable_elser_model && var.plan != "platinum" ? tobool("When var.enable_elser_model is set to true, a value for var.plan must be 'platinum' in order to enable ELSER model.") : true + validate_plan = var.enable_elser_model && var.plan != "platinum" ? tobool("When 'enable_elser_model' is set to true, the 'plan' must be set to 'platinum' in order to enable ELSER model.") : true # tflint-ignore: terraform_unused_declarations - validate_es_user = var.enable_elser_model && !((length(var.service_credential_names) > 0 && length([for k, v in var.service_credential_names : k if v == "Administrator"]) > 0) || var.admin_pass != null) ? tobool("When var.enable_elser_model is set to true, a value must be passed for var.service_credential_names or var.admin_pass. var.service_credential_names must contain at least one credential name with Administrator role.") : true - - # If no value passed for 'backup_encryption_key_crn' use the value of 'kms_key_crn' and perform validation of 'kms_key_crn' to check if region is supported by backup encryption key. - - # For more info, see https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok and https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups" + validate_es_user = var.enable_elser_model && !((length(var.service_credential_names) > 0 && length([for k, v in var.service_credential_names : k if v == "Administrator"]) > 0) || var.admin_pass != null) ? tobool("When 'enable_elser_model' is set to true, an Administrator role user must be created using the 'service_credential_names' input, or by passing a value for the 'admin_pass' input.") : true +} - backup_encryption_key_crn = var.use_default_backup_encryption_key == true ? null : (var.backup_encryption_key_crn != null ? var.backup_encryption_key_crn : var.kms_key_crn) - parsed_backup_encryption_key_crn = local.backup_encryption_key_crn != null ? split(":", local.backup_encryption_key_crn) : [] - backup_kms_key_id = length(local.parsed_backup_encryption_key_crn) > 0 ? local.parsed_backup_encryption_key_crn[9] : null +######################################################################################################################## +# Locals +######################################################################################################################## - create_backup_kms_policy = local.create_kp_auth_policy == 1 && local.backup_encryption_key_crn != null && var.backup_encryption_key_crn != null +locals { + # If 'use_ibm_owned_encryption_key' is true or 'use_default_backup_encryption_key' is true, default to null. + # If no value is passed for 'backup_encryption_key_crn', then default to use 'kms_key_crn'. + backup_encryption_key_crn = var.use_ibm_owned_encryption_key || var.use_default_backup_encryption_key ? null : (var.backup_encryption_key_crn != null ? var.backup_encryption_key_crn : var.kms_key_crn) # Determine if auto scaling is enabled auto_scaling_enabled = var.auto_scaling == null ? [] : [1] @@ -29,22 +35,55 @@ locals { # Determine if host_flavor is used host_flavor_set = var.member_host_flavor != null ? true : false - create_kp_auth_policy = var.kms_encryption_enabled == false || var.skip_iam_authorization_policy ? 0 : 1 +} + +######################################################################################################################## +# Parse info from KMS key CRNs +######################################################################################################################## + +module "kms_key_crn_parser" { + count = var.use_ibm_owned_encryption_key ? 0 : 1 + source = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser" + version = "1.1.0" + crn = var.kms_key_crn +} - parsed_kms_key_crn = var.kms_key_crn != null ? split(":", var.kms_key_crn) : [] - kms_service = length(local.parsed_kms_key_crn) > 0 ? local.parsed_kms_key_crn[4] : null - kms_scope = length(local.parsed_kms_key_crn) > 0 ? local.parsed_kms_key_crn[6] : null - kms_account_id = length(local.parsed_kms_key_crn) > 0 ? split("/", local.kms_scope)[1] : null - kms_key_id = length(local.parsed_kms_key_crn) > 0 ? local.parsed_kms_key_crn[9] : null +module "backup_key_crn_parser" { + count = var.use_ibm_owned_encryption_key ? 0 : 1 + source = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser" + version = "1.1.0" + crn = local.backup_encryption_key_crn +} + +# Put parsed values into locals +locals { + kms_service = !var.use_ibm_owned_encryption_key ? module.kms_key_crn_parser[0].service_name : null + kms_account_id = !var.use_ibm_owned_encryption_key ? module.kms_key_crn_parser[0].account_id : null + kms_key_id = !var.use_ibm_owned_encryption_key ? module.kms_key_crn_parser[0].resource : null + kms_key_instance_guid = !var.use_ibm_owned_encryption_key ? module.kms_key_crn_parser[0].service_instance : null + backup_kms_service = !var.use_ibm_owned_encryption_key ? module.backup_key_crn_parser[0].service_name : null + backup_kms_account_id = !var.use_ibm_owned_encryption_key ? module.backup_key_crn_parser[0].account_id : null + backup_kms_key_id = !var.use_ibm_owned_encryption_key ? module.backup_key_crn_parser[0].resource : null + backup_kms_key_instance_guid = !var.use_ibm_owned_encryption_key ? module.backup_key_crn_parser[0].service_instance : null +} + +######################################################################################################################## +# KMS IAM Authorization Policies +######################################################################################################################## + +locals { + # only create auth policy if 'use_ibm_owned_encryption_key' is false, and 'skip_iam_authorization_policy' is false + create_kms_auth_policy = !var.use_ibm_owned_encryption_key && !var.skip_iam_authorization_policy ? 1 : 0 + # only create backup auth policy if 'use_ibm_owned_encryption_key' is false, 'skip_iam_authorization_policy' is false and 'use_same_kms_key_for_backups' is false + create_backup_kms_auth_policy = !var.use_ibm_owned_encryption_key && !var.skip_iam_authorization_policy && !var.use_same_kms_key_for_backups ? 1 : 0 } -# Create IAM Access Policy to allow Key protect to access Elasticsearch instance resource "ibm_iam_authorization_policy" "policy" { - count = local.create_kp_auth_policy + count = local.create_kms_auth_policy source_service_name = "databases-for-elasticsearch" source_resource_group_id = var.resource_group_id roles = ["Reader"] - description = "Allow all Elastic Search instances in the resource group ${var.resource_group_id} to read the ${local.kms_service} key ${local.kms_key_id} from the instance GUID ${var.existing_kms_instance_guid}" + description = "Allow all Elastic Search instances in the resource group ${var.resource_group_id} to read the ${local.kms_service} key ${local.kms_key_id} from the instance GUID ${local.kms_key_instance_guid}" resource_attributes { name = "serviceName" operator = "stringEquals" @@ -58,7 +97,7 @@ resource "ibm_iam_authorization_policy" "policy" { resource_attributes { name = "serviceInstance" operator = "stringEquals" - value = var.existing_kms_instance_guid + value = local.kms_key_instance_guid } resource_attributes { name = "resourceType" @@ -79,32 +118,31 @@ resource "ibm_iam_authorization_policy" "policy" { # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 resource "time_sleep" "wait_for_authorization_policy" { - count = local.create_kp_auth_policy - depends_on = [ibm_iam_authorization_policy.policy] - + count = local.create_kms_auth_policy + depends_on = [ibm_iam_authorization_policy.policy] create_duration = "30s" } resource "ibm_iam_authorization_policy" "backup_kms_policy" { - count = local.create_backup_kms_policy ? 1 : 0 + count = local.create_backup_kms_auth_policy source_service_name = "databases-for-elasticsearch" source_resource_group_id = var.resource_group_id roles = ["Reader"] - description = "Allow all Elastic Search instances in the Resource Group ${var.resource_group_id} to read the ${local.kms_service} key ${local.backup_kms_key_id} from the instance GUID ${var.existing_kms_instance_guid}" + description = "Allow all Elastic Search instances in the Resource Group ${var.resource_group_id} to read the ${local.backup_kms_service} key ${local.backup_kms_key_id} from the instance GUID ${local.backup_kms_key_instance_guid}" resource_attributes { name = "serviceName" operator = "stringEquals" - value = local.kms_service + value = local.backup_kms_service } resource_attributes { name = "accountId" operator = "stringEquals" - value = local.kms_account_id + value = local.backup_kms_account_id } resource_attributes { name = "serviceInstance" operator = "stringEquals" - value = var.existing_kms_instance_guid + value = local.backup_kms_key_instance_guid } resource_attributes { name = "resourceType" @@ -125,10 +163,15 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy" { # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 resource "time_sleep" "wait_for_backup_kms_authorization_policy" { + count = local.create_backup_kms_auth_policy depends_on = [ibm_iam_authorization_policy.backup_kms_policy] create_duration = "30s" } +######################################################################################################################## +# Elasticsearch instance +######################################################################################################################## + resource "ibm_database" "elasticsearch" { depends_on = [time_sleep.wait_for_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy] name = var.name @@ -254,7 +297,7 @@ resource "ibm_database" "elasticsearch" { } timeouts { - create = "120m" #Extending provisioning time to 120 minutes + create = "120m" # Extending provisioning time to 120 minutes update = "120m" delete = "15m" } @@ -268,9 +311,9 @@ resource "ibm_resource_tag" "elasticsearch_tag" { } -############################################################################## +######################################################################################################################## # Context Based Restrictions -############################################################################## +######################################################################################################################## module "cbr_rule" { count = length(var.cbr_rules) > 0 ? length(var.cbr_rules) : 0 diff --git a/modules/fscloud/README.md b/modules/fscloud/README.md index d89eaa74..d3128ce7 100644 --- a/modules/fscloud/README.md +++ b/modules/fscloud/README.md @@ -34,13 +34,12 @@ No resources. | [admin\_pass](#input\_admin\_pass) | The password for the database administrator. If the admin password is null then the admin user ID cannot be accessed. More users can be specified in a user block. | `string` | `null` | no | | [auto\_scaling](#input\_auto\_scaling) | Optional rules to allow the database to increase resources in response to usage. Only a single autoscaling block is allowed. Make sure you understand the effects of autoscaling, especially for production environments. See https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-autoscaling&interface=cli#autoscaling-considerations in the IBM Cloud Docs. | 
object({
    disk = object({
      capacity_enabled             = optional(bool, false)
      free_space_less_than_percent = optional(number, 10)
      io_above_percent             = optional(number, 90)
      io_enabled                   = optional(bool, false)
      io_over_period               = optional(string, "15m")
      rate_increase_percent        = optional(number, 10)
      rate_limit_mb_per_member     = optional(number, 3670016)
      rate_period_seconds          = optional(number, 900)
      rate_units                   = optional(string, "mb")
    })
    memory = object({
      io_above_percent         = optional(number, 90)
      io_enabled               = optional(bool, false)
      io_over_period           = optional(string, "15m")
      rate_increase_percent    = optional(number, 10)
      rate_limit_mb_per_member = optional(number, 114688)
      rate_period_seconds      = optional(number, 900)
      rate_units               = optional(string, "mb")
    })
  })
| `null` | no | | [backup\_crn](#input\_backup\_crn) | The CRN of a backup resource to restore from. The backup is created by a database deployment with the same service ID. The backup is loaded after provisioning and the new deployment starts up that uses that data. A backup CRN is in the format crn:v1:<…>:backup:. If omitted, the database is provisioned empty. | `string` | `null` | no | -| [backup\_encryption\_key\_crn](#input\_backup\_encryption\_key\_crn) | The Hyper Protect Crypto Services (HPCS) or Key Protect root key CRN to use for encrypting the disk that holds deployment backups. There are region limitations for backup encryption. See https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups (HPCS) and https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok (Key Protect). | `string` | `null` | no | +| [backup\_encryption\_key\_crn](#input\_backup\_encryption\_key\_crn) | The CRN of a Key Protect or Hyper Protect Crypto Services encryption key that you want to use for encrypting the disk that holds deployment backups. Applies only if `use_ibm_owned_encryption_key` is false and `use_same_kms_key_for_backups` is false. If no value is passed, and `use_same_kms_key_for_backups` is true, the value of `kms_key_crn` is used. Alternatively set `use_default_backup_encryption_key` to true to use the IBM Cloud Databases default encryption. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups). | `string` | `null` | no | | [cbr\_rules](#input\_cbr\_rules) | (Optional, list) List of CBR rules to create | 
list(object({
    description = string
    account_id  = string
    rule_contexts = list(object({
      attributes = optional(list(object({
        name  = string
        value = string
    }))) }))
    enforcement_mode = string
  }))
| `[]` | no | | [elasticsearch\_version](#input\_elasticsearch\_version) | Version of the Elasticsearch instance. If no value is passed, the current preferred version of IBM Cloud Databases is used. | `string` | `null` | no | | [elser\_model\_type](#input\_elser\_model\_type) | Trained ELSER model to be used for Elastic's Natural Language Processing. Possible values: `.elser_model_1`, `.elser_model_2` and `.elser_model_2_linux-x86_64`. [Learn more](https://www.elastic.co/guide/en/machine-learning/current/ml-nlp-elser.html) | `string` | `".elser_model_2_linux-x86_64"` | no | | [enable\_elser\_model](#input\_enable\_elser\_model) | Set it to true to install and start the Elastic's Natural Language Processing model. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-elser-embeddings-elasticsearch) | `bool` | `false` | no | -| [existing\_kms\_instance\_guid](#input\_existing\_kms\_instance\_guid) | The GUID of the Hyper Protect Crypto Services (HPCS) or Key Protect instance. | `string` | `null` | no | -| [kms\_key\_crn](#input\_kms\_key\_crn) | The Hyper Protect Crypto Services (HPCS) or Key Protect root key CRN to use for disk encryption. | `string` | `null` | no | +| [kms\_key\_crn](#input\_kms\_key\_crn) | The CRN of a Key Protect or Hyper Protect Crypto Services encryption key to encrypt your data. Applies only if `use_ibm_owned_encryption_key` is false. By default this key is used for both deployment data and backups, but this behaviour can be altered using the `use_same_kms_key_for_backups` and `backup_encryption_key_crn` inputs. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups). | `string` | `null` | no | | [member\_cpu\_count](#input\_member\_cpu\_count) | Allocated dedicated CPU per member. For shared CPU, set to 0. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling) | `number` | `0` | no | | [member\_disk\_mb](#input\_member\_disk\_mb) | Allocated disk per-member. [Learn more}(https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling) | `number` | `5120` | no | | [member\_host\_flavor](#input\_member\_host\_flavor) | Allocated host flavor per member. [Learn more](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/database#host_flavor). | `string` | `null` | no | @@ -51,9 +50,11 @@ No resources. | [region](#input\_region) | The region where you want to deploy your instance. Must be the same region as the Hyper Protect Crypto Services instance. | `string` | `"us-south"` | no | | [resource\_group\_id](#input\_resource\_group\_id) | The resource group ID where the Elasticsearch instance will be created. | `string` | n/a | yes | | [service\_credential\_names](#input\_service\_credential\_names) | Map of name, role for service credentials that you want to create for the database | `map(string)` | `{}` | no | -| [skip\_iam\_authorization\_policy](#input\_skip\_iam\_authorization\_policy) | Set to true to skip the creation of an IAM authorization policy that permits all ElasticSearch database instances in the resource group to read the encryption key from the Hyper Protect Crypto Services or Key Protect instance. The instance is passed in through the var.existing\_kms\_instance\_guid variable. | `bool` | `false` | no | +| [skip\_iam\_authorization\_policy](#input\_skip\_iam\_authorization\_policy) | Set to true to skip the creation of IAM authorization policies that permits all Databases for Elasticsearch instances in the given resource group 'Reader' access to the Key Protect or Hyper Protect Crypto Services key that was provided in the `kms_key_crn` and `backup_encryption_key_crn` inputs. This policy is required in order to enable KMS encryption, so only skip creation if there is one already present in your account. No policy is created if `use_ibm_owned_encryption_key` is true. | `bool` | `false` | no | | [tags](#input\_tags) | Optional list of tags to be added to the Elasticsearch instance. | `list(any)` | `[]` | no | +| [use\_default\_backup\_encryption\_key](#input\_use\_default\_backup\_encryption\_key) | When `use_ibm_owned_encryption_key` is set to false, backups will be encrypted with either the key specified in `kms_key_crn`, or in `backup_encryption_key_crn` if a value is passed. If you do not want to use your own key for backups encryption, you can set this to `true` to use the IBM Cloud Databases default encryption for backups. Alternatively set `use_ibm_owned_encryption_key` to true to use the default encryption for both backups and deployment data. | `bool` | `false` | no | | [use\_ibm\_owned\_encryption\_key](#input\_use\_ibm\_owned\_encryption\_key) | Set to true to use the default IBM Cloud® Databases randomly generated keys for disk and backups encryption. To control the encryption keys, use the `kms_key_crn` and `backup_encryption_key_crn` inputs. | `bool` | `false` | no | +| [use\_same\_kms\_key\_for\_backups](#input\_use\_same\_kms\_key\_for\_backups) | Set this to false if you wan't to use a different key that you own to encrypt backups. When set to false, a value is required for the `backup_encryption_key_crn` input. Alternatiely set `use_default_backup_encryption_key` to true to use the IBM Cloud Databases default encryption. Applies only if `use_ibm_owned_encryption_key` is false. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups). | `bool` | `true` | no | | [users](#input\_users) | A list of users that you want to create on the database. Multiple blocks are allowed. The user password must be in the range of 10-32 characters. Be warned that in most case using IAM service credentials (via the var.service\_credential\_names) is sufficient to control access to the Elasticsearch instance. This blocks creates native Elasticsearch database users, more info on that can be found here https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-user-management&interface=ui | 
list(object({
    name     = string
    password = string # pragma: allowlist secret
    type     = optional(string)
    role     = optional(string)
  }))
| `[]` | no | ### Outputs diff --git a/modules/fscloud/main.tf b/modules/fscloud/main.tf index a40ceb98..9a52d8e6 100644 --- a/modules/fscloud/main.tf +++ b/modules/fscloud/main.tf @@ -1,36 +1,30 @@ -locals { - # tflint-ignore: terraform_unused_declarations - validate_kms_inputs = !var.use_ibm_owned_encryption_key && (var.kms_key_crn == null || var.existing_kms_instance_guid == null) ? tobool("Values for 'kms_key_crn' and 'existing_kms_instance_guid' must be passed if 'use_ibm_owned_encryption_key' it set to false.") : true - # tflint-ignore: terraform_unused_declarations - validate_kms_inputs_2 = var.use_ibm_owned_encryption_key && (var.kms_key_crn != null || var.backup_encryption_key_crn != null || var.existing_kms_instance_guid != null) ? tobool("'use_ibm_owned_encryption_key' is set to true, but values have been passed for either 'kms_key_crn', 'backup_encryption_key_crn' and/or 'existing_kms_instance_guid'. To use BYOK or KYOK encryption, ensure to set 'use_ibm_owned_encryption_key' to false, and pass values for 'kms_key_crn', 'backup_encryption_key_crn' (optional) and 'existing_kms_instance_guid'. Alternatively do not pass any values for 'kms_key_crn', 'backup_encryption_key_crn' and 'existing_kms_instance_guid' to use the IBM owned encryption keys.") : true -} - module "elasticsearch" { - source = "../../" - resource_group_id = var.resource_group_id - name = var.name - region = var.region - skip_iam_authorization_policy = var.skip_iam_authorization_policy - service_endpoints = "private" - elasticsearch_version = var.elasticsearch_version - kms_encryption_enabled = !var.use_ibm_owned_encryption_key - existing_kms_instance_guid = var.existing_kms_instance_guid - kms_key_crn = var.kms_key_crn - backup_crn = var.backup_crn - backup_encryption_key_crn = var.backup_encryption_key_crn - cbr_rules = var.cbr_rules - access_tags = var.access_tags - tags = var.tags - plan = var.plan - members = var.members - member_memory_mb = var.member_memory_mb - admin_pass = var.admin_pass - users = var.users - member_disk_mb = var.member_disk_mb - member_cpu_count = var.member_cpu_count - member_host_flavor = var.member_host_flavor - auto_scaling = var.auto_scaling - service_credential_names = var.service_credential_names - enable_elser_model = var.enable_elser_model - elser_model_type = var.elser_model_type + source = "../../" + resource_group_id = var.resource_group_id + name = var.name + region = var.region + skip_iam_authorization_policy = var.skip_iam_authorization_policy + service_endpoints = "private" + elasticsearch_version = var.elasticsearch_version + use_ibm_owned_encryption_key = var.use_ibm_owned_encryption_key + use_same_kms_key_for_backups = var.use_same_kms_key_for_backups + use_default_backup_encryption_key = var.use_default_backup_encryption_key + kms_key_crn = var.kms_key_crn + backup_crn = var.backup_crn + backup_encryption_key_crn = var.backup_encryption_key_crn + cbr_rules = var.cbr_rules + access_tags = var.access_tags + tags = var.tags + plan = var.plan + members = var.members + member_memory_mb = var.member_memory_mb + admin_pass = var.admin_pass + users = var.users + member_disk_mb = var.member_disk_mb + member_cpu_count = var.member_cpu_count + member_host_flavor = var.member_host_flavor + auto_scaling = var.auto_scaling + service_credential_names = var.service_credential_names + enable_elser_model = var.enable_elser_model + elser_model_type = var.elser_model_type } diff --git a/modules/fscloud/variables.tf b/modules/fscloud/variables.tf index d6d0bc3c..d54d554e 100644 --- a/modules/fscloud/variables.tf +++ b/modules/fscloud/variables.tf @@ -144,26 +144,50 @@ variable "use_ibm_owned_encryption_key" { variable "kms_key_crn" { type = string - description = "The Hyper Protect Crypto Services (HPCS) or Key Protect root key CRN to use for disk encryption." + description = "The CRN of a Key Protect or Hyper Protect Crypto Services encryption key to encrypt your data. Applies only if `use_ibm_owned_encryption_key` is false. By default this key is used for both deployment data and backups, but this behaviour can be altered using the `use_same_kms_key_for_backups` and `backup_encryption_key_crn` inputs. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." default = null + + validation { + condition = anytrue([ + var.kms_key_crn == null, + can(regex(".*kms.*", var.kms_key_crn)), + can(regex(".*hs-crypto.*", var.kms_key_crn)), + ]) + error_message = "Value must be the KMS key CRN from a Key Protect or Hyper Protect Crypto Services instance." + } +} + +variable "use_same_kms_key_for_backups" { + type = bool + description = "Set this to false if you wan't to use a different key that you own to encrypt backups. When set to false, a value is required for the `backup_encryption_key_crn` input. Alternatiely set `use_default_backup_encryption_key` to true to use the IBM Cloud Databases default encryption. Applies only if `use_ibm_owned_encryption_key` is false. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." + default = true } variable "backup_encryption_key_crn" { type = string - description = "The Hyper Protect Crypto Services (HPCS) or Key Protect root key CRN to use for encrypting the disk that holds deployment backups. There are region limitations for backup encryption. See https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups (HPCS) and https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok (Key Protect)." + description = "The CRN of a Key Protect or Hyper Protect Crypto Services encryption key that you want to use for encrypting the disk that holds deployment backups. Applies only if `use_ibm_owned_encryption_key` is false and `use_same_kms_key_for_backups` is false. If no value is passed, and `use_same_kms_key_for_backups` is true, the value of `kms_key_crn` is used. Alternatively set `use_default_backup_encryption_key` to true to use the IBM Cloud Databases default encryption. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." default = null + + validation { + condition = anytrue([ + var.backup_encryption_key_crn == null, + can(regex(".*kms.*", var.kms_key_crn)), + can(regex(".*hs-crypto.*", var.kms_key_crn)), + ]) + error_message = "Value must be the KMS key CRN from a Key Protect or Hyper Protect Crypto Services instance in one of the supported backup regions." + } } -variable "skip_iam_authorization_policy" { +variable "use_default_backup_encryption_key" { type = bool - description = "Set to true to skip the creation of an IAM authorization policy that permits all ElasticSearch database instances in the resource group to read the encryption key from the Hyper Protect Crypto Services or Key Protect instance. The instance is passed in through the var.existing_kms_instance_guid variable." + description = "When `use_ibm_owned_encryption_key` is set to false, backups will be encrypted with either the key specified in `kms_key_crn`, or in `backup_encryption_key_crn` if a value is passed. If you do not want to use your own key for backups encryption, you can set this to `true` to use the IBM Cloud Databases default encryption for backups. Alternatively set `use_ibm_owned_encryption_key` to true to use the default encryption for both backups and deployment data." default = false } -variable "existing_kms_instance_guid" { - type = string - description = "The GUID of the Hyper Protect Crypto Services (HPCS) or Key Protect instance." - default = null +variable "skip_iam_authorization_policy" { + type = bool + description = "Set to true to skip the creation of IAM authorization policies that permits all Databases for Elasticsearch instances in the given resource group 'Reader' access to the Key Protect or Hyper Protect Crypto Services key that was provided in the `kms_key_crn` and `backup_encryption_key_crn` inputs. This policy is required in order to enable KMS encryption, so only skip creation if there is one already present in your account. No policy is created if `use_ibm_owned_encryption_key` is true." + default = false } ############################################################## diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index 5af0ec1f..4b595433 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -1,59 +1,135 @@ ####################################################################################################################### -# Local Variables +# Resource Group +####################################################################################################################### + +module "resource_group" { + source = "terraform-ibm-modules/resource-group/ibm" + version = "1.1.6" + resource_group_name = var.use_existing_resource_group == false ? (var.prefix != null ? "${var.prefix}-${var.resource_group_name}" : var.resource_group_name) : null + existing_resource_group_name = var.use_existing_resource_group == true ? var.resource_group_name : null +} + +####################################################################################################################### +# KMS related variable validation +# (approach based on https://github.com/hashicorp/terraform/issues/25609#issuecomment-1057614400) +# +# TODO: Replace with terraform cross variable validation: https://github.ibm.com/GoldenEye/issues/issues/10836 ####################################################################################################################### locals { - existing_kms_instance_crn_split = var.existing_kms_instance_crn != null ? split(":", var.existing_kms_instance_crn) : null - existing_kms_instance_guid = var.existing_kms_instance_crn != null ? element(local.existing_kms_instance_crn_split, length(local.existing_kms_instance_crn_split) - 3) : null - existing_kms_instance_region = var.existing_kms_instance_crn != null ? element(local.existing_kms_instance_crn_split, length(local.existing_kms_instance_crn_split) - 5) : null + # tflint-ignore: terraform_unused_declarations + validate_kms_1 = var.use_ibm_owned_encryption_key && (var.existing_kms_instance_crn != null || var.existing_kms_key_crn != null || var.existing_backup_kms_key_crn != null) ? tobool("When setting values for 'existing_kms_instance_crn', 'existing_kms_key_crn' or 'existing_backup_kms_key_crn', the 'use_ibm_owned_encryption_key' input must be set to false.") : true + # tflint-ignore: terraform_unused_declarations + validate_kms_2 = !var.use_ibm_owned_encryption_key && (var.existing_kms_instance_crn == null && var.existing_kms_key_crn == null) ? tobool("When 'use_ibm_owned_encryption_key' is false, a value is required for either 'existing_kms_instance_crn' (to create a new key), or 'existing_kms_key_crn' to use an existing key.") : true +} + +####################################################################################################################### +# KMS encryption key +####################################################################################################################### +locals { + create_new_kms_key = var.existing_db_instance_crn == null && !var.use_ibm_owned_encryption_key && var.existing_kms_key_crn == null ? 1 : 0 # no need to create any KMS resources if using existing Elasticsearch, passing an existing key, or using IBM owned keys elasticsearch_key_name = var.prefix != null ? "${var.prefix}-${var.elasticsearch_key_name}" : var.elasticsearch_key_name elasticsearch_key_ring_name = var.prefix != null ? "${var.prefix}-${var.elasticsearch_key_ring_name}" : var.elasticsearch_key_ring_name +} +module "kms" { + providers = { + ibm = ibm.kms + } + count = local.create_new_kms_key + source = "terraform-ibm-modules/kms-all-inclusive/ibm" + version = "4.17.1" + create_key_protect_instance = false + region = local.kms_region + existing_kms_instance_crn = var.existing_kms_instance_crn + key_ring_endpoint_type = var.kms_endpoint_type + key_endpoint_type = var.kms_endpoint_type + keys = [ + { + key_ring_name = local.elasticsearch_key_ring_name + existing_key_ring = false + keys = [ + { + key_name = local.elasticsearch_key_name + standard_key = false + rotation_interval_month = 3 + dual_auth_delete_enabled = false + force_delete = true + } + ] + } + ] +} - existing_db_instance_guid = var.existing_db_instance_crn != null ? element(split(":", var.existing_db_instance_crn), length(split(":", var.existing_db_instance_crn)) - 3) : null - use_existing_db_instance = var.existing_db_instance_crn != null - - create_cross_account_auth_policy = !var.skip_iam_authorization_policy && var.ibmcloud_kms_api_key != null && !var.use_ibm_owned_encryption_key - create_sm_auth_policy = var.skip_es_sm_auth_policy || var.existing_secrets_manager_instance_crn == null ? 0 : 1 - - kms_key_crn = var.existing_db_instance_crn != null ? null : !var.use_ibm_owned_encryption_key ? var.existing_kms_key_crn != null ? var.existing_kms_key_crn : module.kms[0].keys[format("%s.%s", local.elasticsearch_key_ring_name, local.elasticsearch_key_name)].crn : null - parsed_kms_key_crn = local.kms_key_crn != null ? split(":", local.kms_key_crn) : [] - kms_service = length(local.parsed_kms_key_crn) > 0 ? local.parsed_kms_key_crn[4] : null - kms_scope = length(local.parsed_kms_key_crn) > 0 ? local.parsed_kms_key_crn[6] : null - kms_account_id = length(local.parsed_kms_key_crn) > 0 ? split("/", local.kms_scope)[1] : null - kms_key_id = length(local.parsed_kms_key_crn) > 0 ? local.parsed_kms_key_crn[9] : null +######################################################################################################################## +# Parse KMS info from given CRNs +######################################################################################################################## - elasticsearch_guid = local.use_existing_db_instance ? data.ibm_database.existing_db_instance[0].guid : module.elasticsearch[0].guid +module "kms_instance_crn_parser" { + count = var.existing_kms_instance_crn != null ? 1 : 0 + source = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser" + version = "1.1.0" + crn = var.existing_kms_instance_crn } -####################################################################################################################### -# Resource Group -####################################################################################################################### +module "kms_key_crn_parser" { + count = var.existing_kms_key_crn != null ? 1 : 0 + source = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser" + version = "1.1.0" + crn = var.existing_kms_key_crn +} -module "resource_group" { - source = "terraform-ibm-modules/resource-group/ibm" - version = "1.1.6" - resource_group_name = var.use_existing_resource_group == false ? (var.prefix != null ? "${var.prefix}-${var.resource_group_name}" : var.resource_group_name) : null - existing_resource_group_name = var.use_existing_resource_group == true ? var.resource_group_name : null +module "kms_backup_key_crn_parser" { + count = var.existing_backup_kms_key_crn != null ? 1 : 0 + source = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser" + version = "1.1.0" + crn = var.existing_backup_kms_key_crn } ####################################################################################################################### -# KMS root key for Elasticsearch +# KMS IAM Authorization Policies +# - only created if user passes a value for 'ibmcloud_kms_api_key' (used when KMS is in different account to Elasticsearch) +# - if no value passed for 'ibmcloud_kms_api_key', the auth policy is created by the Elasticsearch module ####################################################################################################################### +# Lookup account ID data "ibm_iam_account_settings" "iam_account_settings" { - count = local.create_cross_account_auth_policy ? 1 : 0 } +locals { + account_id = data.ibm_iam_account_settings.iam_account_settings.account_id + create_cross_account_kms_auth_policy = var.existing_db_instance_crn == null && !var.skip_es_kms_auth_policy && var.ibmcloud_kms_api_key != null && !var.use_ibm_owned_encryption_key + create_cross_account_backup_kms_auth_policy = var.existing_db_instance_crn == null && !var.skip_es_kms_auth_policy && var.ibmcloud_kms_api_key != null && !var.use_ibm_owned_encryption_key && var.existing_backup_kms_key_crn != null + + # If KMS encryption enabled (and existing ES instance is not being passed), parse details from the existing key if being passed, otherwise get it from the key that the DA creates + kms_account_id = var.existing_db_instance_crn != null || var.use_ibm_owned_encryption_key ? null : var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].account_id : module.kms_instance_crn_parser[0].account_id + kms_service = var.existing_db_instance_crn != null || var.use_ibm_owned_encryption_key ? null : var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].service_name : module.kms_instance_crn_parser[0].service_name + kms_instance_guid = var.existing_db_instance_crn != null || var.use_ibm_owned_encryption_key ? null : var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].service_instance : module.kms_instance_crn_parser[0].service_instance + kms_key_crn = var.existing_db_instance_crn != null || var.use_ibm_owned_encryption_key ? null : var.existing_kms_key_crn != null ? var.existing_kms_key_crn : module.kms[0].keys[format("%s.%s", local.elasticsearch_key_ring_name, local.elasticsearch_key_name)].crn + kms_key_id = var.existing_db_instance_crn != null || var.use_ibm_owned_encryption_key ? null : var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].resource : module.kms[0].keys[format("%s.%s", local.elasticsearch_key_ring_name, local.elasticsearch_key_name)].key_id + kms_region = var.existing_db_instance_crn != null || var.use_ibm_owned_encryption_key ? null : var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].region : module.kms_instance_crn_parser[0].region + + # If creating KMS cross account policy for backups, parse backup key details from passed in key CRN + backup_kms_account_id = local.create_cross_account_backup_kms_auth_policy ? module.kms_backup_key_crn_parser[0].account_id : local.kms_account_id + backup_kms_service = local.create_cross_account_backup_kms_auth_policy ? module.kms_backup_key_crn_parser[0].service_name : local.kms_service + backup_kms_instance_guid = local.create_cross_account_backup_kms_auth_policy ? module.kms_backup_key_crn_parser[0].service_instance : local.kms_instance_guid + backup_kms_key_id = local.create_cross_account_backup_kms_auth_policy ? module.kms_backup_key_crn_parser[0].resource : local.kms_key_id + + backup_kms_key_crn = var.existing_db_instance_crn != null || var.use_ibm_owned_encryption_key ? null : var.existing_backup_kms_key_crn + # Always use same key for backups unless user explicially passed a value for 'existing_backup_kms_key_crn' + use_same_kms_key_for_backups = var.existing_backup_kms_key_crn == null ? true : false +} + +# Create auth policy (scoped to exact KMS key) resource "ibm_iam_authorization_policy" "kms_policy" { - count = local.create_cross_account_auth_policy ? 1 : 0 + count = local.create_cross_account_kms_auth_policy ? 1 : 0 provider = ibm.kms - source_service_account = data.ibm_iam_account_settings.iam_account_settings[0].account_id + source_service_account = local.account_id source_service_name = "databases-for-elasticsearch" source_resource_group_id = module.resource_group.resource_group_id roles = ["Reader"] - description = "Allow all Elastic Search instances in the resource group ${module.resource_group.resource_group_id} in the account ${data.ibm_iam_account_settings.iam_account_settings[0].account_id} to read the ${local.kms_service} key ${local.kms_key_id} from the instance GUID ${local.existing_kms_instance_guid}" + description = "Allow all Elastic Search instances in the resource group ${module.resource_group.resource_group_id} in the account ${local.account_id} to read the ${local.kms_service} key ${local.kms_key_id} from the instance GUID ${local.kms_instance_guid}" resource_attributes { name = "serviceName" operator = "stringEquals" @@ -67,7 +143,7 @@ resource "ibm_iam_authorization_policy" "kms_policy" { resource_attributes { name = "serviceInstance" operator = "stringEquals" - value = local.existing_kms_instance_guid + value = local.kms_instance_guid } resource_attributes { name = "resourceType" @@ -88,159 +164,193 @@ resource "ibm_iam_authorization_policy" "kms_policy" { # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 resource "time_sleep" "wait_for_authorization_policy" { - count = local.create_cross_account_auth_policy ? 1 : 0 + count = local.create_cross_account_kms_auth_policy ? 1 : 0 depends_on = [ibm_iam_authorization_policy.kms_policy] create_duration = "30s" } -module "kms" { - providers = { - ibm = ibm.kms +# Create auth policy (scoped to exact KMS key for backups) +resource "ibm_iam_authorization_policy" "backup_kms_policy" { + count = local.create_cross_account_backup_kms_auth_policy ? 1 : 0 + provider = ibm.kms + source_service_account = local.account_id + source_service_name = "databases-for-elasticsearch" + source_resource_group_id = module.resource_group.resource_group_id + roles = ["Reader"] + description = "Allow all Elastic Search instances in the resource group ${module.resource_group.resource_group_id} in the account ${local.account_id} to read the ${local.backup_kms_service} key ${local.backup_kms_key_id} from the instance GUID ${local.backup_kms_instance_guid}" + resource_attributes { + name = "serviceName" + operator = "stringEquals" + value = local.backup_kms_service + } + resource_attributes { + name = "accountId" + operator = "stringEquals" + value = local.backup_kms_account_id + } + resource_attributes { + name = "serviceInstance" + operator = "stringEquals" + value = local.backup_kms_instance_guid + } + resource_attributes { + name = "resourceType" + operator = "stringEquals" + value = "key" + } + resource_attributes { + name = "resource" + operator = "stringEquals" + value = local.backup_kms_key_id + } + # Scope of policy now includes the key, so ensure to create new policy before + # destroying old one to prevent any disruption to every day services. + lifecycle { + create_before_destroy = true } - count = var.existing_kms_key_crn != null || local.use_existing_db_instance || var.use_ibm_owned_encryption_key ? 0 : 1 # no need to create any KMS resources if passing an existing key or using IBM owned keys - source = "terraform-ibm-modules/kms-all-inclusive/ibm" - version = "4.17.1" - create_key_protect_instance = false - region = local.existing_kms_instance_region - existing_kms_instance_crn = var.existing_kms_instance_crn - key_ring_endpoint_type = var.kms_endpoint_type - key_endpoint_type = var.kms_endpoint_type - keys = [ - { - key_ring_name = local.elasticsearch_key_ring_name - existing_key_ring = false - keys = [ - { - key_name = local.elasticsearch_key_name - standard_key = false - rotation_interval_month = 3 - dual_auth_delete_enabled = false - force_delete = true - } - ] - } - ] } +# workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 +resource "time_sleep" "wait_for_backup_kms_authorization_policy" { + count = local.create_cross_account_backup_kms_auth_policy ? 1 : 0 + depends_on = [ibm_iam_authorization_policy.backup_kms_policy] + create_duration = "30s" +} ####################################################################################################################### -# KMS backup encryption key for Elasticsearch +# Elasticsearch admin password ####################################################################################################################### +resource "random_password" "admin_password" { + count = var.admin_pass == null ? 1 : 0 + length = 32 + special = true + override_special = "-_" + min_numeric = 1 +} + locals { - existing_backup_kms_instance_guid = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].service_instance : null - existing_backup_kms_instance_region = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].region : null + # _- are invalid first characters + # if - replace first char with J + # elseif _ replace first char with K + # else use asis + generated_admin_password = startswith(random_password.admin_password[0].result, "-") ? "J${substr(random_password.admin_password[0].result, 1, -1)}" : startswith(random_password.admin_password[0].result, "_") ? "K${substr(random_password.admin_password[0].result, 1, -1)}" : random_password.admin_password[0].result - backup_key_name = var.prefix != null ? "${var.prefix}-backup-encryption-${var.elasticsearch_key_name}" : "backup-encryption-${var.elasticsearch_key_name}" - backup_key_ring_name = var.prefix != null ? "${var.prefix}-backup-encryption-${var.elasticsearch_key_ring_name}" : "backup-encryption-${var.elasticsearch_key_ring_name}" - backup_kms_key_crn = var.existing_backup_kms_key_crn != null ? var.existing_backup_kms_key_crn : var.existing_backup_kms_instance_crn != null ? module.backup_kms[0].keys[format("%s.%s", local.backup_key_ring_name, local.backup_key_name)].crn : null - backup_kms_service_name = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].service_name : null + # admin password to use + admin_pass = var.admin_pass == null ? local.generated_admin_password : var.admin_pass } -# If existing KMS intance CRN passed, parse details from it -module "backup_kms_instance_crn_parser" { - count = var.existing_backup_kms_instance_crn != null ? 1 : 0 +####################################################################################################################### +# Elasticsearch +####################################################################################################################### + +# Look up existing instance details if user passes one +module "es_instance_crn_parser" { + count = var.existing_db_instance_crn != null ? 1 : 0 source = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser" version = "1.1.0" - crn = var.existing_backup_kms_instance_crn + crn = var.existing_db_instance_crn } -resource "ibm_iam_authorization_policy" "backup_kms_policy" { - count = local.existing_backup_kms_instance_guid == local.existing_kms_instance_guid ? 0 : var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? !var.skip_iam_authorization_policy ? 1 : 0 : 0 - provider = ibm.kms - source_service_account = local.create_cross_account_auth_policy ? data.ibm_iam_account_settings.iam_account_settings[0].account_id : null - source_service_name = "databases-for-elasticsearch" - source_resource_group_id = module.resource_group.resource_group_id - target_service_name = local.backup_kms_service_name - target_resource_instance_id = local.existing_backup_kms_instance_guid - roles = ["Reader"] - description = "Allow all Elasticsearch instances in the resource group ${module.resource_group.resource_group_id} to read from the ${local.backup_kms_service_name} instance GUID ${local.existing_backup_kms_instance_guid}" +# Existing instance local vars +locals { + existing_elasticsearch_guid = var.existing_db_instance_crn != null ? module.es_instance_crn_parser[0].guid : null + existing_elasticsearch_region = var.existing_db_instance_crn != null ? module.es_instance_crn_parser[0].region : null + + # Validate the region input matches region detected in existing instance CRN (approach based on https://github.com/hashicorp/terraform/issues/25609#issuecomment-1057614400) + # tflint-ignore: terraform_unused_declarations + validate_existing_instance_region = var.existing_db_instance_crn != null && var.region != local.existing_elasticsearch_region ? tobool("The region detected in the 'existing_db_instance_crn' value must match the value of the 'region' input variable when passing an existing instance.") : true } -# workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 -resource "time_sleep" "wait_for_backup_kms_authorization_policy" { - depends_on = [ibm_iam_authorization_policy.backup_kms_policy] - create_duration = "30s" +# Do a data lookup on the resource GUID to get more info that is needed for the 'ibm_database' data lookup below +data "ibm_resource_instance" "existing_instance_resource" { + count = var.existing_db_instance_crn != null ? 1 : 0 + identifier = local.existing_elasticsearch_guid } -module "backup_kms" { - providers = { - ibm = ibm.kms - } - count = var.use_ibm_owned_encryption_key ? 0 : var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? 1 : 0 - source = "terraform-ibm-modules/kms-all-inclusive/ibm" - version = "4.16.8" - create_key_protect_instance = false - region = local.existing_backup_kms_instance_region - existing_kms_instance_crn = var.existing_backup_kms_instance_crn - key_ring_endpoint_type = var.kms_endpoint_type - key_endpoint_type = var.kms_endpoint_type - keys = [ - { - key_ring_name = local.backup_key_ring_name - existing_key_ring = false - force_delete_key_ring = true - keys = [ - { - key_name = local.backup_key_name - standard_key = false - rotation_interval_month = 3 - dual_auth_delete_enabled = false - force_delete = true - } - ] - } - ] +# Lookup details of existing instance +data "ibm_database" "existing_db_instance" { + count = var.existing_db_instance_crn != null ? 1 : 0 + name = data.ibm_resource_instance.existing_instance_resource[0].name + resource_group_id = data.ibm_resource_instance.existing_instance_resource[0].resource_group_id + location = var.region + service = "databases-for-elasticsearch" } -####################################################################################################################### -# Elasticsearch -####################################################################################################################### +# Lookup existing instance connection details +data "ibm_database_connection" "existing_connection" { + count = var.existing_db_instance_crn != null ? 1 : 0 + endpoint_type = "private" + deployment_id = data.ibm_database.existing_db_instance[0].id + user_id = data.ibm_database.existing_db_instance[0].adminuser + user_type = "database" +} +# Create new instance module "elasticsearch" { - count = local.use_existing_db_instance ? 0 : 1 - source = "../../modules/fscloud" - depends_on = [time_sleep.wait_for_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy] - resource_group_id = module.resource_group.resource_group_id - name = var.prefix != null ? "${var.prefix}-${var.name}" : var.name - region = var.region - plan = var.plan - skip_iam_authorization_policy = var.skip_iam_authorization_policy || local.create_cross_account_auth_policy - elasticsearch_version = var.elasticsearch_version - existing_kms_instance_guid = local.existing_kms_instance_guid - use_ibm_owned_encryption_key = var.use_ibm_owned_encryption_key - kms_key_crn = local.kms_key_crn - backup_encryption_key_crn = local.backup_kms_key_crn - backup_crn = var.backup_crn - access_tags = var.access_tags - tags = var.tags - admin_pass = local.admin_pass - users = var.users - members = var.members - member_host_flavor = var.member_host_flavor - member_memory_mb = var.member_memory_mb - member_disk_mb = var.member_disk_mb - member_cpu_count = var.member_cpu_count - auto_scaling = var.auto_scaling - service_credential_names = var.service_credential_names - enable_elser_model = var.enable_elser_model - elser_model_type = var.elser_model_type + count = var.existing_db_instance_crn != null ? 0 : 1 + source = "../../modules/fscloud" + depends_on = [time_sleep.wait_for_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy] + resource_group_id = module.resource_group.resource_group_id + name = var.prefix != null ? "${var.prefix}-${var.name}" : var.name + region = var.region + plan = var.plan + skip_iam_authorization_policy = var.skip_es_kms_auth_policy + elasticsearch_version = var.elasticsearch_version + use_ibm_owned_encryption_key = var.use_ibm_owned_encryption_key + kms_key_crn = local.kms_key_crn + backup_encryption_key_crn = local.backup_kms_key_crn + use_same_kms_key_for_backups = local.use_same_kms_key_for_backups + use_default_backup_encryption_key = var.use_default_backup_encryption_key + backup_crn = var.backup_crn + access_tags = var.access_tags + tags = var.tags + admin_pass = local.admin_pass + users = var.users + members = var.members + member_host_flavor = var.member_host_flavor + member_memory_mb = var.member_memory_mb + member_disk_mb = var.member_disk_mb + member_cpu_count = var.member_cpu_count + auto_scaling = var.auto_scaling + service_credential_names = var.service_credential_names + enable_elser_model = var.enable_elser_model + elser_model_type = var.elser_model_type } -resource "random_password" "admin_password" { - count = var.admin_pass == null ? 1 : 0 - length = 32 - special = true - override_special = "-_" - min_numeric = 1 +locals { + elasticsearch_guid = var.existing_db_instance_crn != null ? data.ibm_database.existing_db_instance[0].guid : module.elasticsearch[0].guid + elasticsearch_id = var.existing_db_instance_crn != null ? data.ibm_database.existing_db_instance[0].id : module.elasticsearch[0].id + elasticsearch_version = var.existing_db_instance_crn != null ? data.ibm_database.existing_db_instance[0].version : module.elasticsearch[0].version + elasticsearch_crn = var.existing_db_instance_crn != null ? var.existing_db_instance_crn : module.elasticsearch[0].crn + elasticsearch_hostname = var.existing_db_instance_crn != null ? data.ibm_database_connection.existing_connection[0].https[0].hosts[0].hostname : module.elasticsearch[0].hostname + elasticsearch_port = var.existing_db_instance_crn != null ? data.ibm_database_connection.existing_connection[0].https[0].hosts[0].port : module.elasticsearch[0].port + elasticsearch_cert = var.existing_db_instance_crn != null ? data.ibm_database_connection.existing_connection[0].https[0].certificate[0].certificate_base64 : module.elasticsearch[0].certificate_base64 + elasticsearch_username = var.existing_db_instance_crn != null ? data.ibm_database.existing_db_instance[0].adminuser : "admin" } +####################################################################################################################### +# Secrets management +####################################################################################################################### + locals { - # _- are invalid first characters - # if - replace first char with J - # elseif _ replace first char with K - # else use asis - admin_password = startswith(random_password.admin_password[0].result, "-") ? "J${substr(random_password.admin_password[0].result, 1, -1)}" : startswith(random_password.admin_password[0].result, "_") ? "K${substr(random_password.admin_password[0].result, 1, -1)}" : random_password.admin_password[0].result + ## Variable validation (approach based on https://github.com/hashicorp/terraform/issues/25609#issuecomment-1057614400) + # tflint-ignore: terraform_unused_declarations + validate_sm_crn = length(local.service_credential_secrets) > 0 && var.existing_secrets_manager_instance_crn == null ? tobool("`existing_secrets_manager_instance_crn` is required when adding service credentials to a secrets manager secret.") : false + # tflint-ignore: terraform_unused_declarations + validate_sm_sg = var.existing_secrets_manager_instance_crn != null && var.admin_pass_sm_secret_group == null ? tobool("`admin_pass_sm_secret_group` is required when `existing_secrets_manager_instance_crn` is set.") : false + # tflint-ignore: terraform_unused_declarations + validate_sm_sn = var.existing_secrets_manager_instance_crn != null && var.admin_pass_sm_secret_name == null ? tobool("`admin_pass_sm_secret_name` is required when `existing_secrets_manager_instance_crn` is set.") : false + + create_sm_auth_policy = var.skip_es_sm_auth_policy || var.existing_secrets_manager_instance_crn == null ? 0 : 1 +} + +# Parse the Secrets Manager CRN +module "sm_instance_crn_parser" { + count = var.existing_secrets_manager_instance_crn != null ? 1 : 0 + source = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser" + version = "1.1.0" + crn = var.existing_secrets_manager_instance_crn } # create a service authorization between Secrets Manager and the target service (Elastic Search) @@ -263,6 +373,7 @@ resource "time_sleep" "wait_for_es_authorization_policy" { } locals { + # Build the structure of the service credential type secret service_credential_secrets = [ for service_credentials in var.service_credential_secrets : { secret_group_name = service_credentials.secret_group_name @@ -278,14 +389,14 @@ locals { service_credentials_ttl = secret.service_credentials_ttl service_credential_secret_description = secret.service_credential_secret_description service_credentials_source_service_role = secret.service_credentials_source_service_role - service_credentials_source_service_crn = local.use_existing_db_instance ? data.ibm_database.existing_db_instance[0].id : module.elasticsearch[0].crn + service_credentials_source_service_crn = local.elasticsearch_crn secret_type = "service_credentials" #checkov:skip=CKV_SECRET_6 } ] } ] - admin_pass = var.admin_pass == null ? local.admin_password : var.admin_pass + # Build the structure of the arbitrary credential type secret for admin password admin_pass_secret = [{ secret_group_name = var.prefix != null && var.admin_pass_sm_secret_group != null ? "${var.prefix}-${var.admin_pass_sm_secret_group}" : var.admin_pass_sm_secret_group existing_secret_group = var.use_existing_admin_pass_sm_secret_group @@ -296,20 +407,15 @@ locals { } ] }] - secrets = concat(local.service_credential_secrets, local.admin_pass_secret) - - existing_secrets_manager_instance_crn_split = var.existing_secrets_manager_instance_crn != null ? split(":", var.existing_secrets_manager_instance_crn) : null - existing_secrets_manager_instance_guid = var.existing_secrets_manager_instance_crn != null ? element(local.existing_secrets_manager_instance_crn_split, length(local.existing_secrets_manager_instance_crn_split) - 3) : null - existing_secrets_manager_instance_region = var.existing_secrets_manager_instance_crn != null ? element(local.existing_secrets_manager_instance_crn_split, length(local.existing_secrets_manager_instance_crn_split) - 5) : null - # tflint-ignore: terraform_unused_declarations - validate_sm_crn = length(local.service_credential_secrets) > 0 && var.existing_secrets_manager_instance_crn == null ? tobool("`existing_secrets_manager_instance_crn` is required when adding service credentials to a secrets manager secret.") : false - # tflint-ignore: terraform_unused_declarations - validate_sm_sg = var.existing_secrets_manager_instance_crn != null && var.admin_pass_sm_secret_group == null ? tobool("`admin_pass_sm_secret_group` is required when `existing_secrets_manager_instance_crn` is set.") : false - # tflint-ignore: terraform_unused_declarations - validate_sm_sn = var.existing_secrets_manager_instance_crn != null && var.admin_pass_sm_secret_name == null ? tobool("`admin_pass_sm_secret_name` is required when `existing_secrets_manager_instance_crn` is set.") : false + # Concatinate into 1 secrets object + secrets = concat(local.service_credential_secrets, local.admin_pass_secret) + # Parse Secrets Manager details from the CRN + existing_secrets_manager_instance_guid = var.existing_secrets_manager_instance_crn != null ? module.sm_instance_crn_parser[0].service_instance : null + existing_secrets_manager_instance_region = var.existing_secrets_manager_instance_crn != null ? module.sm_instance_crn_parser[0].region : null } +# Create secret(s) module "secrets_manager_service_credentials" { count = var.existing_secrets_manager_instance_crn == null ? 0 : 1 depends_on = [time_sleep.wait_for_es_authorization_policy] @@ -321,53 +427,22 @@ module "secrets_manager_service_credentials" { secrets = local.secrets } -# this extra block is needed when passing in an existing ES instance - the database data block -# requires a name and resource_id to retrieve the data -data "ibm_resource_instance" "existing_instance_resource" { - count = local.use_existing_db_instance ? 1 : 0 - identifier = local.existing_db_instance_guid -} - -data "ibm_database" "existing_db_instance" { - count = local.use_existing_db_instance ? 1 : 0 - name = data.ibm_resource_instance.existing_instance_resource[0].name - resource_group_id = data.ibm_resource_instance.existing_instance_resource[0].resource_group_id - location = var.region - service = "databases-for-elasticsearch" -} - -data "ibm_database_connection" "existing_connection" { - count = local.use_existing_db_instance ? 1 : 0 - endpoint_type = "private" - deployment_id = data.ibm_database.existing_db_instance[0].id - user_id = data.ibm_database.existing_db_instance[0].adminuser - user_type = "database" -} - ######################################################################################################################## # Code Engine Kibana Dashboard instance ######################################################################################################################## locals { - code_engine_project_id = var.existing_code_engine_project_id != null ? var.existing_code_engine_project_id : null code_engine_project_name = local.code_engine_project_id != null ? null : var.prefix != null ? "${var.prefix}-code-engine-kibana-project" : "ce-kibana-project" code_engine_app_name = var.prefix != null ? "${var.prefix}-kibana-app" : "ce-kibana-app" - - es_host = local.use_existing_db_instance ? data.ibm_database_connection.existing_connection[0].https[0].hosts[0].hostname : module.elasticsearch[0].hostname - es_port = local.use_existing_db_instance ? data.ibm_database_connection.existing_connection[0].https[0].hosts[0].port : module.elasticsearch[0].port - es_cert = local.use_existing_db_instance ? data.ibm_database_connection.existing_connection[0].https[0].certificate[0].certificate_base64 : module.elasticsearch[0].certificate_base64 - es_username = local.use_existing_db_instance ? data.ibm_database.existing_db_instance[0].adminuser : "admin" - es_password = local.admin_pass - es_data = var.enable_kibana_dashboard ? jsondecode(data.http.es_metadata[0].response_body) : null - es_full_version = var.enable_kibana_dashboard ? (var.elasticsearch_full_version != null ? var.elasticsearch_full_version : local.es_data.version.number) : null - + es_data = var.enable_kibana_dashboard ? jsondecode(data.http.es_metadata[0].response_body) : null + es_full_version = var.enable_kibana_dashboard ? (var.elasticsearch_full_version != null ? var.elasticsearch_full_version : local.es_data.version.number) : null } data "http" "es_metadata" { count = var.enable_kibana_dashboard ? 1 : 0 - url = "https://${local.es_username}:${local.es_password}@${local.es_host}:${local.es_port}" - ca_cert_pem = base64decode(local.es_cert) + url = "https://${local.elasticsearch_username}:${local.admin_pass}@${local.elasticsearch_hostname}:${local.elasticsearch_port}" + ca_cert_pem = base64decode(local.elasticsearch_cert) } module "code_engine_kibana" { @@ -381,7 +456,7 @@ module "code_engine_kibana" { "es-secret" = { format = "generic" data = { - "ELASTICSEARCH_PASSWORD" = local.es_password + "ELASTICSEARCH_PASSWORD" = local.admin_pass } } } @@ -393,12 +468,12 @@ module "code_engine_kibana" { run_env_variables = [{ type = "literal" name = "ELASTICSEARCH_HOSTS" - value = "[\"https://${local.es_host}:${local.es_port}\"]" + value = "[\"https://${local.elasticsearch_hostname}:${local.elasticsearch_port}\"]" }, { type = "literal" name = "ELASTICSEARCH_USERNAME" - value = local.es_username + value = local.elasticsearch_username }, { type = "secret_key_reference" diff --git a/solutions/standard/outputs.tf b/solutions/standard/outputs.tf index 62f1d12a..af0b8f86 100644 --- a/solutions/standard/outputs.tf +++ b/solutions/standard/outputs.tf @@ -4,7 +4,7 @@ output "id" { description = "Elasticsearch instance id" - value = local.use_existing_db_instance ? data.ibm_database.existing_db_instance[0].id : module.elasticsearch[0].id + value = local.elasticsearch_id } output "guid" { @@ -14,39 +14,34 @@ output "guid" { output "version" { description = "Elasticsearch instance version" - value = local.use_existing_db_instance ? data.ibm_database.existing_db_instance[0].version : module.elasticsearch[0].version + value = local.elasticsearch_version } output "crn" { description = "Elasticsearch instance crn" - value = local.use_existing_db_instance ? var.existing_db_instance_crn : module.elasticsearch[0].crn -} - -output "cbr_rule_ids" { - description = "CBR rule ids created to restrict Elasticsearch" - value = local.use_existing_db_instance ? null : module.elasticsearch[0].cbr_rule_ids + value = local.elasticsearch_crn } output "service_credentials_json" { description = "Service credentials json map" - value = local.use_existing_db_instance ? null : module.elasticsearch[0].service_credentials_json + value = var.existing_db_instance_crn != null ? null : module.elasticsearch[0].service_credentials_json sensitive = true } output "service_credentials_object" { description = "Service credentials object" - value = local.use_existing_db_instance ? null : module.elasticsearch[0].service_credentials_object + value = var.existing_db_instance_crn != null ? null : module.elasticsearch[0].service_credentials_object sensitive = true } output "hostname" { description = "Elasticsearch instance hostname" - value = local.use_existing_db_instance ? data.ibm_database_connection.existing_connection[0].https[0].hosts[0].hostname : module.elasticsearch[0].hostname + value = local.elasticsearch_hostname } output "port" { description = "Elasticsearch instance port" - value = local.use_existing_db_instance ? data.ibm_database_connection.existing_connection[0].https[0].hosts[0].port : module.elasticsearch[0].port + value = local.elasticsearch_port } output "secrets_manager_secrets" { diff --git a/solutions/standard/provider.tf b/solutions/standard/provider.tf index 91e435f9..515e60c2 100644 --- a/solutions/standard/provider.tf +++ b/solutions/standard/provider.tf @@ -7,6 +7,6 @@ provider "ibm" { provider "ibm" { alias = "kms" ibmcloud_api_key = var.ibmcloud_kms_api_key != null ? var.ibmcloud_kms_api_key : var.ibmcloud_api_key - region = local.existing_kms_instance_region + region = local.kms_region visibility = var.provider_visibility } diff --git a/solutions/standard/variables.tf b/solutions/standard/variables.tf index b169b2f0..b6378daf 100644 --- a/solutions/standard/variables.tf +++ b/solutions/standard/variables.tf @@ -8,13 +8,6 @@ variable "ibmcloud_api_key" { sensitive = true } -variable "ibmcloud_kms_api_key" { - type = string - description = "The IBM Cloud API key that can create a root key and key ring in the key management service (KMS) instance. If not specified, the 'ibmcloud_api_key' variable is used. Specify this key if the instance in `existing_kms_instance_crn` is in an account that's different from the Elastic Search instance. Leave this input empty if the same account owns both instances." - sensitive = true - default = null -} - variable "provider_visibility" { description = "Set the visibility value for the IBM terraform provider. Supported values are `public`, `private`, `public-and-private`. [Learn more](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/guides/custom-service-endpoints)." type = string @@ -67,7 +60,7 @@ variable "backup_crn" { variable "region" { type = string - description = "The region where you want to deploy your instance." + description = "The region where you want to deploy your instance, or the region in which your existing instance is in." default = "us-south" } @@ -212,21 +205,39 @@ variable "auto_scaling" { # Encryption ############################################################## +variable "use_ibm_owned_encryption_key" { + type = bool + description = "IBM Cloud Databases will secure your deployment's data at rest automatically with an encryption key that IBM hold. Alternatively, you may select your own Key Management System instance and encryption key (Key Protect or Hyper Protect Crypto Services) by setting this to false. If setting to false, a value must be passed for `existing_kms_instance_crn` to create a new key, or `existing_kms_key_crn` and/or `existing_backup_kms_key_crn` to use an existing key." + default = false +} + +variable "existing_kms_instance_crn" { + type = string + description = "The CRN of a Key Protect or Hyper Protect Crypto Services instance. Required to create a new encryption key and key ring which will be used to encrypt both deployment data and backups. Applies only if `use_ibm_owned_encryption_key` is false. To use an existing key, pass values for `existing_kms_key_crn` and/or `existing_backup_kms_key_crn`. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." + default = null +} + variable "existing_kms_key_crn" { type = string - description = "The CRN of a Hyper Protect Crypto Services or Key Protect root key to use for disk encryption. If not specified, a root key is created in the KMS instance specified in the `existing_kms_instance_crn` input. Backup encryption is only supported is some regions ([learn more](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok)), so if you need to use a key from a different region for backup encryption, use the `existing_backup_kms_key_crn` input." + description = "The CRN of a Key Protect or Hyper Protect Crypto Services encryption key to encrypt your data. Applies only if `use_ibm_owned_encryption_key` is false. By default this key is used for both deployment data and backups, but this behaviour can be altered using the optional `existing_backup_kms_key_crn` input. If no value is passed a new key will be created in the instance specified in the `existing_kms_instance_crn` input. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." default = null } -variable "skip_iam_authorization_policy" { +variable "existing_backup_kms_key_crn" { + type = string + description = "The CRN of a Key Protect or Hyper Protect Crypto Services encryption key that you want to use for encrypting the disk that holds deployment backups. Applies only if `use_ibm_owned_encryption_key` is false. If no value is passed, the value of `existing_kms_key_crn` is used. If no value is passed for `existing_kms_key_crn`, a new key will be created in the instance specified in the `existing_kms_instance_crn` input. Alternatively set `use_default_backup_encryption_key` to true to use the IBM Cloud Databases default encryption. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." + default = null +} + +variable "use_default_backup_encryption_key" { type = bool - description = "Whether to create an IAM authorization policy that permits all Databases for Elasticsearch instances in the resource group to read the encryption key from the Hyper Protect Crypto Services instance specified in the `existing_kms_instance_crn` variable." + description = "When `use_ibm_owned_encryption_key` is set to false, backups will be encrypted with either the key specified in `existing_kms_key_crn`, in `existing_backup_kms_key_crn`, or with a new key that will be created in the instance specified in the `existing_kms_instance_crn` input. If you do not want to use your own key for backups encryption, you can set this to `true` to use the IBM Cloud Databases default encryption for backups. Alternatively set `use_ibm_owned_encryption_key` to true to use the default encryption for both backups and deployment data." default = false } variable "kms_endpoint_type" { type = string - description = "The type of endpoint to use to communicate with the KMS instance. Possible values: `public`, `private`." + description = "The type of endpoint to use for communicating with the Key Protect or Hyper Protect Crypto Services instance. Possible values: `public`, `private`." default = "private" validation { condition = can(regex("public|private", var.kms_endpoint_type)) @@ -234,34 +245,31 @@ variable "kms_endpoint_type" { } } -variable "existing_kms_instance_crn" { - type = string - description = "The CRN of a Hyper Protect Crypto Services or Key Protect instance. Required to create a new root key if no value is passed with the `existing_kms_key_crn` input. Also required to create an authorization policy if `skip_iam_authorization_policy` is false. Backup encryption is only supported is some regions ([learn more](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok)), so if you need to use a different instance for backup encryption from a supported region, use the `existing_backup_kms_instance_crn` input." - default = null +variable "skip_es_kms_auth_policy" { + type = bool + description = "Set to true to skip the creation of IAM authorization policies that permits all Databases for Elasticsearch instances in the given resource group 'Reader' access to the Key Protect or Hyper Protect Crypto Services key. This policy is required in order to enable KMS encryption, so only skip creation if there is one already present in your account. No policy is created if `use_ibm_owned_encryption_key` is true." + default = false } -############################################################## -# DA KMS extras -############################################################## - variable "elasticsearch_key_ring_name" { type = string default = "elasticsearch-key-ring" description = "The name for the key ring created for the ElasticSearch key. Applies only if not specifying an existing key or using IBM owned keys. If a prefix input variable is specified, the prefix is added to the name in the `-` format." } -variable "use_ibm_owned_encryption_key" { - type = bool - description = "Set to true to use the default IBM Cloud® Databases randomly generated keys for disk and backups encryption." - default = false -} - variable "elasticsearch_key_name" { type = string default = "elasticsearch-key" description = "The name for the key created for the ElasticSearch key. Applies only if not specifying an existing key or using IBM owned keys. If a prefix input variable is specified, the prefix is added to the name in the `-` format." } +variable "ibmcloud_kms_api_key" { + type = string + description = "The IBM Cloud API key that can create a root key and key ring in the key management service (KMS) instance. If not specified, the 'ibmcloud_api_key' variable is used. Specify this key if the instance in `existing_kms_instance_crn` is in an account that's different from the Elastic Search instance. Leave this input empty if the same account owns both instances." + sensitive = true + default = null +} + ############################################################################## ## Secrets Manager Service Credentials ############################################################################## @@ -360,18 +368,3 @@ variable "elasticsearch_full_version" { type = string default = null } - -############################################################## -# Backup Encryption -############################################################## -variable "existing_backup_kms_key_crn" { - type = string - description = "The CRN of an Hyper Protect Crypto Services or Key Protect encryption key that you want to use to encrypt database backups. If no value is passed, the value of `existing_kms_key_crn` is used. If no value is passed for that, a new key will be created in the provided KMS instance and used for both disk encryption, and backup encryption." - default = null -} - -variable "existing_backup_kms_instance_crn" { - description = "The CRN of an Hyper Protect Crypto Services or Key Protect instance that you want to use to encrypt database backups. If no value is passed, the value of the `existing_kms_instance_crn` input will be used, however backup encryption is only supported in certain regions so you need to ensure the KMS for backup is coming from one of the supported regions. [Learn more](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok)" - type = string - default = null -} diff --git a/tests/pr_test.go b/tests/pr_test.go index 727e210f..c30f39f3 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -61,10 +61,9 @@ func TestRunFSCloudExample(t *testing.T) { */ //ResourceGroup: resourceGroup, TerraformVars: map[string]interface{}{ - "elasticsearch_version": latestVersion, // Always lock this test into the latest supported elasticsearch version - "access_tags": permanentResources["accessTags"], - "existing_kms_instance_guid": permanentResources["hpcs_south"], - "kms_key_crn": permanentResources["hpcs_south_root_key_crn"], + "elasticsearch_version": latestVersion, // Always lock this test into the latest supported elasticsearch version + "access_tags": permanentResources["accessTags"], + "kms_key_crn": permanentResources["hpcs_south_root_key_crn"], }, CloudInfoService: sharedInfoSvc, }) diff --git a/variables.tf b/variables.tf index 7a6e3184..92baf748 100644 --- a/variables.tf +++ b/variables.tf @@ -176,21 +176,15 @@ variable "auto_scaling" { # Encryption ############################################################## -variable "kms_encryption_enabled" { +variable "use_ibm_owned_encryption_key" { type = bool - description = "Whether to specify the keys used to encrypt data in the database. Specify `true` to identify the encryption keys. If set to `false`, the data is encrypted with randomly generated keys. [Learn more about Key Protect integration](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect). [Learn more about HPCS integration](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs)." - default = false -} - -variable "use_default_backup_encryption_key" { - type = bool - description = "Whether to use the IBM Cloud Databases generated keys for backup encryption." - default = false + description = "IBM Cloud Databases will secure your deployment's data at rest automatically with an encryption key that IBM hold. Alternatively, you may select your own Key Management System instance and encryption key (Key Protect or Hyper Protect Crypto Services) by setting this to false. If setting to false, a value must be passed for the `kms_key_crn` input." + default = true } variable "kms_key_crn" { type = string - description = "The root key CRN of the Key Protect or Hyper Protect Crypto Services instance to use for disk encryption. Applies only if `kms_encryption_enabled` is true." + description = "The CRN of a Key Protect or Hyper Protect Crypto Services encryption key to encrypt your data. Applies only if `use_ibm_owned_encryption_key` is false. By default this key is used for both deployment data and backups, but this behaviour can be altered using the `use_same_kms_key_for_backups` and `backup_encryption_key_crn` inputs. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." default = null validation { @@ -199,31 +193,41 @@ variable "kms_key_crn" { can(regex(".*kms.*", var.kms_key_crn)), can(regex(".*hs-crypto.*", var.kms_key_crn)), ]) - error_message = "Value must be the root key CRN of the Key Protect or Hyper Protect Crypto Services instance." + error_message = "Value must be the KMS key CRN from a Key Protect or Hyper Protect Crypto Services instance." } } +variable "use_same_kms_key_for_backups" { + type = bool + description = "Set this to false if you wan't to use a different key that you own to encrypt backups. When set to false, a value is required for the `backup_encryption_key_crn` input. Alternatiely set `use_default_backup_encryption_key` to true to use the IBM Cloud Databases default encryption. Applies only if `use_ibm_owned_encryption_key` is false. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." + default = true +} + variable "backup_encryption_key_crn" { type = string - description = "The CRN of a Hyper Protect Crypto Services use for encrypting the disk that holds deployment backups. There are limitation per region on the Hyper Protect Crypto Services and region for those services. See https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups" + description = "The CRN of a Key Protect or Hyper Protect Crypto Services encryption key that you want to use for encrypting the disk that holds deployment backups. Applies only if `use_ibm_owned_encryption_key` is false and `use_same_kms_key_for_backups` is false. If no value is passed, and `use_same_kms_key_for_backups` is true, the value of `kms_key_crn` is used. Alternatively set `use_default_backup_encryption_key` to true to use the IBM Cloud Databases default encryption. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." default = null validation { - condition = var.backup_encryption_key_crn == null ? true : length(regexall("^crn:v1:bluemix:public:kms:(us-south|us-east|eu-de):a/[[:xdigit:]]{32}:[[:xdigit:]]{8}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{12}:key:[[:xdigit:]]{8}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{12}$|^crn:v1:bluemix:public:hs-crypto:[a-z-]+:a/[[:xdigit:]]{32}:[[:xdigit:]]{8}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{12}:key:[[:xdigit:]]{8}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{12}$", var.backup_encryption_key_crn)) > 0 - error_message = "Valid values for backup_encryption_key_crn is null, a Hyper Protect Crypto Services key CRN or a Key Protect key CRN from us-south, us-east or eu-de" + condition = anytrue([ + var.backup_encryption_key_crn == null, + can(regex(".*kms.*", var.kms_key_crn)), + can(regex(".*hs-crypto.*", var.kms_key_crn)), + ]) + error_message = "Value must be the KMS key CRN from a Key Protect or Hyper Protect Crypto Services instance in one of the supported backup regions." } } -variable "skip_iam_authorization_policy" { +variable "use_default_backup_encryption_key" { type = bool - description = "Whether to create an IAM authorization policy that permits all Databases for Elasticsearch instances in the resource group to read the encryption key from the Hyper Protect Crypto Services instance specified in the `existing_kms_instance_guid` variable. If set to `false`, specify a value for the KMS instance in the `existing_kms_instance_guid` variable. No policy is created if `kms_encryption_enabled` is false." + description = "When `use_ibm_owned_encryption_key` is set to false, backups will be encrypted with either the key specified in `kms_key_crn`, or in `backup_encryption_key_crn` if a value is passed. If you do not want to use your own key for backups encryption, you can set this to `true` to use the IBM Cloud Databases default encryption for backups. Alternatively set `use_ibm_owned_encryption_key` to true to use the default encryption for both backups and deployment data." default = false } -variable "existing_kms_instance_guid" { - type = string - description = "The GUID of a Hyper Protect Crypto Services or Key Protect instance for the CRN specified in `kms_key_crn` and `backup_encryption_key_crn`. Applies only if `kms_encryption_enabled` is true, `skip_iam_authorization_policy` is false, and you specify values for `kms_key_crn` or `backup_encryption_key_crn`." - default = null +variable "skip_iam_authorization_policy" { + type = bool + description = "Set to true to skip the creation of IAM authorization policies that permits all Databases for Elasticsearch instances in the given resource group 'Reader' access to the Key Protect or Hyper Protect Crypto Services key that was provided in the `kms_key_crn` and `backup_encryption_key_crn` inputs. This policy is required in order to enable KMS encryption, so only skip creation if there is one already present in your account. No policy is created if `use_ibm_owned_encryption_key` is true." + default = false } ##############################################################