From 5f8ad6bb48c4474d509b9da00e5b29d6269cc23e Mon Sep 17 00:00:00 2001 From: Aatreyee Mukherjee Date: Fri, 18 Apr 2025 17:41:24 +0530 Subject: [PATCH 01/22] initial commit --- solutions/standard/main.tf | 1 + solutions/standard/variables.tf | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index 92c26140..cf76997e 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -463,6 +463,7 @@ module "code_engine_kibana" { (local.code_engine_app_name) = { image_reference = var.kibana_image_digest != null ? "${var.kibana_registry_namespace_image}@${var.kibana_image_digest}" : "${var.kibana_registry_namespace_image}:${local.kibana_version}" image_port = var.kibana_image_port + image_secret = var.image_secret run_env_variables = [{ type = "literal" name = "ELASTICSEARCH_HOSTS" diff --git a/solutions/standard/variables.tf b/solutions/standard/variables.tf index 6c7f985e..06b3786b 100644 --- a/solutions/standard/variables.tf +++ b/solutions/standard/variables.tf @@ -397,6 +397,12 @@ variable "kibana_image_port" { default = 5601 } +variable "image_secret" { + description = "The name of the image registry access secret." + type = string + default = null +} + variable "kibana_visibility" { description = "Specify the visibility of Kibana application in order to define which endpoint is available for receiving the requests. Valid values are 'local_public', 'local_private' and 'local' and it is only applicable if `enable_kibana_dashboard` is true. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/solutions/standard/DA-types.md#options-for-kibana_visibility)." type = string From 6175b855102a677b1c3f4a8e5915903019bdf012 Mon Sep 17 00:00:00 2001 From: Aatreyee Mukherjee Date: Mon, 21 Apr 2025 13:28:13 +0530 Subject: [PATCH 02/22] inital commit --- ibm_catalog.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ibm_catalog.json b/ibm_catalog.json index bbf504fb..947288ee 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -368,6 +368,9 @@ { "key": "kibana_image_port" }, + { + "key": "image_secret" + }, { "key": "kibana_visibility", "options": [ From 8416d6b2193179afdeefacd4aee7bf026a582897 Mon Sep 17 00:00:00 2001 From: Aatreyee Mukherjee Date: Mon, 12 May 2025 16:16:23 +0530 Subject: [PATCH 03/22] fixes --- ibm_catalog.json | 11 ++++++++++- solutions/standard/main.tf | 14 +++++++++++++- solutions/standard/variables.tf | 20 +++++++++++++++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/ibm_catalog.json b/ibm_catalog.json index 947288ee..01de65e6 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -369,7 +369,16 @@ "key": "kibana_image_port" }, { - "key": "image_secret" + "key": "kibana_image_secret" + }, + { + "key": "kibana_registry_personal_access_token" + }, + { + "key": "kibana_registry_server" + }, + { + "key": "kibana_registry_username" }, { "key": "kibana_visibility", diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index c299d248..4523725c 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -443,6 +443,18 @@ data "http" "es_metadata" { ca_cert_pem = base64decode(local.elasticsearch_cert) } +resource "ibm_code_engine_secret" "image_registry_secret" { + name = var.kibana_image_secret + project_id = var.existing_code_engine_project_id + format = "registry" + + data = { + username = var.kibana_registry_username + password = var.kibana_registry_personal_access_token + server = var.kibana_registry_server + } +} + module "code_engine_kibana" { count = var.enable_kibana_dashboard ? 1 : 0 source = "terraform-ibm-modules/code-engine/ibm" @@ -463,7 +475,7 @@ module "code_engine_kibana" { (local.code_engine_app_name) = { image_reference = var.kibana_image_digest != null ? "${var.kibana_registry_namespace_image}@${var.kibana_image_digest}" : "${var.kibana_registry_namespace_image}:${local.kibana_version}" image_port = var.kibana_image_port - image_secret = var.image_secret + image_secret = var.kibana_image_secret run_env_variables = [{ type = "literal" name = "ELASTICSEARCH_HOSTS" diff --git a/solutions/standard/variables.tf b/solutions/standard/variables.tf index 06b3786b..36c0669e 100644 --- a/solutions/standard/variables.tf +++ b/solutions/standard/variables.tf @@ -380,6 +380,12 @@ variable "kibana_registry_namespace_image" { default = "docker.elastic.co/kibana/kibana" } +variable "kibana_registry_server" { + type = string + description = "The server URL of the container registry used to pull the Kibana image." + default = "https://index.docker.io/v1/" +} + variable "kibana_image_digest" { type = string description = "When `enable_kibana_dashboard` is set to true, Kibana is deployed using an image tag compatible with the Elasticsearch version. Alternatively, an image digest in the format `sha256:xxxxx...` can also be specified but it must correspond to a version compatible with the Elasticsearch instance." @@ -391,13 +397,14 @@ variable "kibana_image_digest" { } + variable "kibana_image_port" { description = "Specify the port number used to connect to the Kibana service exposed by the container image. Default port is 5601 and it is only applicable if `enable_kibana_dashboard` is true" type = number default = 5601 } -variable "image_secret" { +variable "kibana_image_secret" { description = "The name of the image registry access secret." type = string default = null @@ -413,6 +420,17 @@ variable "kibana_visibility" { } } +variable "kibana_registry_username" { + description = "Username for the for the container registry." + type = string +} + +variable "kibana_registry_personal_access_token" { + description = "Pesonal access token for the container registry." + type = string + sensitive = true +} + ############################################################## # Context-based restriction (CBR) ############################################################## From b3fd5c7e949e75bbd0802afef7ce704be314bf18 Mon Sep 17 00:00:00 2001 From: Aatreyee Mukherjee Date: Wed, 4 Jun 2025 12:11:32 +0530 Subject: [PATCH 04/22] added resource_block to create secrets --- common-dev-assets | 2 +- ibm_catalog.json | 3 +++ solutions/standard/main.tf | 1 + solutions/standard/variables.tf | 12 +++++++++++ tests/pr_test.go | 38 +++++++++++++++++++++++++++++++-- 5 files changed, 53 insertions(+), 3 deletions(-) diff --git a/common-dev-assets b/common-dev-assets index 9fdee868..d0c2b2a1 160000 --- a/common-dev-assets +++ b/common-dev-assets @@ -1 +1 @@ -Subproject commit 9fdee86807c74a35061fc3327c2a01215897cb7f +Subproject commit d0c2b2a1d3cdf41f6a844bc17816199fe3f8b20f diff --git a/ibm_catalog.json b/ibm_catalog.json index 1fc7b06a..93493f88 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -358,6 +358,9 @@ { "key": "existing_code_engine_project_id" }, + { + "key": "use_existing_registry_secret" + }, { "key": "kibana_registry_namespace_image" }, diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index 4523725c..a332b2bd 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -444,6 +444,7 @@ data "http" "es_metadata" { } resource "ibm_code_engine_secret" "image_registry_secret" { + count = var.enable_kibana_dashboard && !var.use_existing_registry_secret ? 1 : 0 name = var.kibana_image_secret project_id = var.existing_code_engine_project_id format = "registry" diff --git a/solutions/standard/variables.tf b/solutions/standard/variables.tf index 36c0669e..4a430d79 100644 --- a/solutions/standard/variables.tf +++ b/solutions/standard/variables.tf @@ -346,6 +346,12 @@ variable "admin_pass_secrets_manager_secret_name" { default = "elasticsearch-admin-password" } +variable "use_existing_registry_secret" { + description = "Set to true to use an existing image registry secret instead of creating a new one." + type = bool + default = false +} + ############################################################## # Kibana Configuration ############################################################## @@ -408,6 +414,10 @@ variable "kibana_image_secret" { description = "The name of the image registry access secret." type = string default = null + validation { + condition = !var.enable_kibana_dashboard || var.use_existing_registry_secret || (var.kibana_image_secret != null && var.kibana_image_secret != "") + error_message = "You must provide a valid secret name for Kibana image registry access." + } } variable "kibana_visibility" { @@ -423,11 +433,13 @@ variable "kibana_visibility" { variable "kibana_registry_username" { description = "Username for the for the container registry." type = string + default = null } variable "kibana_registry_personal_access_token" { description = "Pesonal access token for the container registry." type = string + default = null sensitive = true } diff --git a/tests/pr_test.go b/tests/pr_test.go index 9925204a..428895b1 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -61,6 +61,8 @@ func TestMain(m *testing.M) { func TestRunStandardSolutionSchematics(t *testing.T) { t.Parallel() + enableKibana := false + options := testschematic.TestSchematicOptionsDefault(&testschematic.TestSchematicOptions{ Testing: t, TarIncludePatterns: []string{ @@ -93,7 +95,6 @@ func TestRunStandardSolutionSchematics(t *testing.T) { }, }, } - options.TerraformVars = []testschematic.TestSchematicTerraformVar{ {Name: "ibmcloud_api_key", Value: options.RequiredEnvironmentVars["TF_VAR_ibmcloud_api_key"], DataType: "string", Secure: true}, {Name: "access_tags", Value: permanentResources["accessTags"], DataType: "list(string)"}, @@ -109,11 +110,44 @@ func TestRunStandardSolutionSchematics(t *testing.T) { {Name: "admin_pass", Value: GetRandomAdminPassword(t), DataType: "string"}, {Name: "admin_pass_secrets_manager_secret_group", Value: options.Prefix, DataType: "string"}, {Name: "admin_pass_secrets_manager_secret_name", Value: options.Prefix, DataType: "string"}, - {Name: "enable_kibana_dashboard", Value: true, DataType: "bool"}, + {Name: "enable_kibana_dashboard", Value: enableKibana, DataType: "bool"}, {Name: "provider_visibility", Value: "private", DataType: "string"}, {Name: "prefix", Value: options.Prefix, DataType: "string"}, {Name: "admin_pass", Value: GetRandomAdminPassword(t), DataType: "string"}, } + + if enableKibana { + existingProjectID := os.Getenv("EXISTING_CODE_ENGINE_PROJECT_ID") + kibanaImageSecret := os.Getenv("KIBANA_IMAGE_SECRET") + kibanaRegistryUsername := os.Getenv("KIBANA_REGISTRY_USERNAME") + kibanaRegistryToken := os.Getenv("KIBANA_REGISTRY_PERSONAL_ACCESS_TOKEN") + kibanaRegistryServer := os.Getenv("KIBANA_REGISTRY_SERVER") + + if existingProjectID == "" { + t.Fatal("existing_code_engine_project_id env var must be set when enable_kibana_dashboard is true") + } + if kibanaImageSecret == "" { + t.Fatal("kibana_image_secret env var must be set when enable_kibana_dashboard is true") + } + if kibanaRegistryUsername == "" { + t.Fatal("kibana_registry_username env var must be set when enable_kibana_dashboard is true") + } + if kibanaRegistryToken == "" { + t.Fatal("kibana_personal_access_token env var must be set when enable_kibana_dashboard is true") + } + if kibanaRegistryServer == "" { + t.Fatal("kibana_registry_server env var must be set when enable_kibana_dashboard is true") + } + + options.TerraformVars = append(options.TerraformVars, + testschematic.TestSchematicTerraformVar{Name: "existing_code_engine_project_id", Value: existingProjectID, DataType: "string"}, + testschematic.TestSchematicTerraformVar{Name: "kibana_image_secret", Value: kibanaImageSecret, DataType: "string"}, + testschematic.TestSchematicTerraformVar{Name: "kibana_registry_username", Value: kibanaRegistryUsername, DataType: "string"}, + testschematic.TestSchematicTerraformVar{Name: "kibana_registry_personal_access_token", Value: kibanaRegistryToken, DataType: "string"}, + testschematic.TestSchematicTerraformVar{Name: "kibana_registry_server", Value: kibanaRegistryServer, DataType: "string"}, + ) + } + err := options.RunSchematicTest() assert.Nil(t, err, "This should not have errored") } From 69166062b50ef9f7c2755c161fbc9725dc314b56 Mon Sep 17 00:00:00 2001 From: aatreyee257 Date: Wed, 4 Jun 2025 13:18:13 +0530 Subject: [PATCH 05/22] added support for existin secret --- solutions/standard/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index df2d1548..1828dfa6 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -457,7 +457,7 @@ module "code_engine_kibana" { (local.code_engine_app_name) = { image_reference = var.kibana_image_digest != null ? "${var.kibana_registry_namespace_image}@${var.kibana_image_digest}" : "${var.kibana_registry_namespace_image}:${local.kibana_version}" image_port = var.kibana_image_port - image_secret = var.kibana_image_secret + image_secret = var.use_existing_registry_secret ? var.kibana_image_secret : ibm_code_engine_secret.image_registry_secret[0].name run_env_variables = [{ type = "literal" name = "ELASTICSEARCH_HOSTS" From 912fddd523d69f1a62ecc12ea86c9053ec3940a1 Mon Sep 17 00:00:00 2001 From: aatreyee257 Date: Wed, 4 Jun 2025 20:32:46 +0530 Subject: [PATCH 06/22] fixed logic --- solutions/standard/variables.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/solutions/standard/variables.tf b/solutions/standard/variables.tf index d2084983..7bf72737 100644 --- a/solutions/standard/variables.tf +++ b/solutions/standard/variables.tf @@ -475,10 +475,10 @@ variable "kibana_image_secret" { description = "The name of the image registry access secret." type = string default = null - validation { - condition = !var.enable_kibana_dashboard || var.use_existing_registry_secret || (var.kibana_image_secret != null && var.kibana_image_secret != "") - error_message = "You must provide a valid secret name for Kibana image registry access." - } + # validation { + # condition = !var.enable_kibana_dashboard || var.use_existing_registry_secret || (var.kibana_image_secret != null && length(var.kibana_image_secret)>0 ) + # error_message = "You must provide a valid secret name for Kibana image registry access." + # } } variable "kibana_visibility" { From a8bcf52f34c854c72ca92aaaba1cee1aadd6525d Mon Sep 17 00:00:00 2001 From: aatreyee257 Date: Thu, 5 Jun 2025 13:12:29 +0530 Subject: [PATCH 07/22] changes --- solutions/standard/main.tf | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index 1828dfa6..07b9dd62 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -424,10 +424,13 @@ data "http" "es_metadata" { ca_cert_pem = base64decode(local.elasticsearch_cert) } -resource "ibm_code_engine_secret" "image_registry_secret" { - count = var.enable_kibana_dashboard && !var.use_existing_registry_secret ? 1 : 0 +module "image_registry_secret" { + count = var.enable_kibana_dashboard && !var.use_existing_registry_secret ? 1 : 0 + source = "terraform-ibm-modules/code-engine/ibm//modules/secret" + version = "4.2.4" + name = var.kibana_image_secret - project_id = var.existing_code_engine_project_id + project_id = local.code_engine_project_id format = "registry" data = { @@ -437,6 +440,7 @@ resource "ibm_code_engine_secret" "image_registry_secret" { } } + module "code_engine_kibana" { count = var.enable_kibana_dashboard ? 1 : 0 source = "terraform-ibm-modules/code-engine/ibm" @@ -457,7 +461,7 @@ module "code_engine_kibana" { (local.code_engine_app_name) = { image_reference = var.kibana_image_digest != null ? "${var.kibana_registry_namespace_image}@${var.kibana_image_digest}" : "${var.kibana_registry_namespace_image}:${local.kibana_version}" image_port = var.kibana_image_port - image_secret = var.use_existing_registry_secret ? var.kibana_image_secret : ibm_code_engine_secret.image_registry_secret[0].name + image_secret = var.use_existing_registry_secret ? var.kibana_image_secret : module.image_registry_secret[0].name run_env_variables = [{ type = "literal" name = "ELASTICSEARCH_HOSTS" From ce29449ffd075dd359ce1d6e7b358d583cf8b0bf Mon Sep 17 00:00:00 2001 From: aatreyee257 Date: Thu, 5 Jun 2025 13:18:32 +0530 Subject: [PATCH 08/22] changes --- solutions/standard/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index 07b9dd62..47f0f9ae 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -424,7 +424,7 @@ data "http" "es_metadata" { ca_cert_pem = base64decode(local.elasticsearch_cert) } -module "image_registry_secret" { +module "secret" { count = var.enable_kibana_dashboard && !var.use_existing_registry_secret ? 1 : 0 source = "terraform-ibm-modules/code-engine/ibm//modules/secret" version = "4.2.4" @@ -461,7 +461,7 @@ module "code_engine_kibana" { (local.code_engine_app_name) = { image_reference = var.kibana_image_digest != null ? "${var.kibana_registry_namespace_image}@${var.kibana_image_digest}" : "${var.kibana_registry_namespace_image}:${local.kibana_version}" image_port = var.kibana_image_port - image_secret = var.use_existing_registry_secret ? var.kibana_image_secret : module.image_registry_secret[0].name + image_secret = var.use_existing_registry_secret ? var.kibana_image_secret : module.secret[0].name run_env_variables = [{ type = "literal" name = "ELASTICSEARCH_HOSTS" From 860e5ee888923b58edab7caa787ba828e0d72e13 Mon Sep 17 00:00:00 2001 From: aatreyee257 Date: Tue, 10 Jun 2025 09:20:50 +0530 Subject: [PATCH 09/22] fixed error --- tests/pr_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pr_test.go b/tests/pr_test.go index 428895b1..1d7d827f 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -314,7 +314,7 @@ func TestPlanValidation(t *testing.T) { // Test the DA when using Kibana dashboard and existing KMS instance var standardSolutionWithKibanaDashboardVars = map[string]interface{}{ - "enable_kibana_dashboard": true, + "enable_kibana_dashboard": false, "existing_kms_instance_crn": permanentResources["hpcs_south_crn"], "plan": "enterprise", } From 461cce196d30048a8c272a9e8e6ae4465438a415 Mon Sep 17 00:00:00 2001 From: aatreyee257 Date: Thu, 12 Jun 2025 21:11:03 +0530 Subject: [PATCH 10/22] added condition and updated provider version --- README.md | 2 +- examples/backup-restore/version.tf | 2 +- examples/basic/version.tf | 2 +- examples/complete/version.tf | 2 +- solutions/standard/variables.tf | 4 --- solutions/standard/version.tf | 2 +- tests/pr_test.go | 43 ++++++++++++++++++++++++------ version.tf | 2 +- 8 files changed, 41 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index bb2b53cc..dcab2f25 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ You need the following permissions to run this module. | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.9.0 | -| [ibm](#requirement\_ibm) | >= 1.70.0, <2.0.0 | +| [ibm](#requirement\_ibm) | >= 1.79.0, <2.0.0 | | [null](#requirement\_null) | >= 3.2.1, < 4.0.0 | | [time](#requirement\_time) | >= 0.9.1 | diff --git a/examples/backup-restore/version.tf b/examples/backup-restore/version.tf index 6f39952e..3419af8f 100644 --- a/examples/backup-restore/version.tf +++ b/examples/backup-restore/version.tf @@ -5,7 +5,7 @@ terraform { # module's version.tf (basic example), and 1 example that will always use the latest provider version (complete example). ibm = { source = "IBM-Cloud/ibm" - version = ">=1.70.0, <2.0.0" + version = ">=1.79.0, <2.0.0" } } } diff --git a/examples/basic/version.tf b/examples/basic/version.tf index 509f9700..0ed82b08 100644 --- a/examples/basic/version.tf +++ b/examples/basic/version.tf @@ -4,7 +4,7 @@ terraform { # Pin to the lowest provider version of the range defined in the main module's version.tf to ensure lowest version still works ibm = { source = "IBM-Cloud/ibm" - version = "1.70.0" + version = "1.79.0" } # The elasticsearch provider is not actually required by the module itself, just this example, so OK to use ">=" here instead of locking into a version elasticsearch = { diff --git a/examples/complete/version.tf b/examples/complete/version.tf index c07b9389..0ce61882 100644 --- a/examples/complete/version.tf +++ b/examples/complete/version.tf @@ -4,7 +4,7 @@ terraform { required_providers { ibm = { source = "IBM-Cloud/ibm" - version = ">=1.70.0, <2.0.0" + version = ">=1.79.0, <2.0.0" } } } diff --git a/solutions/standard/variables.tf b/solutions/standard/variables.tf index 7bf72737..4ced6018 100644 --- a/solutions/standard/variables.tf +++ b/solutions/standard/variables.tf @@ -475,10 +475,6 @@ variable "kibana_image_secret" { description = "The name of the image registry access secret." type = string default = null - # validation { - # condition = !var.enable_kibana_dashboard || var.use_existing_registry_secret || (var.kibana_image_secret != null && length(var.kibana_image_secret)>0 ) - # error_message = "You must provide a valid secret name for Kibana image registry access." - # } } variable "kibana_visibility" { diff --git a/solutions/standard/version.tf b/solutions/standard/version.tf index 88d2a852..c96df0a6 100644 --- a/solutions/standard/version.tf +++ b/solutions/standard/version.tf @@ -5,7 +5,7 @@ terraform { required_providers { ibm = { source = "IBM-Cloud/ibm" - version = "1.78.3" + version = "1.79.0" } time = { source = "hashicorp/time" diff --git a/tests/pr_test.go b/tests/pr_test.go index 1d7d827f..41eddf55 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -313,10 +313,38 @@ func TestPlanValidation(t *testing.T) { } // Test the DA when using Kibana dashboard and existing KMS instance - var standardSolutionWithKibanaDashboardVars = map[string]interface{}{ - "enable_kibana_dashboard": false, - "existing_kms_instance_crn": permanentResources["hpcs_south_crn"], - "plan": "enterprise", + + tfVarsMap := map[string]map[string]interface{}{} + + if os.Getenv("ENABLE_KIBANA_DASHBOARD") == "true" { + tfVarsMap["standardSolutionWithKibanaDashboardVars"] = map[string]interface{}{ + "existing_kms_instance_crn": permanentResources["hpcs_south_crn"], + "kibana_image_secret": os.Getenv("KIBANA_IMAGE_SECRET"), + "kibana_registry_username": os.Getenv("KIBANA_REGISTRY_USERNAME"), + "kibana_registry_personal_access_token": os.Getenv("KIBANA_REGISTRY_PERSONAL_ACCESS_TOKEN"), + "kibana_registry_server": os.Getenv("KIBANA_REGISTRY_SERVER"), + "existingProjectID": os.Getenv("EXISTING_CODE_ENGINE_PROJECT_ID"), + "plan": "enterprise", + } + } + + for name, tfVars := range tfVarsMap { + t.Run(name, func(t *testing.T) { + for key, value := range tfVars { + options.TerraformOptions.Vars[key] = value + } + + _, err := terraform.InitE(t, options.TerraformOptions) + require.NoError(t, err) + + output, err := terraform.PlanE(t, options.TerraformOptions) + assert.Nil(t, err) + assert.NotNil(t, output) + + for key := range tfVars { + delete(options.TerraformOptions.Vars, key) + } + }) } // Test the DA when using IBM owned encryption key @@ -325,10 +353,9 @@ func TestPlanValidation(t *testing.T) { } // Create a map of the variables - tfVarsMap := map[string]map[string]interface{}{ - "standardSolutionWithElserModelVars": standardSolutionWithElserModelVars, - "standardSolutionWithKibanaDashboardVars": standardSolutionWithKibanaDashboardVars, - "standardSolutionWithUseIbmOwnedEncKey": standardSolutionWithUseIbmOwnedEncKey, + tfVarsMap = map[string]map[string]interface{}{ + "standardSolutionWithElserModelVars": standardSolutionWithElserModelVars, + "standardSolutionWithUseIbmOwnedEncKey": standardSolutionWithUseIbmOwnedEncKey, } _, initErr := terraform.InitE(t, options.TerraformOptions) diff --git a/version.tf b/version.tf index 0f6877bf..55b757f6 100644 --- a/version.tf +++ b/version.tf @@ -4,7 +4,7 @@ terraform { required_providers { ibm = { source = "ibm-cloud/ibm" - version = ">= 1.70.0, <2.0.0" + version = ">= 1.79.0, <2.0.0" } null = { source = "hashicorp/null" From ad0bccab73364188943b6375a81e6f6cd7c14665 Mon Sep 17 00:00:00 2001 From: aatreyee257 Date: Mon, 16 Jun 2025 10:57:54 +0530 Subject: [PATCH 11/22] provider version update --- README.md | 2 +- examples/backup-restore/version.tf | 2 +- examples/basic/version.tf | 2 +- examples/complete/version.tf | 2 +- examples/fscloud/version.tf | 2 +- solutions/standard/version.tf | 2 +- version.tf | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index dcab2f25..32567b0b 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ You need the following permissions to run this module. | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.9.0 | -| [ibm](#requirement\_ibm) | >= 1.79.0, <2.0.0 | +| [ibm](#requirement\_ibm) | >= 1.79.1, <2.0.0 | | [null](#requirement\_null) | >= 3.2.1, < 4.0.0 | | [time](#requirement\_time) | >= 0.9.1 | diff --git a/examples/backup-restore/version.tf b/examples/backup-restore/version.tf index 3419af8f..b211acf0 100644 --- a/examples/backup-restore/version.tf +++ b/examples/backup-restore/version.tf @@ -5,7 +5,7 @@ terraform { # module's version.tf (basic example), and 1 example that will always use the latest provider version (complete example). ibm = { source = "IBM-Cloud/ibm" - version = ">=1.79.0, <2.0.0" + version = ">=1.79.2, <2.0.0" } } } diff --git a/examples/basic/version.tf b/examples/basic/version.tf index 0ed82b08..5677e7b1 100644 --- a/examples/basic/version.tf +++ b/examples/basic/version.tf @@ -4,7 +4,7 @@ terraform { # Pin to the lowest provider version of the range defined in the main module's version.tf to ensure lowest version still works ibm = { source = "IBM-Cloud/ibm" - version = "1.79.0" + version = "1.79.2" } # The elasticsearch provider is not actually required by the module itself, just this example, so OK to use ">=" here instead of locking into a version elasticsearch = { diff --git a/examples/complete/version.tf b/examples/complete/version.tf index 0ce61882..3f0c1bab 100644 --- a/examples/complete/version.tf +++ b/examples/complete/version.tf @@ -4,7 +4,7 @@ terraform { required_providers { ibm = { source = "IBM-Cloud/ibm" - version = ">=1.79.0, <2.0.0" + version = ">=1.79.2, <2.0.0" } } } diff --git a/examples/fscloud/version.tf b/examples/fscloud/version.tf index cdc76d38..03d8bdf3 100644 --- a/examples/fscloud/version.tf +++ b/examples/fscloud/version.tf @@ -4,7 +4,7 @@ terraform { # Use latest version of provider in non-basic examples to verify latest version works with module ibm = { source = "IBM-Cloud/ibm" - version = ">= 1.70.0" + version = ">= 1.79.2" } } } diff --git a/solutions/standard/version.tf b/solutions/standard/version.tf index c96df0a6..b7118580 100644 --- a/solutions/standard/version.tf +++ b/solutions/standard/version.tf @@ -5,7 +5,7 @@ terraform { required_providers { ibm = { source = "IBM-Cloud/ibm" - version = "1.79.0" + version = "1.79.2" } time = { source = "hashicorp/time" diff --git a/version.tf b/version.tf index 55b757f6..5391ac4d 100644 --- a/version.tf +++ b/version.tf @@ -4,7 +4,7 @@ terraform { required_providers { ibm = { source = "ibm-cloud/ibm" - version = ">= 1.79.0, <2.0.0" + version = ">= 1.79.1, <2.0.0" } null = { source = "hashicorp/null" From 9f09893bed534e5de819e2e01de735b0dc5a3bed Mon Sep 17 00:00:00 2001 From: aatreyee257 Date: Mon, 23 Jun 2025 22:22:03 +0530 Subject: [PATCH 12/22] changes --- ibm_catalog.json | 3 ++ solutions/standard/main.tf | 4 +- solutions/standard/variables.tf | 6 +++ tests/pr_test.go | 79 ++++----------------------------- 4 files changed, 20 insertions(+), 72 deletions(-) diff --git a/ibm_catalog.json b/ibm_catalog.json index 0164b864..511529fa 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -383,6 +383,9 @@ { "key": "kibana_registry_username" }, + { + "key": "use_private_registry" + }, { "key": "kibana_visibility", "options": [ diff --git a/solutions/standard/main.tf b/solutions/standard/main.tf index 47f0f9ae..88db2ae5 100644 --- a/solutions/standard/main.tf +++ b/solutions/standard/main.tf @@ -425,7 +425,7 @@ data "http" "es_metadata" { } module "secret" { - count = var.enable_kibana_dashboard && !var.use_existing_registry_secret ? 1 : 0 + count = var.use_private_registry && !var.use_existing_registry_secret ? 1 : 0 source = "terraform-ibm-modules/code-engine/ibm//modules/secret" version = "4.2.4" @@ -461,7 +461,7 @@ module "code_engine_kibana" { (local.code_engine_app_name) = { image_reference = var.kibana_image_digest != null ? "${var.kibana_registry_namespace_image}@${var.kibana_image_digest}" : "${var.kibana_registry_namespace_image}:${local.kibana_version}" image_port = var.kibana_image_port - image_secret = var.use_existing_registry_secret ? var.kibana_image_secret : module.secret[0].name + image_secret = var.use_private_registry ? (var.use_existing_registry_secret ? var.kibana_image_secret : module.secret[0].name) : null run_env_variables = [{ type = "literal" name = "ELASTICSEARCH_HOSTS" diff --git a/solutions/standard/variables.tf b/solutions/standard/variables.tf index 4ced6018..18e515f2 100644 --- a/solutions/standard/variables.tf +++ b/solutions/standard/variables.tf @@ -438,6 +438,12 @@ variable "existing_code_engine_project_id" { variable "enable_kibana_dashboard" { type = bool description = "Set to true to deploy Kibana in Code Engine. NOTE: By default, the Kibana image will be pulled from the official Elastic registry (docker.elastic.co) and is not certified by IBM, however this can be overridden using the `kibana_registry_namespace_image` and `kibana_image_digest` inputs." + default = true +} + +variable "use_private_registry" { + description = "Set to true if the Kibana image is being pulled from a private registry." + type = bool default = false } diff --git a/tests/pr_test.go b/tests/pr_test.go index 41eddf55..98877973 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -61,8 +61,6 @@ func TestMain(m *testing.M) { func TestRunStandardSolutionSchematics(t *testing.T) { t.Parallel() - enableKibana := false - options := testschematic.TestSchematicOptionsDefault(&testschematic.TestSchematicOptions{ Testing: t, TarIncludePatterns: []string{ @@ -110,44 +108,12 @@ func TestRunStandardSolutionSchematics(t *testing.T) { {Name: "admin_pass", Value: GetRandomAdminPassword(t), DataType: "string"}, {Name: "admin_pass_secrets_manager_secret_group", Value: options.Prefix, DataType: "string"}, {Name: "admin_pass_secrets_manager_secret_name", Value: options.Prefix, DataType: "string"}, - {Name: "enable_kibana_dashboard", Value: enableKibana, DataType: "bool"}, + {Name: "enable_kibana_dashboard", Value: true, DataType: "bool"}, {Name: "provider_visibility", Value: "private", DataType: "string"}, {Name: "prefix", Value: options.Prefix, DataType: "string"}, {Name: "admin_pass", Value: GetRandomAdminPassword(t), DataType: "string"}, } - if enableKibana { - existingProjectID := os.Getenv("EXISTING_CODE_ENGINE_PROJECT_ID") - kibanaImageSecret := os.Getenv("KIBANA_IMAGE_SECRET") - kibanaRegistryUsername := os.Getenv("KIBANA_REGISTRY_USERNAME") - kibanaRegistryToken := os.Getenv("KIBANA_REGISTRY_PERSONAL_ACCESS_TOKEN") - kibanaRegistryServer := os.Getenv("KIBANA_REGISTRY_SERVER") - - if existingProjectID == "" { - t.Fatal("existing_code_engine_project_id env var must be set when enable_kibana_dashboard is true") - } - if kibanaImageSecret == "" { - t.Fatal("kibana_image_secret env var must be set when enable_kibana_dashboard is true") - } - if kibanaRegistryUsername == "" { - t.Fatal("kibana_registry_username env var must be set when enable_kibana_dashboard is true") - } - if kibanaRegistryToken == "" { - t.Fatal("kibana_personal_access_token env var must be set when enable_kibana_dashboard is true") - } - if kibanaRegistryServer == "" { - t.Fatal("kibana_registry_server env var must be set when enable_kibana_dashboard is true") - } - - options.TerraformVars = append(options.TerraformVars, - testschematic.TestSchematicTerraformVar{Name: "existing_code_engine_project_id", Value: existingProjectID, DataType: "string"}, - testschematic.TestSchematicTerraformVar{Name: "kibana_image_secret", Value: kibanaImageSecret, DataType: "string"}, - testschematic.TestSchematicTerraformVar{Name: "kibana_registry_username", Value: kibanaRegistryUsername, DataType: "string"}, - testschematic.TestSchematicTerraformVar{Name: "kibana_registry_personal_access_token", Value: kibanaRegistryToken, DataType: "string"}, - testschematic.TestSchematicTerraformVar{Name: "kibana_registry_server", Value: kibanaRegistryServer, DataType: "string"}, - ) - } - err := options.RunSchematicTest() assert.Nil(t, err, "This should not have errored") } @@ -313,38 +279,10 @@ func TestPlanValidation(t *testing.T) { } // Test the DA when using Kibana dashboard and existing KMS instance - - tfVarsMap := map[string]map[string]interface{}{} - - if os.Getenv("ENABLE_KIBANA_DASHBOARD") == "true" { - tfVarsMap["standardSolutionWithKibanaDashboardVars"] = map[string]interface{}{ - "existing_kms_instance_crn": permanentResources["hpcs_south_crn"], - "kibana_image_secret": os.Getenv("KIBANA_IMAGE_SECRET"), - "kibana_registry_username": os.Getenv("KIBANA_REGISTRY_USERNAME"), - "kibana_registry_personal_access_token": os.Getenv("KIBANA_REGISTRY_PERSONAL_ACCESS_TOKEN"), - "kibana_registry_server": os.Getenv("KIBANA_REGISTRY_SERVER"), - "existingProjectID": os.Getenv("EXISTING_CODE_ENGINE_PROJECT_ID"), - "plan": "enterprise", - } - } - - for name, tfVars := range tfVarsMap { - t.Run(name, func(t *testing.T) { - for key, value := range tfVars { - options.TerraformOptions.Vars[key] = value - } - - _, err := terraform.InitE(t, options.TerraformOptions) - require.NoError(t, err) - - output, err := terraform.PlanE(t, options.TerraformOptions) - assert.Nil(t, err) - assert.NotNil(t, output) - - for key := range tfVars { - delete(options.TerraformOptions.Vars, key) - } - }) + var standardSolutionWithKibanaDashboardVars = map[string]interface{}{ + "enable_kibana_dashboard": true, + "existing_kms_instance_crn": permanentResources["hpcs_south_crn"], + "plan": "enterprise", } // Test the DA when using IBM owned encryption key @@ -353,9 +291,10 @@ func TestPlanValidation(t *testing.T) { } // Create a map of the variables - tfVarsMap = map[string]map[string]interface{}{ - "standardSolutionWithElserModelVars": standardSolutionWithElserModelVars, - "standardSolutionWithUseIbmOwnedEncKey": standardSolutionWithUseIbmOwnedEncKey, + tfVarsMap := map[string]map[string]interface{}{ + "standardSolutionWithElserModelVars": standardSolutionWithElserModelVars, + "standardSolutionWithKibanaDashboardVars": standardSolutionWithKibanaDashboardVars, + "standardSolutionWithUseIbmOwnedEncKey": standardSolutionWithUseIbmOwnedEncKey, } _, initErr := terraform.InitE(t, options.TerraformOptions) From 2cb1fe4ee3b24674544a82c4d298d92dcf8c5b18 Mon Sep 17 00:00:00 2001 From: whoffler Date: Fri, 27 Jun 2025 14:07:01 +0100 Subject: [PATCH 13/22] feat: Full DA refactor (#428) BREAKING CHANGE: There is no supported upgrade path from `standard` to the new `Fully configurable` or `Security-enforced` DA variations --- .catalog-onboard-pipeline.yaml | 10 +- .secrets.baseline | 14 +- README.md | 22 +- common-dev-assets | 2 +- cra-config.yaml | 8 +- examples/basic/variables.tf | 1 + examples/complete/main.tf | 2 +- ibm_catalog.json | 530 +++++++++++++++--- main.tf | 44 +- modules/fscloud/main.tf | 6 +- .../deployable-architecture-elasticsearch.svg | 4 +- .../DA-cbr_rules.md | 0 .../DA-types.md | 0 solutions/fully-configurable/README.md | 3 + .../catalogValidationValues.json.template | 9 + .../{standard => fully-configurable}/main.tf | 100 ++-- .../outputs.tf | 26 +- solutions/fully-configurable/provider.tf | 14 + .../variables.tf | 309 +++++----- .../version.tf | 0 solutions/security-enforced/README.md | 3 + .../catalogValidationValues.json.template | 4 +- solutions/security-enforced/main.tf | 58 ++ solutions/security-enforced/outputs.tf | 61 ++ solutions/security-enforced/provider.tf | 1 + solutions/security-enforced/variables.tf | 435 ++++++++++++++ solutions/security-enforced/version.tf | 6 + solutions/standard/README.md | 18 - solutions/standard/moved.tf | 4 - solutions/standard/provider.tf | 12 - tests/pr_test.go | 140 +++-- variables.tf | 6 +- version.tf | 6 +- 33 files changed, 1452 insertions(+), 406 deletions(-) rename solutions/{standard => fully-configurable}/DA-cbr_rules.md (100%) rename solutions/{standard => fully-configurable}/DA-types.md (100%) create mode 100644 solutions/fully-configurable/README.md create mode 100644 solutions/fully-configurable/catalogValidationValues.json.template rename solutions/{standard => fully-configurable}/main.tf (75%) rename solutions/{standard => fully-configurable}/outputs.tf (75%) create mode 100644 solutions/fully-configurable/provider.tf rename solutions/{standard => fully-configurable}/variables.tf (70%) rename solutions/{standard => fully-configurable}/version.tf (100%) create mode 100644 solutions/security-enforced/README.md rename solutions/{standard => security-enforced}/catalogValidationValues.json.template (61%) create mode 100644 solutions/security-enforced/main.tf create mode 100644 solutions/security-enforced/outputs.tf create mode 100644 solutions/security-enforced/provider.tf create mode 100644 solutions/security-enforced/variables.tf create mode 100644 solutions/security-enforced/version.tf delete mode 100644 solutions/standard/README.md delete mode 100644 solutions/standard/moved.tf delete mode 100644 solutions/standard/provider.tf diff --git a/.catalog-onboard-pipeline.yaml b/.catalog-onboard-pipeline.yaml index 9a791ad9..7cb4fa35 100644 --- a/.catalog-onboard-pipeline.yaml +++ b/.catalog-onboard-pipeline.yaml @@ -6,9 +6,17 @@ offerings: catalog_id: 7df1e4ca-d54c-4fd0-82ce-3d13247308cd offering_id: 7ee5876d-6e30-49d1-be25-259a442085e8 variations: - - name: standard + - name: fully-configurable mark_ready: true install_type: fullstack scc: instance_id: 1c7d5f78-9262-44c3-b779-b28fe4d88c37 region: us-south + scope_resource_group_var_name: existing_resource_group_name + - name: security-enforced + mark_ready: true + install_type: fullstack + scc: + instance_id: 1c7d5f78-9262-44c3-b779-b28fe4d88c37 + region: us-south + scope_resource_group_var_name: existing_resource_group_name diff --git a/.secrets.baseline b/.secrets.baseline index a933ea3d..88ee2d53 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "go.sum|^.secrets.baseline$", "lines": null }, - "generated_at": "2025-03-02T09:01:42Z", + "generated_at": "2025-06-24T13:00:25Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -87,7 +87,7 @@ "verified_result": null } ], - "solutions/standard/DA-types.md": [ + "solutions/fully-configurable/DA-types.md": [ { "hashed_secret": "44cdfc3615970ada14420caaaa5c5745fca06002", "is_secret": false, @@ -104,6 +104,16 @@ "type": "Secret Keyword", "verified_result": null } + ], + "tests/pr_test.go": [ + { + "hashed_secret": "8c7c51db5075ebd0369c51e9f14737d9b4c1c21d", + "is_secret": false, + "is_verified": false, + "line_number": 353, + "type": "Base64 High Entropy String", + "verified_result": null + } ] }, "version": "0.13.1+ibm.62.dss", diff --git a/README.md b/README.md index 32567b0b..7bf50d57 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,9 @@ You need the following permissions to run this module. | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.9.0 | -| [ibm](#requirement\_ibm) | >= 1.79.1, <2.0.0 | +| [ibm](#requirement\_ibm) | >= 1.79.2, <2.0.0 | | [null](#requirement\_null) | >= 3.2.1, < 4.0.0 | -| [time](#requirement\_time) | >= 0.9.1 | +| [time](#requirement\_time) | >= 0.9.1, < 1.0.0 | ### Modules @@ -76,16 +76,16 @@ You need the following permissions to run this module. | Name | Type | |------|------| -| [ibm_database.elasticsearch](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/database) | resource | -| [ibm_iam_authorization_policy.backup_kms_policy](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/iam_authorization_policy) | resource | -| [ibm_iam_authorization_policy.policy](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/iam_authorization_policy) | resource | -| [ibm_resource_key.service_credentials](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/resource_key) | resource | -| [ibm_resource_tag.elasticsearch_tag](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/resources/resource_tag) | resource | +| [ibm_database.elasticsearch](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/database) | resource | +| [ibm_iam_authorization_policy.backup_kms_policy](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/iam_authorization_policy) | resource | +| [ibm_iam_authorization_policy.kms_policy](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/iam_authorization_policy) | resource | +| [ibm_resource_key.service_credentials](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/resource_key) | resource | +| [ibm_resource_tag.elasticsearch_tag](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/resource_tag) | resource | | [null_resource.put_vectordb_model](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [null_resource.start_vectordb_model](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [time_sleep.wait_for_authorization_policy](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | | [time_sleep.wait_for_backup_kms_authorization_policy](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | -| [ibm_database_connection.database_connection](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/data-sources/database_connection) | data source | +| [ibm_database_connection.database_connection](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/data-sources/database_connection) | data source | ### Inputs @@ -97,15 +97,15 @@ You need the following permissions to run this module. | [backup\_crn](#input\_backup\_crn) | The CRN of a backup resource to restore from. The backup is created by a database deployment with the same service ID. The backup is loaded after both provisioning is complete and the new deployment that uses that data starts. Specify a backup CRN is in the format `crn:v1:<...>:backup:`. If not specified, the database is provisioned empty. | `string` | `null` | no | | [backup\_encryption\_key\_crn](#input\_backup\_encryption\_key\_crn) | The CRN of a Key Protect or Hyper Protect Crypto Services encryption key that you want to use for encrypting the disk that holds deployment backups. Applies only if `use_ibm_owned_encryption_key` is false and `use_same_kms_key_for_backups` is false. If no value is passed, and `use_same_kms_key_for_backups` is true, the value of `kms_key_crn` is used. Alternatively set `use_default_backup_encryption_key` to true to use the IBM Cloud Databases default encryption. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups). | `string` | `null` | no | | [cbr\_rules](#input\_cbr\_rules) | (Optional, list) List of context-based restrictions rules to create. | 
list(object({
    description = string
    account_id  = string
    rule_contexts = list(object({
      attributes = optional(list(object({
        name  = string
        value = string
    }))) }))
    enforcement_mode = string
    operations = optional(list(object({
      api_types = list(object({
        api_type_id = string
      }))
    })))
  }))
| `[]` | no | +| [cpu\_count](#input\_cpu\_count) | The dedicated CPU per member that is allocated. For shared CPU, set to 0. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling). | `number` | `0` | no | +| [disk\_mb](#input\_disk\_mb) | The disk that is allocated per member. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling). | `number` | `5120` | no | | [elasticsearch\_version](#input\_elasticsearch\_version) | The version of Databases for Elasticsearch to deploy. Possible values: `8.7`, `8.10`, `8.12`, `8.15` which requires an Enterprise Platinum pricing plan. If no value is specified, the current preferred version for IBM Cloud Databases is used. | `string` | `null` | no | | [elser\_model\_type](#input\_elser\_model\_type) | Trained ELSER model to be used for Elastic's Natural Language Processing. Possible values: `.elser_model_1`, `.elser_model_2` and `.elser_model_2_linux-x86_64`. Applies only if also 'plan' is set to 'platinum'. [Learn more](https://www.elastic.co/guide/en/machine-learning/current/ml-nlp-elser.html) | `string` | `".elser_model_2_linux-x86_64"` | no | | [enable\_elser\_model](#input\_enable\_elser\_model) | Set it to true to install and start the Elastic's Natural Language Processing model. Applies only if also 'plan' is set to 'platinum'. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-elser-embeddings-elasticsearch) | `bool` | `false` | no | | [kms\_key\_crn](#input\_kms\_key\_crn) | The CRN of a Key Protect or Hyper Protect Crypto Services encryption key to encrypt your data. Applies only if `use_ibm_owned_encryption_key` is false. By default this key is used for both deployment data and backups, but this behaviour can be altered using the `use_same_kms_key_for_backups` and `backup_encryption_key_crn` inputs. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups). | `string` | `null` | no | -| [member\_cpu\_count](#input\_member\_cpu\_count) | The dedicated CPU per member that is allocated. For shared CPU, set to 0. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling). | `number` | `0` | no | -| [member\_disk\_mb](#input\_member\_disk\_mb) | The disk that is allocated per member. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling). | `number` | `5120` | no | | [member\_host\_flavor](#input\_member\_host\_flavor) | The host flavor per member. [Learn more](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/database#host_flavor). | `string` | `null` | no | -| [member\_memory\_mb](#input\_member\_memory\_mb) | The memory per member that is allocated. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling) | `number` | `4096` | no | | [members](#input\_members) | The number of members that are allocated. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling). | `number` | `3` | no | +| [memory\_mb](#input\_memory\_mb) | The memory per member that is allocated. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling) | `number` | `4096` | no | | [name](#input\_name) | The name of the Databases for Elasticsearch instance. | `string` | n/a | yes | | [plan](#input\_plan) | The pricing plan for the Databases for Elasticsearch instance. Must be `enterprise` or `platinum` if the `elasticsearch_version` variable is set to `8.10` or later. | `string` | `"enterprise"` | no | | [region](#input\_region) | The region where you want to deploy your instance. | `string` | `"us-south"` | no | diff --git a/common-dev-assets b/common-dev-assets index d0c2b2a1..b76eee7e 160000 --- a/common-dev-assets +++ b/common-dev-assets @@ -1 +1 @@ -Subproject commit d0c2b2a1d3cdf41f6a844bc17816199fe3f8b20f +Subproject commit b76eee7ec33d5b81b30828e37a02d2df595e0909 diff --git a/cra-config.yaml b/cra-config.yaml index 8a1d6b57..b87cc8be 100644 --- a/cra-config.yaml +++ b/cra-config.yaml @@ -1,12 +1,12 @@ # More info about this file at https://github.com/terraform-ibm-modules/common-pipeline-assets/blob/main/.github/workflows/terraform-test-pipeline.md#cra-config-yaml version: "v1" CRA_TARGETS: - - CRA_TARGET: "solutions/standard" # Target directory for CRA scan. If not provided, the CRA Scan will not be run. + - CRA_TARGET: "solutions/fully-configurable" # Target directory for CRA scan. If not provided, the CRA Scan will not be run. CRA_IGNORE_RULES_FILE: "cra-tf-validate-ignore-rules.json" # CRA Ignore file to use. If not provided, it checks the repo root directory for `cra-tf-validate-ignore-rules.json` PROFILE_ID: "fe96bd4d-9b37-40f2-b39f-a62760e326a3" # SCC profile ID (currently set to 'IBM Cloud Framework for Financial Services' '1.7.0' profile). CRA_ENVIRONMENT_VARIABLES: TF_VAR_existing_kms_instance_crn: "crn:v1:bluemix:public:hs-crypto:us-south:a/abac0df06b644a9cabc6e44f55b3880e:e6dce284-e80f-46e1-a3c1-830f7adff7a9::" - TF_VAR_existing_kms_key_crn: "crn:v1:bluemix:public:hs-crypto:us-south:a/abac0df06b644a9cabc6e44f55b3880e:e6dce284-e80f-46e1-a3c1-830f7adff7a9:key:76170fae-4e0c-48c3-8ebe-326059ebb533" + TF_VAR_existing_resource_group_name: "geretain-test-elasticsearch" + TF_VAR_kms_encryption_enabled: true TF_VAR_provider_visibility: "public" - TF_VAR_resource_group_name: "test-es-cra" - TF_VAR_use_existing_resource_group: false + TF_VAR_prefix: "test" diff --git a/examples/basic/variables.tf b/examples/basic/variables.tf index 18b25c7e..3f0e4007 100644 --- a/examples/basic/variables.tf +++ b/examples/basic/variables.tf @@ -50,6 +50,7 @@ variable "service_endpoints" { error_message = "Valid values for service_endpoints are 'public', 'public-and-private', and 'private'" } } + variable "member_host_flavor" { type = string description = "The host flavor per member. [Learn more](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/database#host_flavor)." diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 60b6b53d..22d70de8 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -80,7 +80,7 @@ module "icd_elasticsearch" { tags = var.resource_tags auto_scaling = var.auto_scaling member_host_flavor = "multitenant" - member_memory_mb = 4096 + memory_mb = 4096 # Example of how to use different KMS keys for data and backups use_ibm_owned_encryption_key = false diff --git a/ibm_catalog.json b/ibm_catalog.json index 511529fa..aa3a4ccd 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -9,6 +9,7 @@ "target_terraform", "terraform", "data_management", + "database", "solution" ], "keywords": [ @@ -22,61 +23,79 @@ "nosql" ], "short_description": "Creates and configures an instance of IBM Cloud Databases for Elasticsearch.", - "long_description": "This architecture supports creating and configuring an instance of Databases for Elasticsearch with KMS encryption.", + "long_description": "This architecture supports creating and configuring an instance of [Databases for Elasticsearch](https://www.ibm.com/products/databases-for-elasticsearch), with optional KMS encryption.\n\nℹ️ This Terraform-based automation is part of a broader suite of IBM-maintained Infrastructure as Code (IaC) assets, each following the naming pattern \"Cloud automation for *servicename*\" and focusing on single IBM Cloud service. These single-service deployable architectures can be used on their own to streamline and automate service deployments through an [IaC approach](https://cloud.ibm.com/docs/secure-enterprise?topic=secure-enterprise-understanding-projects), or assembled together into a broader [automated IaC stack](https://cloud.ibm.com/docs/secure-enterprise?topic=secure-enterprise-config-stack) to automate the deployment of an end-to-end solution architecture.", "offering_docs_url": "https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/README.md", "offering_icon_url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/main/images/elasticsearch_icon.svg", "provider_name": "IBM", "support_details": "This product is in the community registry, as such support is handled through the originated repo. If you experience issues please open an issue in the repository [https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/issues](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/issues). Please note this product is not supported via the IBM Cloud Support Center.", "features": [ { - "title": "Creates an instance of Databases for Elasticsearch", - "description": "Creates and configures an IBM Cloud Databases for Elasticsearch instance." - }, - { - "title": "Supports KMS encryption", - "description": "Provides KMS encryption for the data that you store in the database." + "title": "KMS encryption", + "description": "Provides [KMS encryption](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-key-protect&interface=ui) for the data that you store in the database, enhancing data security." }, { - "title": "Supports autoscaling", - "description": "Provides the autoscaling to allow the database to increase resources in response to usage." + "title": "Autoscaling", + "description": "Provides the [autoscaling](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-autoscaling&interface=ui) to allow the database to increase resources in response to usage." }, { - "title": "Attaches access tags", + "title": "Access tags", "description": "Attaches access tags to the Elasticsearch instance." }, { - "title": "Supports backup restoration", - "description": "Provides database restoration using a backup created by a deployment with the same service ID." + "title": "Backup restoration", + "description": "Provides [database restoration](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-dashboard-backups&interface=ui) using a backup created by a deployment with the same service ID." } ], "flavors": [ { - "label": "Standard", - "name": "standard", + "label": "Fully configurable", + "name": "fully-configurable", + "index": 1, "install_type": "fullstack", - "working_directory": "solutions/standard", + "working_directory": "solutions/fully-configurable", "compliance": { "authority": "scc-v3", "profiles": [ { "profile_name": "IBM Cloud Framework for Financial Services", - "profile_version": "1.6.0" + "profile_version": "1.7.0" } ] }, "iam_permissions": [ + { + "role_crns": [ + "crn:v1:bluemix:public:iam::::role:Viewer" + ], + "service_name": "Resource group only", + "notes": "Viewer access is required in the resource group you want to provision in." + }, { "role_crns": [ "crn:v1:bluemix:public:iam::::role:Editor" ], "service_name": "databases-for-elasticsearch" + }, + { + "role_crns": [ + "crn:v1:bluemix:public:iam::::role:Editor" + ], + "service_name": "kms", + "notes": "[Optional] Editor access is required to create keys. It is only required when using Key Protect for encryption." + }, + { + "role_crns": [ + "crn:v1:bluemix:public:iam::::role:Editor" + ], + "service_name": "hs-crypto", + "notes": "[Optional] Editor access is required to create keys in HPCS. It is only required when using HPCS for encryption." } ], "architecture": { "features": [ { - "title": " Creates an instance of Databases for Elasticsearch", - "description": "This architecture creates an instance of IBM Cloud Databases for Elasticsearch with KMS encryption. It accepts or creates a resource group, and provides autoscaling rules." + "title": " ", + "description": "Configured to use IBM secure by default standards, but can be edited to fit your use case." } ], "diagrams": [ @@ -96,6 +115,7 @@ }, { "key": "provider_visibility", + "hidden": true, "options": [ { "displayname": "private", @@ -112,16 +132,19 @@ ] }, { - "key": "use_existing_resource_group", - "required": true - }, - { - "key": "resource_group_name", - "required": true + "key": "existing_resource_group_name", + "display_name": "resource_group", + "custom_config": { + "type": "resource_group", + "grouping": "deployment", + "original_grouping": "deployment", + "config_constraints": { + "identifier": "rg_name" + } + } }, { - "key": "prefix", - "required": true + "key": "prefix" }, { "key": "region", @@ -177,34 +200,10 @@ } ] }, - { - "key": "name" - }, - { - "key": "existing_elasticsearch_instance_crn" - }, - { - "key": "plan", - "options": [ - { - "displayname": "enterprise", - "value": "enterprise" - }, - { - "displayname": "platinum", - "value": "platinum" - } - ] - }, { "key": "elasticsearch_version", - "required": false, - "default_value": "__NULL__", + "required": true, "options": [ - { - "displayname": "preferred", - "value": "__NULL__" - }, { "displayname": "8.15", "value": "8.15" @@ -220,13 +219,42 @@ ] }, { - "key": "tags" + "key": "name" }, { - "key": "access_tags" + "key": "plan", + "options": [ + { + "displayname": "enterprise", + "value": "enterprise" + }, + { + "displayname": "platinum", + "value": "platinum" + } + ] }, { - "key": "auto_scaling" + "key": "resource_tags", + "type": "array", + "custom_config": { + "grouping": "deployment", + "original_grouping": "deployment", + "config_constraints": { + "type": "string" + } + } + }, + { + "key": "access_tags", + "type": "array", + "custom_config": { + "grouping": "deployment", + "original_grouping": "deployment", + "config_constraints": { + "type": "string" + } + } }, { "key": "members" @@ -244,19 +272,46 @@ "key": "member_host_flavor" }, { - "key": "admin_pass" + "key": "auto_scaling" }, { - "key": "users" + "key": "service_endpoints", + "options": [ + { + "displayname": "private", + "value": "private" + }, + { + "displayname": "public", + "value": "public" + }, + { + "displayname": "public-and-private", + "value": "public-and-private" + } + ] }, { "key": "service_credential_names" }, + { + "key": "service_credential_secrets", + "type": "array", + "custom_config": { + "type": "textarea", + "grouping": "deployment", + "original_grouping": "deployment" + } + }, + { + "key": "admin_pass" + }, { "key": "existing_secrets_manager_instance_crn" }, { "key": "existing_secrets_manager_endpoint_type", + "hidden": true, "options": [ { "displayname": "public", @@ -269,25 +324,41 @@ ] }, { - "key": "service_credential_secrets" + "key": "skip_elasticsearch_to_secrets_manager_auth_policy" }, { - "key": "use_existing_admin_pass_secrets_manager_secret_group" + "key": "admin_pass_secrets_manager_secret_group" }, { "key": "admin_pass_secrets_manager_secret_name" }, { - "key": "admin_pass_secrets_manager_secret_group" + "key": "use_existing_admin_pass_secrets_manager_secret_group" }, { - "key": "skip_elasticsearch_to_secrets_manager_auth_policy" + "key": "users", + "type": "array", + "custom_config": { + "type": "textarea", + "grouping": "deployment", + "original_grouping": "deployment" + } }, { "key": "ibmcloud_kms_api_key" }, + { + "key": "kms_encryption_enabled" + }, + { + "key": "existing_kms_instance_crn" + }, + { + "key": "existing_kms_key_crn" + }, { "key": "kms_endpoint_type", + "hidden": true, "options": [ { "displayname": "public", @@ -300,14 +371,13 @@ ] }, { - "key": "use_ibm_owned_encryption_key" + "key": "key_ring_name" }, { - "key": "existing_kms_instance_crn", - "required": true + "key": "key_name" }, { - "key": "existing_kms_key_crn" + "key": "backup_crn" }, { "key": "existing_backup_kms_key_crn" @@ -316,16 +386,10 @@ "key": "use_default_backup_encryption_key" }, { - "key": "elasticsearch_key_ring_name" - }, - { - "key": "elasticsearch_key_name" + "key": "skip_elasticsearch_kms_auth_policy" }, { - "key": "skip_es_kms_auth_policy" - }, - { - "key": "backup_crn" + "key": "existing_elasticsearch_instance_crn" }, { "key": "enable_elser_model" @@ -407,6 +471,328 @@ "key": "cbr_rules" } ] + }, + { + "label": "Security-enforced", + "name": "security-enforced", + "index": 2, + "install_type": "fullstack", + "working_directory": "solutions/security-enforced", + "compliance": { + "authority": "scc-v3", + "profiles": [ + { + "profile_name": "IBM Cloud Framework for Financial Services", + "profile_version": "1.7.0" + } + ] + }, + "iam_permissions": [ + { + "role_crns": [ + "crn:v1:bluemix:public:iam::::role:Viewer" + ], + "service_name": "Resource group only", + "notes": "Viewer access is required in the resource group you want to provision in." + }, + { + "role_crns": [ + "crn:v1:bluemix:public:iam::::role:Editor" + ], + "service_name": "databases-for-redis" + }, + { + "role_crns": [ + "crn:v1:bluemix:public:iam::::role:Editor" + ], + "service_name": "kms", + "notes": "[Optional] Editor access is required to create keys. It is required only if KMS encryption is enabled." + }, + { + "role_crns": [ + "crn:v1:bluemix:public:iam::::role:Editor" + ], + "service_name": "hs-crypto", + "notes": "[Optional] Editor access is required to create keys in HPCS. It is only required when using HPCS for encryption." + } + ], + "architecture": { + "features": [ + { + "title": " ", + "description": "Configured to use IBM secure by default standards that can't be changed." + } + ], + "diagrams": [ + { + "diagram": { + "caption": "Databases for Elasticsearch instance on IBM Cloud", + "url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/main/reference-architecture/deployable-architecture-elasticsearch.svg", + "type": "image/svg+xml" + }, + "description": "This architecture supports creating and configuring an instance of Databases for Elasticsearch instance with KMS encryption." + } + ] + }, + "configuration": [ + { + "key": "ibmcloud_api_key" + }, + { + "key": "existing_resource_group_name", + "display_name": "resource_group", + "custom_config": { + "type": "resource_group", + "grouping": "deployment", + "original_grouping": "deployment", + "config_constraints": { + "identifier": "rg_name" + } + } + }, + { + "key": "prefix" + }, + { + "key": "region", + "required": true, + "options": [ + { + "displayname": "Chennai (che01)", + "value": "che01" + }, + { + "displayname": "Dallas (us-south)", + "value": "us-south" + }, + { + "displayname": "Frankfurt (eu-de)", + "value": "eu-de" + }, + { + "displayname": "London (eu-gb)", + "value": "eu-gb" + }, + { + "displayname": "Madrid (eu-es)", + "value": "eu-es" + }, + { + "displayname": "Osaka (jp-osa)", + "value": "jp-osa" + }, + { + "displayname": "Paris (par01)", + "value": "par01" + }, + { + "displayname": "Sao Paulo (br-sao)", + "value": "br-sao" + }, + { + "displayname": "Sydney (au-syd)", + "value": "au-syd" + }, + { + "displayname": "Toronto (ca-tor)", + "value": "ca-tor" + }, + { + "displayname": "Tokyo (jp-tok)", + "value": "jp-tok" + }, + { + "displayname": "Washington (us-east)", + "value": "us-east" + } + ] + }, +{ + "key": "plan", + "options": [ + { + "displayname": "enterprise", + "value": "enterprise" + }, + { + "displayname": "platinum", + "value": "platinum" + } + ] + }, + { + "key": "elasticsearch_version", + "required": true, + "options": [ + { + "displayname": "8.15", + "value": "8.15" + }, + { + "displayname": "8.12", + "value": "8.12" + }, + { + "displayname": "8.10", + "value": "8.10" + } + ] + }, + { + "key": "name" + }, + { + "key": "resource_tags", + "type": "array", + "custom_config": { + "grouping": "deployment", + "original_grouping": "deployment", + "config_constraints": { + "type": "string" + } + } + }, + { + "key": "access_tags", + "type": "array", + "custom_config": { + "grouping": "deployment", + "original_grouping": "deployment", + "config_constraints": { + "type": "string" + } + } + }, + { + "key": "members" + }, + { + "key": "member_memory_mb" + }, + { + "key": "member_cpu_count" + }, + { + "key": "member_disk_mb" + }, + { + "key": "member_host_flavor" + }, + { + "key": "auto_scaling" + }, + { + "key": "service_credential_names" + }, + { + "key": "service_credential_secrets", + "type": "array", + "custom_config": { + "type": "textarea", + "grouping": "deployment", + "original_grouping": "deployment" + } + }, + { + "key": "admin_pass" + }, + { + "key": "existing_secrets_manager_instance_crn" + }, + { + "key": "skip_elasticsearch_to_secrets_manager_auth_policy" + }, + { + "key": "admin_pass_secrets_manager_secret_group" + }, + { + "key": "admin_pass_secrets_manager_secret_name" + }, + { + "key": "use_existing_admin_pass_secrets_manager_secret_group" + }, + { + "key": "users", + "type": "array", + "custom_config": { + "type": "textarea", + "grouping": "deployment", + "original_grouping": "deployment" + } + }, + { + "key": "ibmcloud_kms_api_key" + }, + { + "key": "existing_kms_instance_crn", + "required": true + }, + { + "key": "existing_kms_key_crn" + }, + { + "key": "key_ring_name" + }, + { + "key": "key_name" + }, + { + "key": "backup_crn" + }, + { + "key": "existing_backup_kms_key_crn" + }, + { + "key": "skip_elasticsearch_kms_auth_policy" + }, + { + "key": "existing_elasticsearch_instance_crn" + }, + { + "key": "enable_elser_model" + }, + { + "key": "elser_model_type", + "options": [ + { + "displayname": ".elser_model_1", + "value": ".elser_model_1" + }, + { + "displayname": ".elser_model_2", + "value": ".elser_model_2" + }, + { + "displayname": ".elser_model_2_linux-x86_64", + "value": ".elser_model_2_linux-x86_64" + } + ] + }, + { + "key": "enable_kibana_dashboard" + }, + { + "key": "kibana_code_engine_new_project_name" + }, + { + "key": "kibana_code_engine_new_app_name" + }, + { + "key": "existing_code_engine_project_id" + }, + { + "key": "kibana_registry_namespace_image" + }, + { + "key": "kibana_image_digest" + }, + { + "key": "kibana_image_port" + }, + { + "key": "cbr_rules" + } + ] } ] } diff --git a/main.tf b/main.tf index 6a870a48..a22ebec4 100644 --- a/main.tf +++ b/main.tf @@ -1,15 +1,10 @@ -######################################################################################################################## -# Input variable validation -# (approach based on https://github.com/hashicorp/terraform/issues/25609#issuecomment-1057614400) -# -# TODO: Replace with terraform cross variable validation: https://github.ibm.com/GoldenEye/issues/issues/10836 -######################################################################################################################## - ######################################################################################################################## # Locals ######################################################################################################################## locals { + # If no value passed for 'backup_encryption_key_crn' use the value of 'kms_key_crn' and perform validation of 'kms_key_crn' to check if region is supported by backup encryption key. + # If 'use_ibm_owned_encryption_key' is true or 'use_default_backup_encryption_key' is true, default to null. # If no value is passed for 'backup_encryption_key_crn', then default to use 'kms_key_crn'. backup_encryption_key_crn = var.use_ibm_owned_encryption_key || var.use_default_backup_encryption_key ? null : (var.backup_encryption_key_crn != null ? var.backup_encryption_key_crn : var.kms_key_crn) @@ -68,12 +63,13 @@ locals { create_backup_kms_auth_policy = !var.use_ibm_owned_encryption_key && !var.skip_iam_authorization_policy && !var.use_same_kms_key_for_backups ? 1 : 0 } -resource "ibm_iam_authorization_policy" "policy" { +# Create IAM Authorization Policies to allow Elasticsearch to access KMS for the encryption key +resource "ibm_iam_authorization_policy" "kms_policy" { count = local.create_kms_auth_policy source_service_name = "databases-for-elasticsearch" source_resource_group_id = var.resource_group_id roles = ["Reader"] - description = "Allow all Elastic Search instances in the resource group ${var.resource_group_id} to read the ${local.kms_service} key ${local.kms_key_id} from the instance GUID ${local.kms_key_instance_guid}" + description = "Allow all Elasticsearch instances in the resource group ${var.resource_group_id} to read the ${local.kms_service} key ${local.kms_key_id} from the instance GUID ${local.kms_key_instance_guid}" resource_attributes { name = "serviceName" operator = "stringEquals" @@ -108,8 +104,9 @@ resource "ibm_iam_authorization_policy" "policy" { # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 resource "time_sleep" "wait_for_authorization_policy" { - count = local.create_kms_auth_policy - depends_on = [ibm_iam_authorization_policy.policy] + count = local.create_kms_auth_policy + depends_on = [ibm_iam_authorization_policy.kms_policy] + create_duration = "30s" } @@ -118,7 +115,7 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy" { source_service_name = "databases-for-elasticsearch" source_resource_group_id = var.resource_group_id roles = ["Reader"] - description = "Allow all Elastic Search instances in the Resource Group ${var.resource_group_id} to read the ${local.backup_kms_service} key ${local.backup_kms_key_id} from the instance GUID ${local.backup_kms_key_instance_guid}" + description = "Allow all Elasticsearch instances in the Resource Group ${var.resource_group_id} to read the ${local.backup_kms_service} key ${local.backup_kms_key_id} from the instance GUID ${local.backup_kms_key_instance_guid}" resource_attributes { name = "serviceName" operator = "stringEquals" @@ -193,12 +190,12 @@ resource "ibm_database" "elasticsearch" { dynamic "group" { for_each = local.host_flavor_set && var.member_host_flavor != "multitenant" && var.backup_crn == null ? [1] : [] content { - group_id = "member" # Only member type is allowed for elasticsearch + group_id = "member" # Only member type is allowed for IBM Cloud Databases host_flavor { id = var.member_host_flavor } disk { - allocation_mb = var.member_disk_mb + allocation_mb = var.disk_mb } members { allocation_count = var.members @@ -210,18 +207,18 @@ resource "ibm_database" "elasticsearch" { dynamic "group" { for_each = local.host_flavor_set && var.member_host_flavor == "multitenant" && var.backup_crn == null ? [1] : [] content { - group_id = "member" # Only member type is allowed for elasticsearch + group_id = "member" # Only member type is allowed for IBM Cloud Databases host_flavor { id = var.member_host_flavor } disk { - allocation_mb = var.member_disk_mb + allocation_mb = var.disk_mb } memory { - allocation_mb = var.member_memory_mb + allocation_mb = var.memory_mb } cpu { - allocation_count = var.member_cpu_count + allocation_count = var.cpu_count } members { allocation_count = var.members @@ -231,17 +228,17 @@ resource "ibm_database" "elasticsearch" { ## This block is for if host_flavor IS NOT set dynamic "group" { - for_each = local.host_flavor_set == false && var.backup_crn == null ? [1] : [] + for_each = !local.host_flavor_set && var.backup_crn == null ? [1] : [] content { - group_id = "member" # Only member type is allowed for elasticsearch + group_id = "member" # Only member type is allowed for IBM Cloud Databases memory { - allocation_mb = var.member_memory_mb + allocation_mb = var.memory_mb } disk { - allocation_mb = var.member_disk_mb + allocation_mb = var.disk_mb } cpu { - allocation_count = var.member_cpu_count + allocation_count = var.cpu_count } members { allocation_count = var.members @@ -300,7 +297,6 @@ resource "ibm_resource_tag" "elasticsearch_tag" { tag_type = "access" } - ######################################################################################################################## # Context Based Restrictions ######################################################################################################################## diff --git a/modules/fscloud/main.tf b/modules/fscloud/main.tf index 9a52d8e6..c3d9c9a7 100644 --- a/modules/fscloud/main.tf +++ b/modules/fscloud/main.tf @@ -17,11 +17,11 @@ module "elasticsearch" { tags = var.tags plan = var.plan members = var.members - member_memory_mb = var.member_memory_mb + memory_mb = var.member_memory_mb admin_pass = var.admin_pass users = var.users - member_disk_mb = var.member_disk_mb - member_cpu_count = var.member_cpu_count + disk_mb = var.member_disk_mb + cpu_count = var.member_cpu_count member_host_flavor = var.member_host_flavor auto_scaling = var.auto_scaling service_credential_names = var.service_credential_names diff --git a/reference-architecture/deployable-architecture-elasticsearch.svg b/reference-architecture/deployable-architecture-elasticsearch.svg index 5ca6ee89..367df641 100644 --- a/reference-architecture/deployable-architecture-elasticsearch.svg +++ b/reference-architecture/deployable-architecture-elasticsearch.svg @@ -1,4 +1,4 @@ - + -
IBM Cloud
IBM Cloud
KMS Encryption
KMS Encryption
Region
Region
Resource Group
Resource Group
IBM Cloud Elastic Search Instance
IBM Cloud El...Kibana Code Engine App
IBM Cloud Code Engine Project
IBM Cloud Code Engine ProjectText is not SVG - cannot display \ No newline at end of file +IBM CloudRegionResource GroupDatabase for Elasticsearch
ES
ES
IBM Cloud Code Engine Project
IBM Cloud Code Engine ProjectKibana Code Engine App
[Optional] KMS
[Optional] KMS
Key Ring
Key Ringelasticsearch-keyText is not SVG - cannot display \ No newline at end of file diff --git a/solutions/standard/DA-cbr_rules.md b/solutions/fully-configurable/DA-cbr_rules.md similarity index 100% rename from solutions/standard/DA-cbr_rules.md rename to solutions/fully-configurable/DA-cbr_rules.md diff --git a/solutions/standard/DA-types.md b/solutions/fully-configurable/DA-types.md similarity index 100% rename from solutions/standard/DA-types.md rename to solutions/fully-configurable/DA-types.md diff --git a/solutions/fully-configurable/README.md b/solutions/fully-configurable/README.md new file mode 100644 index 00000000..f099860c --- /dev/null +++ b/solutions/fully-configurable/README.md @@ -0,0 +1,3 @@ +# Cloud automation for Elasticsearch (Fully Configurable) + +:exclamation: **Important:** This solution is not intended to be called by other modules because it contains a provider configuration and is not compatible with the `for_each`, `count`, and `depends_on` arguments. For more information, see [Providers Within Modules](https://developer.hashicorp.com/terraform/language/modules/develop/providers). diff --git a/solutions/fully-configurable/catalogValidationValues.json.template b/solutions/fully-configurable/catalogValidationValues.json.template new file mode 100644 index 00000000..fbb545af --- /dev/null +++ b/solutions/fully-configurable/catalogValidationValues.json.template @@ -0,0 +1,9 @@ +{ + "ibmcloud_api_key": $VALIDATION_APIKEY, + "region": "us-south", + "resource_tags": $TAGS, + "name": $PREFIX, + "existing_resource_group_name": "geretain-test-permanent", + "kms_encryption_enabled": true, + "existing_kms_instance_crn": $HPCS_US_SOUTH_CRN +} diff --git a/solutions/standard/main.tf b/solutions/fully-configurable/main.tf similarity index 75% rename from solutions/standard/main.tf rename to solutions/fully-configurable/main.tf index 88db2ae5..ba901848 100644 --- a/solutions/standard/main.tf +++ b/solutions/fully-configurable/main.tf @@ -1,37 +1,36 @@ ####################################################################################################################### # Resource Group ####################################################################################################################### +locals { + prefix = var.prefix != null ? trimspace(var.prefix) != "" ? "${var.prefix}-" : "" : "" +} module "resource_group" { source = "terraform-ibm-modules/resource-group/ibm" version = "1.2.0" - resource_group_name = var.use_existing_resource_group == false ? ((var.prefix != null && var.prefix != "") ? "${var.prefix}-${var.resource_group_name}" : var.resource_group_name) : null - existing_resource_group_name = var.use_existing_resource_group == true ? var.resource_group_name : null + existing_resource_group_name = var.existing_resource_group_name } -####################################################################################################################### -# KMS related variable validation -# (approach based on https://github.com/hashicorp/terraform/issues/25609#issuecomment-1057614400) -# -# TODO: Replace with terraform cross variable validation: https://github.ibm.com/GoldenEye/issues/issues/10836 -####################################################################################################################### - ####################################################################################################################### # KMS encryption key ####################################################################################################################### locals { - create_new_kms_key = var.existing_elasticsearch_instance_crn == null && !var.use_ibm_owned_encryption_key && var.existing_kms_key_crn == null ? 1 : 0 # no need to create any KMS resources if using existing Elasticsearch, passing an existing key, or using IBM owned keys - elasticsearch_key_name = (var.prefix != null && var.prefix != "") ? "${var.prefix}-${var.elasticsearch_key_name}" : var.elasticsearch_key_name - elasticsearch_key_ring_name = (var.prefix != null && var.prefix != "") ? "${var.prefix}-${var.elasticsearch_key_ring_name}" : var.elasticsearch_key_ring_name + use_ibm_owned_encryption_key = !var.kms_encryption_enabled + create_new_kms_key = ( + var.kms_encryption_enabled && + var.existing_elasticsearch_instance_crn == null && + var.existing_kms_key_crn == null + ) + elasticsearch_key_name = "${local.prefix}${var.key_name}" + elasticsearch_key_ring_name = "${local.prefix}${var.key_ring_name}" } - module "kms" { providers = { ibm = ibm.kms } - count = local.create_new_kms_key + count = local.create_new_kms_key ? 1 : 0 source = "terraform-ibm-modules/kms-all-inclusive/ibm" version = "5.1.7" create_key_protect_instance = false @@ -49,7 +48,7 @@ module "kms" { standard_key = false rotation_interval_month = 3 dual_auth_delete_enabled = false - force_delete = true + force_delete = true # Force delete must be set to true, or the terraform destroy will fail since the service does not de-register itself from the key until the reclamation period has expired. } ] } @@ -93,24 +92,23 @@ data "ibm_iam_account_settings" "iam_account_settings" { locals { account_id = data.ibm_iam_account_settings.iam_account_settings.account_id - create_cross_account_kms_auth_policy = var.existing_elasticsearch_instance_crn == null && !var.skip_es_kms_auth_policy && var.ibmcloud_kms_api_key != null && !var.use_ibm_owned_encryption_key - create_cross_account_backup_kms_auth_policy = var.existing_elasticsearch_instance_crn == null && !var.skip_es_kms_auth_policy && var.ibmcloud_kms_api_key != null && !var.use_ibm_owned_encryption_key && var.existing_backup_kms_key_crn != null + create_cross_account_kms_auth_policy = var.kms_encryption_enabled && !var.skip_elasticsearch_kms_auth_policy && var.ibmcloud_kms_api_key != null + create_cross_account_backup_kms_auth_policy = var.kms_encryption_enabled && !var.skip_elasticsearch_kms_auth_policy && var.ibmcloud_kms_api_key != null && var.existing_backup_kms_key_crn != null # If KMS encryption enabled (and existing ES instance is not being passed), parse details from the existing key if being passed, otherwise get it from the key that the DA creates - kms_account_id = var.existing_elasticsearch_instance_crn != null || var.use_ibm_owned_encryption_key ? null : var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].account_id : module.kms_instance_crn_parser[0].account_id - kms_service = var.existing_elasticsearch_instance_crn != null || var.use_ibm_owned_encryption_key ? null : var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].service_name : module.kms_instance_crn_parser[0].service_name - kms_instance_guid = var.existing_elasticsearch_instance_crn != null || var.use_ibm_owned_encryption_key ? null : var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].service_instance : module.kms_instance_crn_parser[0].service_instance - kms_key_crn = var.existing_elasticsearch_instance_crn != null || var.use_ibm_owned_encryption_key ? null : var.existing_kms_key_crn != null ? var.existing_kms_key_crn : module.kms[0].keys[format("%s.%s", local.elasticsearch_key_ring_name, local.elasticsearch_key_name)].crn - kms_key_id = var.existing_elasticsearch_instance_crn != null || var.use_ibm_owned_encryption_key ? null : var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].resource : module.kms[0].keys[format("%s.%s", local.elasticsearch_key_ring_name, local.elasticsearch_key_name)].key_id - kms_region = var.existing_elasticsearch_instance_crn != null || var.use_ibm_owned_encryption_key ? null : var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].region : module.kms_instance_crn_parser[0].region + kms_account_id = !var.kms_encryption_enabled || var.existing_elasticsearch_instance_crn != null ? null : var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].account_id : module.kms_instance_crn_parser[0].account_id + kms_service = !var.kms_encryption_enabled || var.existing_elasticsearch_instance_crn != null ? null : var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].service_name : module.kms_instance_crn_parser[0].service_name + kms_instance_guid = !var.kms_encryption_enabled || var.existing_elasticsearch_instance_crn != null ? null : var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].service_instance : module.kms_instance_crn_parser[0].service_instance + kms_key_crn = !var.kms_encryption_enabled || var.existing_elasticsearch_instance_crn != null ? null : var.existing_kms_key_crn != null ? var.existing_kms_key_crn : module.kms[0].keys[format("%s.%s", local.elasticsearch_key_ring_name, local.elasticsearch_key_name)].crn + kms_key_id = !var.kms_encryption_enabled || var.existing_elasticsearch_instance_crn != null ? null : var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].resource : module.kms[0].keys[format("%s.%s", local.elasticsearch_key_ring_name, local.elasticsearch_key_name)].key_id + kms_region = !var.kms_encryption_enabled || var.existing_elasticsearch_instance_crn != null ? null : var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].region : module.kms_instance_crn_parser[0].region # If creating KMS cross account policy for backups, parse backup key details from passed in key CRN backup_kms_account_id = local.create_cross_account_backup_kms_auth_policy ? module.kms_backup_key_crn_parser[0].account_id : local.kms_account_id backup_kms_service = local.create_cross_account_backup_kms_auth_policy ? module.kms_backup_key_crn_parser[0].service_name : local.kms_service backup_kms_instance_guid = local.create_cross_account_backup_kms_auth_policy ? module.kms_backup_key_crn_parser[0].service_instance : local.kms_instance_guid backup_kms_key_id = local.create_cross_account_backup_kms_auth_policy ? module.kms_backup_key_crn_parser[0].resource : local.kms_key_id - - backup_kms_key_crn = var.existing_elasticsearch_instance_crn != null || var.use_ibm_owned_encryption_key ? null : var.existing_backup_kms_key_crn + backup_kms_key_crn = var.existing_elasticsearch_instance_crn != null || !var.kms_encryption_enabled ? null : var.existing_backup_kms_key_crn # Always use same key for backups unless user explicially passed a value for 'existing_backup_kms_key_crn' use_same_kms_key_for_backups = var.existing_backup_kms_key_crn == null ? true : false } @@ -123,7 +121,7 @@ resource "ibm_iam_authorization_policy" "kms_policy" { source_service_name = "databases-for-elasticsearch" source_resource_group_id = module.resource_group.resource_group_id roles = ["Reader"] - description = "Allow all Elastic Search instances in the resource group ${module.resource_group.resource_group_id} in the account ${local.account_id} to read the ${local.kms_service} key ${local.kms_key_id} from the instance GUID ${local.kms_instance_guid}" + description = "Allow all Elasticsearch instances in the resource group ${module.resource_group.resource_group_id} in the account ${local.account_id} to read the ${local.kms_service} key ${local.kms_key_id} from the instance GUID ${local.kms_instance_guid}" resource_attributes { name = "serviceName" operator = "stringEquals" @@ -171,7 +169,7 @@ resource "ibm_iam_authorization_policy" "backup_kms_policy" { source_service_name = "databases-for-elasticsearch" source_resource_group_id = module.resource_group.resource_group_id roles = ["Reader"] - description = "Allow all Elastic Search instances in the resource group ${module.resource_group.resource_group_id} in the account ${local.account_id} to read the ${local.backup_kms_service} key ${local.backup_kms_key_id} from the instance GUID ${local.backup_kms_instance_guid}" + description = "Allow all Elasticsearch instances in the resource group ${module.resource_group.resource_group_id} in the account ${local.account_id} to read the ${local.backup_kms_service} key ${local.backup_kms_key_id} from the instance GUID ${local.backup_kms_instance_guid}" resource_attributes { name = "serviceName" operator = "stringEquals" @@ -228,7 +226,9 @@ locals { # if - replace first char with J # elseif _ replace first char with K # else use asis - admin_pass = var.admin_pass == null ? (startswith(random_password.admin_password[0].result, "-") ? "J${substr(random_password.admin_password[0].result, 1, -1)}" : startswith(random_password.admin_password[0].result, "_") ? "K${substr(random_password.admin_password[0].result, 1, -1)}" : random_password.admin_password[0].result) : var.admin_pass + generated_admin_password = (length(random_password.admin_password) > 0 ? (startswith(random_password.admin_password[0].result, "-") ? "J${substr(random_password.admin_password[0].result, 1, -1)}" : startswith(random_password.admin_password[0].result, "_") ? "K${substr(random_password.admin_password[0].result, 1, -1)}" : random_password.admin_password[0].result) : null) + # admin password to use + admin_pass = var.admin_pass == null ? local.generated_admin_password : var.admin_pass } ####################################################################################################################### @@ -276,31 +276,32 @@ data "ibm_database_connection" "existing_connection" { # Create new instance module "elasticsearch" { count = var.existing_elasticsearch_instance_crn != null ? 0 : 1 - source = "../../modules/fscloud" + source = "../.." depends_on = [time_sleep.wait_for_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy] resource_group_id = module.resource_group.resource_group_id - name = (var.prefix != null && var.prefix != "") ? "${var.prefix}-${var.name}" : var.name + name = "${local.prefix}${var.name}" region = var.region plan = var.plan - skip_iam_authorization_policy = var.skip_es_kms_auth_policy elasticsearch_version = var.elasticsearch_version - use_ibm_owned_encryption_key = var.use_ibm_owned_encryption_key + skip_iam_authorization_policy = var.kms_encryption_enabled ? var.skip_elasticsearch_kms_auth_policy : true + use_ibm_owned_encryption_key = local.use_ibm_owned_encryption_key kms_key_crn = local.kms_key_crn backup_encryption_key_crn = local.backup_kms_key_crn use_same_kms_key_for_backups = local.use_same_kms_key_for_backups use_default_backup_encryption_key = var.use_default_backup_encryption_key - backup_crn = var.backup_crn access_tags = var.access_tags - tags = var.tags + tags = var.resource_tags admin_pass = local.admin_pass users = var.users members = var.members member_host_flavor = var.member_host_flavor - member_memory_mb = var.member_memory_mb - member_disk_mb = var.member_disk_mb - member_cpu_count = var.member_cpu_count + memory_mb = var.member_memory_mb + disk_mb = var.member_disk_mb + cpu_count = var.member_cpu_count auto_scaling = var.auto_scaling service_credential_names = var.service_credential_names + backup_crn = var.backup_crn + service_endpoints = var.service_endpoints enable_elser_model = var.enable_elser_model elser_model_type = var.elser_model_type cbr_rules = var.cbr_rules @@ -322,7 +323,7 @@ locals { ####################################################################################################################### locals { - create_sm_auth_policy = var.skip_elasticsearch_to_secrets_manager_auth_policy || var.existing_secrets_manager_instance_crn == null ? 0 : 1 + create_secrets_manager_auth_policy = var.skip_elasticsearch_to_secrets_manager_auth_policy || var.existing_secrets_manager_instance_crn == null ? 0 : 1 } # Parse the Secrets Manager CRN @@ -333,10 +334,9 @@ module "sm_instance_crn_parser" { crn = var.existing_secrets_manager_instance_crn } -# create a service authorization between Secrets Manager and the target service (Elastic Search) +# create a service authorization between Secrets Manager and the target service (Elasticsearch) resource "ibm_iam_authorization_policy" "secrets_manager_key_manager" { - count = local.create_sm_auth_policy - depends_on = [module.elasticsearch] + count = local.create_secrets_manager_auth_policy source_service_name = "secrets-manager" source_resource_instance_id = local.existing_secrets_manager_instance_guid target_service_name = "databases-for-elasticsearch" @@ -346,14 +346,13 @@ resource "ibm_iam_authorization_policy" "secrets_manager_key_manager" { } # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 -resource "time_sleep" "wait_for_es_authorization_policy" { - count = local.create_sm_auth_policy +resource "time_sleep" "wait_for_elasticsearch_authorization_policy" { + count = local.create_secrets_manager_auth_policy depends_on = [ibm_iam_authorization_policy.secrets_manager_key_manager] create_duration = "30s" } locals { - # Build the structure of the service credential type secret service_credential_secrets = [ for service_credentials in var.service_credential_secrets : { secret_group_name = service_credentials.secret_group_name @@ -378,10 +377,10 @@ locals { # Build the structure of the arbitrary credential type secret for admin password admin_pass_secret = [{ - secret_group_name = (var.prefix != null && var.prefix != "") && var.admin_pass_secrets_manager_secret_group != null ? "${var.prefix}-${var.admin_pass_secrets_manager_secret_group}" : var.admin_pass_secrets_manager_secret_group + secret_group_name = "${local.prefix}${var.admin_pass_secrets_manager_secret_group}" existing_secret_group = var.use_existing_admin_pass_secrets_manager_secret_group secrets = [{ - secret_name = (var.prefix != null && var.prefix != "") && var.admin_pass_secrets_manager_secret_name != null ? "${var.prefix}-${var.admin_pass_secrets_manager_secret_name}" : var.admin_pass_secrets_manager_secret_name + secret_name = "${local.prefix}${var.admin_pass_secrets_manager_secret_name}" secret_type = "arbitrary" secret_payload_password = local.admin_pass } @@ -395,10 +394,9 @@ locals { existing_secrets_manager_instance_region = var.existing_secrets_manager_instance_crn != null ? module.sm_instance_crn_parser[0].region : null } -# Create secret(s) module "secrets_manager_service_credentials" { - count = var.existing_secrets_manager_instance_crn == null ? 0 : 1 - depends_on = [time_sleep.wait_for_es_authorization_policy] + count = length(local.service_credential_secrets) > 0 ? 1 : 0 + depends_on = [time_sleep.wait_for_elasticsearch_authorization_policy] source = "terraform-ibm-modules/secrets-manager/ibm//modules/secrets" version = "2.3.1" existing_sm_instance_guid = local.existing_secrets_manager_instance_guid @@ -413,8 +411,8 @@ module "secrets_manager_service_credentials" { locals { code_engine_project_id = var.existing_code_engine_project_id != null ? var.existing_code_engine_project_id : null - code_engine_project_name = local.code_engine_project_id != null ? null : (var.prefix != null && var.prefix != "") ? "${var.prefix}-${var.kibana_code_engine_new_project_name}" : var.kibana_code_engine_new_project_name - code_engine_app_name = (var.prefix != null && var.prefix != "") ? "${var.prefix}-${var.kibana_code_engine_new_app_name}" : var.kibana_code_engine_new_app_name + code_engine_project_name = local.code_engine_project_id != null ? null : "${local.prefix}${var.kibana_code_engine_new_project_name}" + code_engine_app_name = "${local.prefix}${var.kibana_code_engine_new_app_name}" kibana_version = var.enable_kibana_dashboard ? jsondecode(data.http.es_metadata[0].response_body).version.number : null } diff --git a/solutions/standard/outputs.tf b/solutions/fully-configurable/outputs.tf similarity index 75% rename from solutions/standard/outputs.tf rename to solutions/fully-configurable/outputs.tf index bc00a5ae..1f52d179 100644 --- a/solutions/standard/outputs.tf +++ b/solutions/fully-configurable/outputs.tf @@ -7,16 +7,16 @@ output "id" { value = local.elasticsearch_id } -output "guid" { - description = "Elasticsearch instance guid" - value = local.elasticsearch_guid -} - output "version" { description = "Elasticsearch instance version" value = local.elasticsearch_version } +output "guid" { + description = "Elasticsearch instance guid" + value = local.elasticsearch_guid +} + output "crn" { description = "Elasticsearch instance crn" value = local.elasticsearch_crn @@ -59,3 +59,19 @@ output "kibana_app_endpoint" { description = "Code Engine Kibana endpoint URL" value = var.enable_kibana_dashboard ? module.code_engine_kibana[0].app[local.code_engine_app_name].endpoint : null } + +output "cbr_rule_ids" { + description = "CBR rule ids created to restrict Elasticsearch" + value = var.existing_elasticsearch_instance_crn != null ? null : module.elasticsearch[0].cbr_rule_ids +} + +output "adminuser" { + description = "Database admin user name" + value = var.existing_elasticsearch_instance_crn != null ? null : module.elasticsearch[0].adminuser +} + +output "certificate_base64" { + description = "Database connection certificate" + value = var.existing_elasticsearch_instance_crn != null ? null : module.elasticsearch[0].certificate_base64 + sensitive = true +} diff --git a/solutions/fully-configurable/provider.tf b/solutions/fully-configurable/provider.tf new file mode 100644 index 00000000..e66dac2c --- /dev/null +++ b/solutions/fully-configurable/provider.tf @@ -0,0 +1,14 @@ +provider "ibm" { + ibmcloud_api_key = var.ibmcloud_api_key + region = var.region + visibility = var.provider_visibility + private_endpoint_type = (var.provider_visibility == "private" && var.region == "ca-mon") ? "vpe" : null +} + +provider "ibm" { + alias = "kms" + ibmcloud_api_key = var.ibmcloud_kms_api_key != null ? var.ibmcloud_kms_api_key : var.ibmcloud_api_key + region = local.kms_region + visibility = var.provider_visibility + private_endpoint_type = (var.provider_visibility == "private" && var.region == "ca-mon") ? "vpe" : null +} diff --git a/solutions/standard/variables.tf b/solutions/fully-configurable/variables.tf similarity index 70% rename from solutions/standard/variables.tf rename to solutions/fully-configurable/variables.tf index 18e515f2..2a726d06 100644 --- a/solutions/standard/variables.tf +++ b/solutions/fully-configurable/variables.tf @@ -1,5 +1,5 @@ ############################################################################## -# DA extra +# Input Variables ############################################################################## variable "ibmcloud_api_key" { @@ -8,36 +8,38 @@ variable "ibmcloud_api_key" { sensitive = true } -variable "provider_visibility" { - description = "Set the visibility value for the IBM terraform provider. Supported values are `public`, `private`, `public-and-private`. [Learn more](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/guides/custom-service-endpoints)." +variable "existing_resource_group_name" { type = string - default = "private" - - validation { - condition = contains(["public", "private", "public-and-private"], var.provider_visibility) - error_message = "Invalid visibility option. Allowed values are 'public', 'private', or 'public-and-private'." - } + description = "The name of an existing resource group to provision resource in." + default = "Default" + nullable = false } variable "prefix" { type = string - description = "Prefix to add to all resources created by this solution. To not use any prefix value, you can set this value to `null` or an empty string." - default = "dev" -} + nullable = true + description = "The prefix to be added to all resources created by this solution. To skip using a prefix, set this value to null or an empty string. The prefix must begin with a lowercase letter and may contain only lowercase letters, digits, and hyphens '-'. It should not exceed 16 characters, must not end with a hyphen('-'), and can not contain consecutive hyphens ('--'). Example: prod-0205-cos. [Learn more](https://terraform-ibm-modules.github.io/documentation/#/prefix.md)." -############################################################################## -# Input Variables -############################################################################## - -variable "resource_group_name" { - type = string - description = "The name of a new or an existing resource group to provision the Databases for Elasicsearch in. If a prefix input variable is specified, the prefix is added to the name in the `-` format." -} + validation { + # - null and empty string is allowed + # - Must not contain consecutive hyphens (--): length(regexall("--", var.prefix)) == 0 + # - Starts with a lowercase letter: [a-z] + # - Contains only lowercase letters (a–z), digits (0–9), and hyphens (-) + # - Must not end with a hyphen (-): [a-z0-9] + condition = (var.prefix == null || var.prefix == "" ? true : + alltrue([ + can(regex("^[a-z][-a-z0-9]*[a-z0-9]$", var.prefix)), + length(regexall("--", var.prefix)) == 0 + ]) + ) + error_message = "Prefix must begin with a lowercase letter and may contain only lowercase letters, digits, and hyphens '-'. It must not end with a hyphen('-'), and cannot contain consecutive hyphens ('--')." + } -variable "use_existing_resource_group" { - type = bool - description = "Whether to use an existing resource group." - default = false + validation { + # must not exceed 16 characters in length + condition = length(var.prefix) <= 16 + error_message = "Prefix must not exceed 16 characters." + } } variable "name" { @@ -46,29 +48,9 @@ variable "name" { default = "elasticsearch" } -variable "elasticsearch_version" { - type = string - description = "The version of the Databases for Elasticsearch instance. If no value is specified, the current preferred version of Databases for Elasticsearch is used." - default = null -} - -variable "backup_crn" { - type = string - description = "The CRN of a backup resource to restore from. The backup is created by a database deployment with the same service ID. The backup is loaded after provisioning and the new deployment starts up that uses that data. A backup CRN is in the format crn:v1:<…>:backup:. If omitted, the database is provisioned empty." - default = null - - validation { - condition = anytrue([ - var.backup_crn == null, - can(regex("^crn:.*:backup:", var.backup_crn)) - ]) - error_message = "backup_crn must be null OR starts with 'crn:' and contains ':backup:'" - } -} - variable "region" { + description = "The region where you want to deploy your instance." type = string - description = "The region where you want to deploy your instance, or the region in which your existing instance is in." default = "us-south" validation { @@ -77,28 +59,37 @@ variable "region" { } } -variable "plan" { +variable "existing_elasticsearch_instance_crn" { type = string - description = "The name of the service plan for your Databases for Elasticsearch instance. Possible values: `enterprise`, `platinum`." - default = "platinum" + default = null + description = "The CRN of an existing Databases for Elasticsearch instance. If no value is specified, a new instance is created." } -variable "existing_elasticsearch_instance_crn" { +variable "elasticsearch_version" { + description = "The version of the Databases for Elasticsearch instance. If no value is specified, the current preferred version of Databases for Elasticsearch is used." type = string default = null - description = "The CRN of an existing Databases for Elasticsearch instance. If no value is specified, a new instance is created." +} + +variable "plan" { + type = string + description = "The name of the service plan for your Databases for Elasticsearch instance. Possible values: `enterprise`, `platinum`." + default = "platinum" + nullable = false } variable "enable_elser_model" { type = bool description = "Set it to true to install and start the Elastic's Natural Language Processing model. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-elser-embeddings-elasticsearch)" default = false + nullable = false } variable "elser_model_type" { type = string description = "Trained ELSER model to be used for Elastic's Natural Language Processing. Possible values: `.elser_model_1`, `.elser_model_2` and `.elser_model_2_linux-x86_64`. Applies only if also 'plan' is set to 'platinum' and 'enable_elser_model' is enabled. [Learn more](https://www.elastic.co/guide/en/machine-learning/current/ml-nlp-elser.html)" default = ".elser_model_2_linux-x86_64" + nullable = false validation { condition = contains([".elser_model_1", ".elser_model_2", ".elser_model_2_linux-x86_64"], var.elser_model_type) error_message = "The specified elser_model_type is not a valid selection!" @@ -109,6 +100,16 @@ variable "elser_model_type" { # ICD hosting model properties ############################################################################## +variable "service_endpoints" { + type = string + description = "The type of endpoint of the database instance. Possible values: `public`, `private`, `public-and-private`." + default = "private" + + validation { + condition = can(regex("public|public-and-private|private", var.service_endpoints)) + error_message = "Valid values for service_endpoints are 'public', 'public-and-private', and 'private'" + } +} variable "members" { type = number @@ -116,6 +117,12 @@ variable "members" { default = 3 } +variable "member_memory_mb" { + type = number + description = "The memory per member that is allocated. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling)" + default = 4096 +} + variable "member_cpu_count" { type = number description = "The dedicated CPU per member that is allocated. For shared CPU, set to 0. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling)." @@ -140,15 +147,15 @@ variable "member_host_flavor" { } } -variable "member_memory_mb" { - type = number - description = "The memory per member that is allocated. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling)" - default = 4096 +variable "service_credential_names" { + description = "Map of name, role for service credentials that you want to create for the database. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/solutions/fully-configurable/DA-types.md#svc-credential-name)" + type = map(string) + default = {} } variable "admin_pass" { type = string - description = "The password for the database administrator. If the admin password is null, then it is created automatically. You must set 'existing_secrets_manager_instance_crn' to store admin pass into secrets manager. You can specify more users in a user block." + description = "The password for the database administrator. If no admin password is provided (i.e., it is null), one will be generated automatically. Additional users can be added using a user block." default = null sensitive = true } @@ -157,23 +164,17 @@ variable "users" { type = list(object({ name = string password = string # pragma: allowlist secret - type = optional(string) + type = string # "type" is required to generate the connection string for the outputs. role = optional(string) })) - description = "The list of users that have access to the database. Multiple blocks are allowed. The user password must be 10-32 characters. In most cases, you can use IAM service credentials (by specifying `service_credential_names`) to control access to the database instance. This block creates native database users. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/tree/main/solutions/standard/DA-types.md)." default = [] sensitive = true + description = "A list of users that you want to create on the database. Multiple blocks are allowed. The user password must be in the range of 10-32 characters. Be warned that in most case using IAM service credentials (via the var.service_credential_names) is sufficient to control access to the Elasticsearch instance. This blocks creates native Elasticsearch database users. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/solutions/fully-configurable/DA-types.md#users)" } -variable "service_credential_names" { - type = map(string) - description = "The map of name and role for service credentials that you want to create for the database. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/tree/main/solutions/standard/DA-types.md)." - default = {} -} - -variable "tags" { - type = list(any) - description = "The list of tags to be added to the Databases for Elasticsearch instance." +variable "resource_tags" { + type = list(string) + description = "The list of resource tags to be added to the Databases for Elasticsearch instance." default = [] } @@ -183,98 +184,46 @@ variable "access_tags" { default = [] } -############################################################## -# Auto Scaling -############################################################## - -variable "auto_scaling" { - type = object({ - disk = object({ - capacity_enabled = optional(bool, false) - free_space_less_than_percent = optional(number, 10) - io_above_percent = optional(number, 90) - io_enabled = optional(bool, false) - io_over_period = optional(string, "15m") - rate_increase_percent = optional(number, 10) - rate_limit_mb_per_member = optional(number, 3670016) - rate_period_seconds = optional(number, 900) - rate_units = optional(string, "mb") - }) - memory = object({ - io_above_percent = optional(number, 90) - io_enabled = optional(bool, false) - io_over_period = optional(string, "15m") - rate_increase_percent = optional(number, 10) - rate_limit_mb_per_member = optional(number, 114688) - rate_period_seconds = optional(number, 900) - rate_units = optional(string, "mb") - }) - }) - description = "The rules to allow the database to increase resources in response to usage. Only a single autoscaling block is allowed. Make sure you understand the effects of autoscaling, especially for production environments. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/tree/main/solutions/standard/DA-types.md)." - default = null -} - ############################################################## # Encryption ############################################################## -variable "use_ibm_owned_encryption_key" { +variable "kms_encryption_enabled" { type = bool - description = "IBM Cloud Databases will secure your deployment's data at rest automatically with an encryption key that IBM hold. Alternatively, you may select your own Key Management System instance and encryption key (Key Protect or Hyper Protect Crypto Services) by setting this to false. If setting to false, a value must be passed for `existing_kms_instance_crn` to create a new key, or `existing_kms_key_crn` and/or `existing_backup_kms_key_crn` to use an existing key." + description = "Set to true to enable KMS encryption using customer-managed keys. When enabled, you must provide a value for at least one of the following: existing_kms_instance_crn, existing_kms_key_crn, or existing_backup_kms_key_crn. If set to false, IBM-owned encryption is used (i.e., encryption keys managed and held by IBM)." default = false - # this validation ensures IBM-owned key is not used when KMS details are provided validation { - condition = ( + condition = (!var.kms_encryption_enabled || var.existing_elasticsearch_instance_crn != null || - !(var.use_ibm_owned_encryption_key && ( - var.existing_kms_instance_crn != null || - var.existing_kms_key_crn != null || - var.existing_backup_kms_key_crn != null - )) + var.existing_kms_instance_crn != null || + var.existing_kms_key_crn != null || + var.existing_backup_kms_key_crn != null ) - error_message = "When setting values for 'existing_kms_instance_crn', 'existing_kms_key_crn' or 'existing_backup_kms_key_crn', the 'use_ibm_owned_encryption_key' input must be set to false." + error_message = "When 'kms_encryption_enabled' is true, you must provide either 'existing_backup_kms_key_crn', 'existing_kms_instance_crn' (to create a new key) or 'existing_kms_key_crn' (to use an existing key)." } - # this validation ensures key info is provided when IBM-owned key is disabled and no Elasticsearch instance is given validation { - condition = !( - var.existing_elasticsearch_instance_crn == null && - var.use_ibm_owned_encryption_key == false && - var.existing_kms_instance_crn == null && - var.existing_kms_key_crn == null - ) - error_message = "When 'use_ibm_owned_encryption_key' is false, you must provide either 'existing_kms_instance_crn' (to create a new key) or 'existing_kms_key_crn' (to use an existing key)." + condition = (var.existing_kms_instance_crn == null && var.existing_kms_key_crn == null && var.existing_backup_kms_key_crn == null) || var.kms_encryption_enabled + error_message = "When either 'existing_kms_instance_crn', 'existing_kms_key_crn' or 'existing_backup_kms_key_crn' is set then 'kms_encryption_enabled' must be set to true." } } variable "existing_kms_instance_crn" { type = string - description = "The CRN of a Key Protect or Hyper Protect Crypto Services instance. Required to create a new encryption key and key ring which will be used to encrypt both deployment data and backups. Applies only if `use_ibm_owned_encryption_key` is false. To use an existing key, pass values for `existing_kms_key_crn` and/or `existing_backup_kms_key_crn`. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." + description = "The CRN of a Key Protect or Hyper Protect Crypto Services instance. Required to create a new encryption key and key ring which will be used to encrypt both deployment data and backups. To use an existing key, pass values for `existing_kms_key_crn` and/or `existing_backup_kms_key_crn`. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." default = null } variable "existing_kms_key_crn" { type = string - description = "The CRN of a Key Protect or Hyper Protect Crypto Services encryption key to encrypt your data. Applies only if `use_ibm_owned_encryption_key` is false. By default this key is used for both deployment data and backups, but this behaviour can be altered using the optional `existing_backup_kms_key_crn` input. If no value is passed a new key will be created in the instance specified in the `existing_kms_instance_crn` input. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." - default = null -} - -variable "existing_backup_kms_key_crn" { - type = string - description = "The CRN of a Key Protect or Hyper Protect Crypto Services encryption key that you want to use for encrypting the disk that holds deployment backups. Applies only if `use_ibm_owned_encryption_key` is false. If no value is passed, the value of `existing_kms_key_crn` is used. If no value is passed for `existing_kms_key_crn`, a new key will be created in the instance specified in the `existing_kms_instance_crn` input. Alternatively set `use_default_backup_encryption_key` to true to use the IBM Cloud Databases default encryption. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." + description = "The CRN of a Key Protect or Hyper Protect Crypto Services encryption key to encrypt your data. By default this key is used for both deployment data and backups, but this behaviour can be altered using the optional `existing_backup_kms_key_crn` input. If no value is passed a new key will be created in the instance specified in the `existing_kms_instance_crn` input. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." default = null } -variable "use_default_backup_encryption_key" { - type = bool - description = "When `use_ibm_owned_encryption_key` is set to false, backups will be encrypted with either the key specified in `existing_kms_key_crn`, in `existing_backup_kms_key_crn`, or with a new key that will be created in the instance specified in the `existing_kms_instance_crn` input. If you do not want to use your own key for backups encryption, you can set this to `true` to use the IBM Cloud Databases default encryption for backups. Alternatively set `use_ibm_owned_encryption_key` to true to use the default encryption for both backups and deployment data." - default = false -} - variable "kms_endpoint_type" { type = string - description = "The type of endpoint to use for communicating with the Key Protect or Hyper Protect Crypto Services instance. Possible values: `public`, `private`." + description = "The type of endpoint to use for communicating with the Key Protect or Hyper Protect Crypto Services instance. Possible values: `public`, `private`. Applies only if `existing_kms_key_crn` is not specified." default = "private" validation { condition = can(regex("public|private", var.kms_endpoint_type)) @@ -282,34 +231,102 @@ variable "kms_endpoint_type" { } } -variable "skip_es_kms_auth_policy" { +variable "skip_elasticsearch_kms_auth_policy" { type = bool - description = "Set to true to skip the creation of IAM authorization policies that permits all Databases for Elasticsearch instances in the given resource group 'Reader' access to the Key Protect or Hyper Protect Crypto Services key. This policy is required in order to enable KMS encryption, so only skip creation if there is one already present in your account. No policy is created if `use_ibm_owned_encryption_key` is true." + description = "Set to true to skip the creation of IAM authorization policies that permits all Databases for Elasticsearch instances in the given resource group 'Reader' access to the Key Protect or Hyper Protect Crypto Services key. This policy is required in order to enable KMS encryption, so only skip creation if there is one already present in your account. No policy is created if `kms_encryption_enabled` is false." default = false } -variable "elasticsearch_key_ring_name" { +variable "ibmcloud_kms_api_key" { + type = string + description = "The IBM Cloud API key that can create a root key and key ring in the key management service (KMS) instance. If not specified, the 'ibmcloud_api_key' variable is used. Specify this key if the instance in `existing_kms_instance_crn` is in an account that's different from the Elastic Search instance. Leave this input empty if the same account owns both instances." + sensitive = true + default = null +} + +variable "key_ring_name" { type = string default = "elasticsearch-key-ring" - description = "The name for the key ring created for the ElasticSearch key. Applies only if not specifying an existing key or using IBM owned keys. If a prefix input variable is specified, the prefix is added to the name in the `-` format." + description = "The name for the key ring created for the ElasticSearch key. Applies only if not specifying an existing key. If a prefix input variable is specified, the prefix is added to the name in the `-` format." } -variable "elasticsearch_key_name" { +variable "key_name" { type = string default = "elasticsearch-key" - description = "The name for the key created for the ElasticSearch key. Applies only if not specifying an existing key or using IBM owned keys. If a prefix input variable is specified, the prefix is added to the name in the `-` format." + description = "The name for the key created for the ElasticSearch key. Applies only if not specifying an existing key. If a prefix input variable is specified, the prefix is added to the name in the `-` format." } -variable "ibmcloud_kms_api_key" { +variable "existing_backup_kms_key_crn" { type = string - description = "The IBM Cloud API key that can create a root key and key ring in the key management service (KMS) instance. If not specified, the 'ibmcloud_api_key' variable is used. Specify this key if the instance in `existing_kms_instance_crn` is in an account that's different from the Elastic Search instance. Leave this input empty if the same account owns both instances." - sensitive = true + description = "The CRN of a Key Protect or Hyper Protect Crypto Services encryption key that you want to use for encrypting the disk that holds deployment backups. Applies only if `kms_encryption_enabled` is true. If no value is passed, the value of `existing_kms_key_crn` is used. If no value is passed for `existing_kms_key_crn`, a new key will be created in the instance specified in the `existing_kms_instance_crn` input. Alternatively set `kms_encryption_enabled` to false to use the IBM Cloud Databases default encryption. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." default = null } -############################################################################## -## Secrets Manager Service Credentials -############################################################################## +variable "use_default_backup_encryption_key" { + type = bool + description = "When `kms_encryption_enabled` is set to true, backups will be encrypted with either the key specified in `existing_kms_key_crn`, in `existing_backup_kms_key_crn`, or with a new key that will be created in the instance specified in the `existing_kms_instance_crn` input. If you do not want to use your own key for backups encryption, you can set this to `true` to use the IBM Cloud Databases default encryption for backups. Alternatively set `kms_encryption_enabled` to false to use the default encryption for both backups and deployment data." + default = false +} + +variable "backup_crn" { + type = string + description = "The CRN of a backup resource to restore from. The backup is created by a database deployment with the same service ID. The backup is loaded after provisioning and the new deployment starts up that uses that data. A backup CRN is in the format crn:v1:<…>:backup:. If omitted, the database is provisioned empty." + default = null + + validation { + condition = anytrue([ + var.backup_crn == null, + can(regex("^crn:.*:backup:", var.backup_crn)) + ]) + error_message = "backup_crn must be null OR starts with 'crn:' and contains ':backup:'" + } +} + +variable "provider_visibility" { + description = "Set the visibility value for the IBM terraform provider. Supported values are `public`, `private`, `public-and-private`. [Learn more](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/guides/custom-service-endpoints)." + type = string + default = "private" + + validation { + condition = contains(["public", "private", "public-and-private"], var.provider_visibility) + error_message = "Invalid visibility option. Allowed values are 'public', 'private', or 'public-and-private'." + } +} + +############################################################## +# Auto Scaling +############################################################## + +variable "auto_scaling" { + type = object({ + disk = object({ + capacity_enabled = optional(bool, false) + free_space_less_than_percent = optional(number, 10) + io_above_percent = optional(number, 90) + io_enabled = optional(bool, false) + io_over_period = optional(string, "15m") + rate_increase_percent = optional(number, 10) + rate_limit_mb_per_member = optional(number, 3670016) + rate_period_seconds = optional(number, 900) + rate_units = optional(string, "mb") + }) + memory = object({ + io_above_percent = optional(number, 90) + io_enabled = optional(bool, false) + io_over_period = optional(string, "15m") + rate_increase_percent = optional(number, 10) + rate_limit_mb_per_member = optional(number, 114688) + rate_period_seconds = optional(number, 900) + rate_units = optional(string, "mb") + }) + }) + description = "Optional rules to allow the database to increase resources in response to usage. Only a single autoscaling block is allowed. Make sure you understand the effects of autoscaling, especially for production environments. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/solutions/fully-configurable/DA-types.md#autoscaling)" + default = null +} + +############################################################################# +# Secrets Manager Service Credentials +############################################################################# variable "existing_secrets_manager_instance_crn" { type = string @@ -345,7 +362,7 @@ variable "service_credential_secrets" { })) })) default = [] - description = "Service credential secrets configuration for Databases for Elasticsearch. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/solutions/standard/DA-types.md#service-credential-secrets)." + description = "Service credential secrets configuration for Databases for Elasticsearch. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/solutions/fully-configurable/DA-types.md#service-credential-secrets)." validation { # Service roles CRNs can be found at https://cloud.ibm.com/iam/roles, select the IBM Cloud Database and select the role @@ -370,7 +387,7 @@ variable "service_credential_secrets" { variable "skip_elasticsearch_to_secrets_manager_auth_policy" { type = bool default = false - description = "Whether an IAM authorization policy is created for Secrets Manager instance to create a service credential secrets for Databases for Elasticsearch. Set to `true` to use an existing policy." + description = "Whether an IAM authorization policy is created for Secrets Manager instance to create a service credential secrets for Databases for Elasticsearch. If set to false, the Secrets Manager instance passed by the user is granted the Key Manager access to the Elasticsearch instance created by the Deployable Architecture. Set to `true` to use an existing policy. The value of this is ignored if any value for 'existing_secrets_manager_instance_crn' is not passed." } variable "admin_pass_secrets_manager_secret_group" { @@ -484,7 +501,7 @@ variable "kibana_image_secret" { } variable "kibana_visibility" { - description = "Specify the visibility of Kibana application in order to define which endpoint is available for receiving the requests. Valid values are 'local_public', 'local_private' and 'local' and it is only applicable if `enable_kibana_dashboard` is true. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/solutions/standard/DA-types.md#options-for-kibana_visibility)." + description = "Specify the visibility of Kibana application in order to define which endpoint is available for receiving the requests. Valid values are 'local_public', 'local_private' and 'local' and it is only applicable if `enable_kibana_dashboard` is true. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/solutions/fully-configurable/DA-types.md#options-for-kibana_visibility)." type = string default = "local_private" validation { @@ -526,6 +543,6 @@ variable "cbr_rules" { })) }))) })) - description = "(Optional, list) List of context-based restrictions rules to create. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/tree/main/solutions/standard/DA-cbr_rules.md)" + description = "(Optional, list) List of context-based restrictions rules to create. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/tree/main/solutions/fully-configurable/DA-cbr_rules.md)" default = [] } diff --git a/solutions/standard/version.tf b/solutions/fully-configurable/version.tf similarity index 100% rename from solutions/standard/version.tf rename to solutions/fully-configurable/version.tf diff --git a/solutions/security-enforced/README.md b/solutions/security-enforced/README.md new file mode 100644 index 00000000..da8cc16f --- /dev/null +++ b/solutions/security-enforced/README.md @@ -0,0 +1,3 @@ +# Cloud automation for Elasticsearch (Security-enforced) + +:exclamation: **Important:** This solution is not intended to be called by other modules because it contains a provider configuration and is not compatible with the `for_each`, `count`, and `depends_on` arguments. For more information, see [Providers Within Modules](https://developer.hashicorp.com/terraform/language/modules/develop/providers). diff --git a/solutions/standard/catalogValidationValues.json.template b/solutions/security-enforced/catalogValidationValues.json.template similarity index 61% rename from solutions/standard/catalogValidationValues.json.template rename to solutions/security-enforced/catalogValidationValues.json.template index e69e502f..693f534a 100644 --- a/solutions/standard/catalogValidationValues.json.template +++ b/solutions/security-enforced/catalogValidationValues.json.template @@ -1,8 +1,8 @@ { "ibmcloud_api_key": $VALIDATION_APIKEY, "region": "us-south", - "tags": $TAGS, + "resource_tags": $TAGS, "name": $PREFIX, - "resource_group_name": $PREFIX, + "existing_resource_group_name": "geretain-test-permanent", "existing_kms_instance_crn": $HPCS_US_SOUTH_CRN } diff --git a/solutions/security-enforced/main.tf b/solutions/security-enforced/main.tf new file mode 100644 index 00000000..12f28ba8 --- /dev/null +++ b/solutions/security-enforced/main.tf @@ -0,0 +1,58 @@ +module "elasticsearch" { + source = "../fully-configurable" + ibmcloud_api_key = var.ibmcloud_api_key + existing_resource_group_name = var.existing_resource_group_name + prefix = var.prefix + name = var.name + provider_visibility = "private" + region = var.region + existing_elasticsearch_instance_crn = var.existing_elasticsearch_instance_crn + elasticsearch_version = var.elasticsearch_version + backup_crn = var.backup_crn + plan = var.plan + enable_elser_model = var.enable_elser_model + elser_model_type = var.elser_model_type + # ICD hosting model properties + members = var.members + member_memory_mb = var.member_memory_mb + member_cpu_count = var.member_cpu_count + member_disk_mb = var.member_disk_mb + member_host_flavor = var.member_host_flavor + service_credential_names = var.service_credential_names + admin_pass = var.admin_pass + users = var.users + resource_tags = var.resource_tags + access_tags = var.access_tags + # Encryption + kms_encryption_enabled = true + existing_kms_instance_crn = var.existing_kms_instance_crn + existing_kms_key_crn = var.existing_kms_key_crn + kms_endpoint_type = "private" + skip_elasticsearch_kms_auth_policy = var.skip_elasticsearch_kms_auth_policy + ibmcloud_kms_api_key = var.ibmcloud_kms_api_key + key_ring_name = var.key_ring_name + key_name = var.key_name + existing_backup_kms_key_crn = var.existing_backup_kms_key_crn + use_default_backup_encryption_key = false + # Auto Scaling + auto_scaling = var.auto_scaling + # Secrets Manager Service Credentials + existing_secrets_manager_instance_crn = var.existing_secrets_manager_instance_crn + existing_secrets_manager_endpoint_type = "private" + service_credential_secrets = var.service_credential_secrets + skip_elasticsearch_to_secrets_manager_auth_policy = var.skip_elasticsearch_to_secrets_manager_auth_policy + admin_pass_secrets_manager_secret_group = var.admin_pass_secrets_manager_secret_group + use_existing_admin_pass_secrets_manager_secret_group = var.use_existing_admin_pass_secrets_manager_secret_group + admin_pass_secrets_manager_secret_name = var.admin_pass_secrets_manager_secret_name + service_endpoints = "private" + # Kibana Configuration + kibana_code_engine_new_project_name = var.kibana_code_engine_new_project_name + kibana_code_engine_new_app_name = var.kibana_code_engine_new_app_name + existing_code_engine_project_id = var.existing_code_engine_project_id + enable_kibana_dashboard = var.enable_kibana_dashboard + kibana_registry_namespace_image = var.kibana_registry_namespace_image + kibana_image_digest = var.kibana_image_digest + kibana_image_port = var.kibana_image_port + kibana_visibility = "local_private" + cbr_rules = var.cbr_rules +} diff --git a/solutions/security-enforced/outputs.tf b/solutions/security-enforced/outputs.tf new file mode 100644 index 00000000..34b0f96f --- /dev/null +++ b/solutions/security-enforced/outputs.tf @@ -0,0 +1,61 @@ +############################################################################## +# Outputs +############################################################################## + +output "id" { + description = "Elasticsearch instance id" + value = module.elasticsearch.id +} + +output "version" { + description = "Elasticsearch instance version" + value = module.elasticsearch.version +} + +output "guid" { + description = "Elasticsearch instance guid" + value = module.elasticsearch.guid +} + +output "crn" { + description = "Elasticsearch instance crn" + value = module.elasticsearch.crn +} + +output "service_credentials_json" { + description = "Service credentials json map" + value = module.elasticsearch.service_credentials_json + sensitive = true +} + +output "service_credentials_object" { + description = "Service credentials object" + value = module.elasticsearch.service_credentials_object + sensitive = true +} + +output "hostname" { + description = "Database connection hostname" + value = module.elasticsearch.hostname +} + +output "port" { + description = "Database connection port" + value = module.elasticsearch.port +} + +output "cbr_rule_ids" { + description = "CBR rule ids created to restrict Elasticsearch" + value = module.elasticsearch.cbr_rule_ids +} + +output "adminuser" { + description = "Database admin user name" + value = module.elasticsearch.adminuser +} + +output "certificate_base64" { + description = "Database connection certificate" + value = module.elasticsearch.certificate_base64 + sensitive = true +} diff --git a/solutions/security-enforced/provider.tf b/solutions/security-enforced/provider.tf new file mode 100644 index 00000000..4c6add22 --- /dev/null +++ b/solutions/security-enforced/provider.tf @@ -0,0 +1 @@ +# Explicit provider config not required here as provider config in fully-configurable is used diff --git a/solutions/security-enforced/variables.tf b/solutions/security-enforced/variables.tf new file mode 100644 index 00000000..456d7d0b --- /dev/null +++ b/solutions/security-enforced/variables.tf @@ -0,0 +1,435 @@ +############################################################################## +# Input Variables +############################################################################## + +variable "ibmcloud_api_key" { + type = string + description = "The IBM Cloud API key to deploy resources." + sensitive = true +} + +variable "existing_resource_group_name" { + type = string + description = "The name of an existing resource group to provision resource in." + default = "Default" + nullable = false +} + +variable "prefix" { + type = string + nullable = true + description = "The prefix to be added to all resources created by this solution. To skip using a prefix, set this value to null or an empty string. The prefix must begin with a lowercase letter and may contain only lowercase letters, digits, and hyphens '-'. It should not exceed 16 characters, must not end with a hyphen('-'), and can not contain consecutive hyphens ('--'). Example: prod-0205-cos. [Learn more](https://terraform-ibm-modules.github.io/documentation/#/prefix.md)." + + validation { + # - null and empty string is allowed + # - Must not contain consecutive hyphens (--): length(regexall("--", var.prefix)) == 0 + # - Starts with a lowercase letter: [a-z] + # - Contains only lowercase letters (a–z), digits (0–9), and hyphens (-) + # - Must not end with a hyphen (-): [a-z0-9] + condition = (var.prefix == null || var.prefix == "" ? true : + alltrue([ + can(regex("^[a-z][-a-z0-9]*[a-z0-9]$", var.prefix)), + length(regexall("--", var.prefix)) == 0 + ]) + ) + error_message = "Prefix must begin with a lowercase letter and may contain only lowercase letters, digits, and hyphens '-'. It must not end with a hyphen('-'), and cannot contain consecutive hyphens ('--')." + } + + validation { + # must not exceed 16 characters in length + condition = length(var.prefix) <= 16 + error_message = "Prefix must not exceed 16 characters." + } +} + +variable "name" { + type = string + description = "The name of the Databases for Elasticsearch instance. If a prefix input variable is specified, the prefix is added to the name in the `-` format." + default = "elasticsearch" +} + +variable "region" { + description = "The region where you want to deploy your instance." + type = string + default = "us-south" +} + +variable "existing_elasticsearch_instance_crn" { + type = string + default = null + description = "The CRN of an existing Databases for Elasticsearch instance. If no value is specified, a new instance is created." +} + +variable "elasticsearch_version" { + description = "The version of the Databases for Elasticsearch instance. If no value is specified, the current preferred version of Databases for Elasticsearch is used." + type = string + default = null +} + +variable "plan" { + type = string + description = "The name of the service plan for your Databases for Elasticsearch instance. Possible values: `enterprise`, `platinum`." + default = "platinum" + nullable = false +} + +variable "enable_elser_model" { + type = bool + description = "Set it to true to install and start the Elastic's Natural Language Processing model. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-elser-embeddings-elasticsearch)" + default = false + nullable = false +} + +variable "elser_model_type" { + type = string + description = "Trained ELSER model to be used for Elastic's Natural Language Processing. Possible values: `.elser_model_1`, `.elser_model_2` and `.elser_model_2_linux-x86_64`. Applies only if also 'plan' is set to 'platinum' and 'enable_elser_model' is enabled. [Learn more](https://www.elastic.co/guide/en/machine-learning/current/ml-nlp-elser.html)" + default = ".elser_model_2_linux-x86_64" + nullable = false + validation { + condition = contains([".elser_model_1", ".elser_model_2", ".elser_model_2_linux-x86_64"], var.elser_model_type) + error_message = "The specified elser_model_type is not a valid selection!" + } +} + +############################################################################## +# ICD hosting model properties +############################################################################## + +variable "members" { + type = number + description = "The number of members that are allocated. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling)." + default = 3 +} + +variable "member_memory_mb" { + type = number + description = "The memory per member that is allocated. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling)" + default = 4096 +} + +variable "member_cpu_count" { + type = number + description = "The dedicated CPU per member that is allocated. For shared CPU, set to 0. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling)." + default = 3 +} + +variable "member_disk_mb" { + type = number + description = "The disk that is allocated per member. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling)." + default = 5120 +} + +# Use new hosting model for all DA +variable "member_host_flavor" { + type = string + description = "The host flavor per member. [Learn more](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/database#host_flavor)." + default = "b3c.4x16.encrypted" + # Prevent null or "", require multitenant or a machine type + validation { + condition = (length(var.member_host_flavor) > 0) + error_message = "Member host flavor must be specified. [Learn more](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/database#host_flavor)." + } +} + +variable "service_credential_names" { + description = "Map of name, role for service credentials that you want to create for the database. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/solutions/fully-configurable/DA-types.md#svc-credential-name)" + type = map(string) + default = {} +} + +variable "admin_pass" { + type = string + description = "The password for the database administrator. If no admin password is provided (i.e., it is null), one will be generated automatically. Additional users can be added using a user block." + default = null + sensitive = true +} + +variable "users" { + type = list(object({ + name = string + password = string # pragma: allowlist secret + type = string # "type" is required to generate the connection string for the outputs. + role = optional(string) + })) + default = [] + sensitive = true + description = "A list of users that you want to create on the database. Multiple blocks are allowed. The user password must be in the range of 10-32 characters. Be warned that in most case using IAM service credentials (via the var.service_credential_names) is sufficient to control access to the Elasticsearch instance. This blocks creates native Elasticsearch database users. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/solutions/fully-configurable/DA-types.md#users)" +} + +variable "resource_tags" { + type = list(string) + description = "The list of resource tags to be added to the Databases for Elasticsearch instance." + default = [] +} + +variable "access_tags" { + type = list(string) + description = "A list of access tags to apply to the Databases for Elasticsearch instance created by the solution. [Learn more](https://cloud.ibm.com/docs/account?topic=account-access-tags-tutorial)." + default = [] +} + +############################################################## +# Encryption +############################################################## + +variable "existing_kms_instance_crn" { + type = string + description = "The CRN of a Key Protect or Hyper Protect Crypto Services instance. Required to create a new encryption key and key ring which will be used to encrypt both deployment data and backups. To use an existing key, pass values for `existing_kms_key_crn` and/or `existing_backup_kms_key_crn`. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." + default = null +} + +variable "existing_kms_key_crn" { + type = string + description = "The CRN of a Key Protect or Hyper Protect Crypto Services encryption key to encrypt your data. By default this key is used for both deployment data and backups, but this behaviour can be altered using the optional `existing_backup_kms_key_crn` input. If no value is passed a new key will be created in the instance specified in the `existing_kms_instance_crn` input. Bare in mind that backups encryption is only available in certain regions. See [Bring your own key for backups](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok) and [Using the HPCS Key for Backup encryption](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs#use-hpcs-backups)." + default = null + + + validation { + condition = ( + (var.existing_kms_key_crn != null && var.existing_kms_instance_crn == null) || + (var.existing_kms_key_crn == null && var.existing_kms_instance_crn != null) + ) + error_message = "Either existing_kms_key_crn or existing_kms_instance_crn must be set, but not both." + } +} + +variable "skip_elasticsearch_kms_auth_policy" { + type = bool + description = "Whether to create an IAM authorization policy that permits all Databases for Elasticsearch instances in the resource group to read the encryption key from the Hyper Protect Crypto Services instance specified in the `existing_kms_instance_crn` variable." + default = false +} + +variable "ibmcloud_kms_api_key" { + type = string + description = "The IBM Cloud API key that can create a root key and key ring in the key management service (KMS) instance. If not specified, the 'ibmcloud_api_key' variable is used. Specify this key if the instance in `existing_kms_instance_crn` is in an account that's different from the ElasticSearch instance. Leave this input empty if the same account owns both instances." + sensitive = true + default = null +} + +variable "key_ring_name" { + type = string + default = "elasticsearch-key-ring" + description = "The name for the key ring created for the Databases for ElasticSearch key. Applies only if not specifying an existing key. If a prefix input variable is specified, the prefix is added to the name in the `-` format." +} + +variable "key_name" { + type = string + default = "elasticsearch-key" + description = "The name for the key created for the Databases for ElasticSearch key. Applies only if not specifying an existing key. If a prefix input variable is specified, the prefix is added to the name in the `-` format." +} + +variable "existing_backup_kms_key_crn" { + type = string + description = "The CRN of a Key Protect or Hyper Protect Crypto Services encryption key that you want to use for encrypting the disk that holds deployment backups. If no value is passed, the value of `existing_kms_key_crn` is used. If no value is passed for `existing_kms_key_crn`, a new key will be created in the instance specified in the `existing_kms_instance_crn` input." + default = null +} + +variable "backup_crn" { + type = string + description = "The CRN of a backup resource to restore from. The backup is created by a database deployment with the same service ID. The backup is loaded after provisioning and the new deployment starts up that uses that data. A backup CRN is in the format crn:v1:<…>:backup:. If omitted, the database is provisioned empty." + default = null + + validation { + condition = anytrue([ + var.backup_crn == null, + can(regex("^crn:.*:backup:", var.backup_crn)) + ]) + error_message = "backup_crn must be null OR starts with 'crn:' and contains ':backup:'" + } +} + +############################################################## +# Auto Scaling +############################################################## + +variable "auto_scaling" { + type = object({ + disk = object({ + capacity_enabled = optional(bool, false) + free_space_less_than_percent = optional(number, 10) + io_above_percent = optional(number, 90) + io_enabled = optional(bool, false) + io_over_period = optional(string, "15m") + rate_increase_percent = optional(number, 10) + rate_limit_mb_per_member = optional(number, 3670016) + rate_period_seconds = optional(number, 900) + rate_units = optional(string, "mb") + }) + memory = object({ + io_above_percent = optional(number, 90) + io_enabled = optional(bool, false) + io_over_period = optional(string, "15m") + rate_increase_percent = optional(number, 10) + rate_limit_mb_per_member = optional(number, 114688) + rate_period_seconds = optional(number, 900) + rate_units = optional(string, "mb") + }) + }) + description = "Optional rules to allow the database to increase resources in response to usage. Only a single autoscaling block is allowed. Make sure you understand the effects of autoscaling, especially for production environments. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/tree/main/solutions/fully-configurable/DA-types.md#autoscaling)" + default = null +} + +############################################################################# +# Secrets Manager Service Credentials +############################################################################# + +variable "existing_secrets_manager_instance_crn" { + type = string + default = null + description = "The CRN of existing secrets manager to use to create service credential secrets for Databases for Elasticsearch instance." +} + +variable "service_credential_secrets" { + type = list(object({ + secret_group_name = string + secret_group_description = optional(string) + existing_secret_group = optional(bool) + service_credentials = list(object({ + secret_name = string + service_credentials_source_service_role_crn = string + secret_labels = optional(list(string)) + secret_auto_rotation = optional(bool) + secret_auto_rotation_unit = optional(string) + secret_auto_rotation_interval = optional(number) + service_credentials_ttl = optional(string) + service_credential_secret_description = optional(string) + + })) + })) + default = [] + description = "Service credential secrets configuration for Databases for Elasticsearch. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/blob/main/solutions/fully-configurable/DA-types.md#service-credential-secrets)." + + validation { + # Service roles CRNs can be found at https://cloud.ibm.com/iam/roles, select the IBM Cloud Database and select the role + condition = alltrue([ + for group in var.service_credential_secrets : alltrue([ + # crn:v?:bluemix; two non-empty segments; three possibly empty segments; :serviceRole or role: non-empty segment + for credential in group.service_credentials : can(regex("^crn:v[0-9]:bluemix(:..*){2}(:.*){3}:(serviceRole|role):..*$", credential.service_credentials_source_service_role_crn)) + ]) + ]) + error_message = "service_credentials_source_service_role_crn must be a serviceRole CRN. See https://cloud.ibm.com/iam/roles" + } + + validation { + condition = ( + length(var.service_credential_secrets) == 0 || + var.existing_secrets_manager_instance_crn != null + ) + error_message = "`existing_secrets_manager_instance_crn` is required when adding service credentials to a secrets manager secret." + } +} + +variable "skip_elasticsearch_to_secrets_manager_auth_policy" { + type = bool + default = false + description = "Whether an IAM authorization policy is created for Secrets Manager instance to create a service credential secrets for Databases for Elasticsearch. If set to false, the Secrets Manager instance passed by the user is granted the Key Manager access to the Elasticsearch instance created by the Deployable Architecture. Set to `true` to use an existing policy. The value of this is ignored if any value for 'existing_secrets_manager_instance_crn' is not passed." +} + +variable "admin_pass_secrets_manager_secret_group" { + type = string + description = "The name of a new or existing secrets manager secret group for admin password. To use existing secret group, `use_existing_admin_pass_secrets_manager_secret_group` must be set to `true`. If a prefix input variable is specified, the prefix is added to the name in the `-` format." + default = "elasticsearch-secrets" + + validation { + condition = ( + var.existing_secrets_manager_instance_crn == null || + var.admin_pass_secrets_manager_secret_group != null + ) + error_message = "`admin_pass_secrets_manager_secret_group` is required when `existing_secrets_manager_instance_crn` is set." + } +} + +variable "use_existing_admin_pass_secrets_manager_secret_group" { + type = bool + description = "Whether to use an existing secrets manager secret group for admin password." + default = false +} + +variable "admin_pass_secrets_manager_secret_name" { + type = string + description = "The name of a new elasticsearch administrator secret. If a prefix input variable is specified, the prefix is added to the name in the `-` format." + default = "elasticsearch-admin-password" + + validation { + condition = ( + var.existing_secrets_manager_instance_crn == null || + var.admin_pass_secrets_manager_secret_name != null + ) + error_message = "`admin_pass_secrets_manager_secret_name` is required when `existing_secrets_manager_instance_crn` is set." + } +} + +############################################################## +# Kibana Configuration +############################################################## + +variable "kibana_code_engine_new_project_name" { + type = string + description = "The Code Engine project name. If a prefix input variable is specified, the prefix is added to the name in the `-` format." + default = "ce-kibana-project" +} + +variable "kibana_code_engine_new_app_name" { + type = string + description = "The Code Engine application name. If a prefix input variable is specified, the prefix is added to the name in the `-` format." + default = "ce-kibana-app" +} + +variable "existing_code_engine_project_id" { + type = string + description = "Existing code engine project ID to deploy Kibana. If no value is passed, a new code engine project will be created." + default = null +} + +variable "enable_kibana_dashboard" { + type = bool + description = "Set to true to deploy Kibana in Code Engine. NOTE: By default, the Kibana image will be pulled from the official Elastic registry (docker.elastic.co) and is not certified by IBM, however this can be overridden using the `kibana_registry_namespace_image` and `kibana_image_digest` inputs." + default = false +} + +variable "kibana_registry_namespace_image" { + type = string + description = "The Kibana image reference in the format of `[registry-url]/[namespace]/[image]`. This value is used only when `enable_kibana_dashboard` is set to true." + default = "docker.elastic.co/kibana/kibana" +} + +variable "kibana_image_digest" { + type = string + description = "When `enable_kibana_dashboard` is set to true, Kibana is deployed using an image tag compatible with the Elasticsearch version. Alternatively, an image digest in the format `sha256:xxxxx...` can also be specified but it must correspond to a version compatible with the Elasticsearch instance." + default = null + validation { + condition = var.kibana_image_digest == null || can(regex("^sha256:", var.kibana_image_digest)) + error_message = "If provided, the value of kibana_image_digest must start with 'sha256:'." + } + + +} +variable "kibana_image_port" { + description = "Specify the port number used to connect to the Kibana service exposed by the container image. Default port is 5601 and it is only applicable if `enable_kibana_dashboard` is true" + type = number + default = 5601 +} + +############################################################## +# Context-based restriction (CBR) +############################################################## + +variable "cbr_rules" { + type = list(object({ + description = string + account_id = string + rule_contexts = list(object({ + attributes = optional(list(object({ + name = string + value = string + }))) })) + enforcement_mode = string + operations = optional(list(object({ + api_types = list(object({ + api_type_id = string + })) + }))) + })) + description = "(Optional, list) List of context-based restrictions rules to create. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-icd-elasticsearch/tree/main/solutions/fully-configurable/DA-cbr_rules.md)" + default = [] +} diff --git a/solutions/security-enforced/version.tf b/solutions/security-enforced/version.tf new file mode 100644 index 00000000..70b38b5b --- /dev/null +++ b/solutions/security-enforced/version.tf @@ -0,0 +1,6 @@ +terraform { + required_version = ">= 1.9.0" + # Lock DA into an exact provider version - renovate automation will keep it updated + required_providers { + } +} diff --git a/solutions/standard/README.md b/solutions/standard/README.md deleted file mode 100644 index 8fd577c9..00000000 --- a/solutions/standard/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# IBM Cloud Databases for Elasticsearch - -This architecture creates an instance of IBM Cloud Databases for Elasticsearch and supports provisioning of the following resources: - -- A resource group, if one is not passed in. -- A KMS root key, if one is not passed in. -- An IBM Cloud Databases for Elasticsearch instance with KMS encryption. -- Autoscaling rules for the database instance, if provided. -- Install and start the Elastic's Natural Language Processing model, if enabled. -- Kibana dashboard for Elasticsearch, if enabled. - -**Note on accessing Kibana:** If Kibana is enabled, you can access the Kibana application over a IBM private network using the method outlined [here](https://cloud.ibm.com/docs/codeengine?topic=codeengine-vpe). - -**Note on setting kibana_visibility:** When the Kibana application visibility is changed from private to public using kibana_visibility variable, it will become accessible from the public Internet. However, access via the IBM Cloud private network will no longer be available. This change takes effect immediately, potentially impacting active users or integrations. It is important to consider the associated security implications before proceeding. - -![fscloud-elastic-search](../../reference-architecture/deployable-architecture-elasticsearch.svg) - -:exclamation: **Important:** This solution is not intended to be called by other modules because it contains a provider configuration and is not compatible with the `for_each`, `count`, and `depends_on` arguments. For more information, see [Providers Within Modules](https://developer.hashicorp.com/terraform/language/modules/develop/providers). diff --git a/solutions/standard/moved.tf b/solutions/standard/moved.tf deleted file mode 100644 index ddd70738..00000000 --- a/solutions/standard/moved.tf +++ /dev/null @@ -1,4 +0,0 @@ -moved { - from = module.elasticsearch - to = module.elasticsearch[0] -} diff --git a/solutions/standard/provider.tf b/solutions/standard/provider.tf deleted file mode 100644 index 515e60c2..00000000 --- a/solutions/standard/provider.tf +++ /dev/null @@ -1,12 +0,0 @@ -provider "ibm" { - ibmcloud_api_key = var.ibmcloud_api_key - region = var.region - visibility = var.provider_visibility -} - -provider "ibm" { - alias = "kms" - ibmcloud_api_key = var.ibmcloud_kms_api_key != null ? var.ibmcloud_kms_api_key : var.ibmcloud_api_key - region = local.kms_region - visibility = var.provider_visibility -} diff --git a/tests/pr_test.go b/tests/pr_test.go index 98877973..9631c8d5 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -25,7 +25,8 @@ import ( const completeExampleTerraformDir = "examples/complete" const fscloudExampleTerraformDir = "examples/fscloud" -const standardSolutionTerraformDir = "solutions/standard" +const fullyConfigurableSolutionTerraformDir = "solutions/fully-configurable" +const securityEnforcedSolutionTerraformDir = "solutions/security-enforced" const latestVersion = "8.15" // Use existing resource group @@ -58,27 +59,25 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestRunStandardSolutionSchematics(t *testing.T) { +func TestRunFullyConfigurableSolutionSchematics(t *testing.T) { t.Parallel() options := testschematic.TestSchematicOptionsDefault(&testschematic.TestSchematicOptions{ Testing: t, TarIncludePatterns: []string{ "*.tf", - fmt.Sprintf("%s/*.tf", standardSolutionTerraformDir), - fmt.Sprintf("%s/*.tf", fscloudExampleTerraformDir), - fmt.Sprintf("%s/*.tf", "modules/fscloud"), + fmt.Sprintf("%s/*.tf", fullyConfigurableSolutionTerraformDir), fmt.Sprintf("%s/*.sh", "scripts"), }, - TemplateFolder: standardSolutionTerraformDir, + TemplateFolder: fullyConfigurableSolutionTerraformDir, BestRegionYAMLPath: regionSelectionPath, - Prefix: "els-sr-da", + Prefix: "els-fc-da", ResourceGroup: resourceGroup, DeleteWorkspaceOnFail: false, WaitJobCompleteMinutes: 60, }) - serviceCredentialSecrets := []map[string]any{ + serviceCredentialSecrets := []map[string]interface{}{ { "secret_group_name": fmt.Sprintf("%s-secret-group", options.Prefix), "service_credentials": []map[string]string{ @@ -96,16 +95,16 @@ func TestRunStandardSolutionSchematics(t *testing.T) { options.TerraformVars = []testschematic.TestSchematicTerraformVar{ {Name: "ibmcloud_api_key", Value: options.RequiredEnvironmentVars["TF_VAR_ibmcloud_api_key"], DataType: "string", Secure: true}, {Name: "access_tags", Value: permanentResources["accessTags"], DataType: "list(string)"}, + {Name: "kms_encryption_enabled", Value: true, DataType: "bool"}, {Name: "existing_kms_instance_crn", Value: permanentResources["hpcs_south_crn"], DataType: "string"}, - {Name: "existing_backup_kms_key_crn", Value: permanentResources["hpcs_south_root_key_crn"], DataType: "string"}, {Name: "kms_endpoint_type", Value: "private", DataType: "string"}, - {Name: "resource_group_name", Value: options.Prefix, DataType: "string"}, + {Name: "existing_resource_group_name", Value: resourceGroup, DataType: "string"}, {Name: "plan", Value: "platinum", DataType: "string"}, {Name: "enable_elser_model", Value: true, DataType: "bool"}, {Name: "service_credential_names", Value: "{\"admin_test\": \"Administrator\", \"editor_test\": \"Editor\"}", DataType: "map(string)"}, {Name: "existing_secrets_manager_instance_crn", Value: permanentResources["secretsManagerCRN"], DataType: "string"}, {Name: "service_credential_secrets", Value: serviceCredentialSecrets, DataType: "list(object)"}, - {Name: "admin_pass", Value: GetRandomAdminPassword(t), DataType: "string"}, + {Name: "service_endpoints", Value: "private", DataType: "string"}, {Name: "admin_pass_secrets_manager_secret_group", Value: options.Prefix, DataType: "string"}, {Name: "admin_pass_secrets_manager_secret_name", Value: options.Prefix, DataType: "string"}, {Name: "enable_kibana_dashboard", Value: true, DataType: "bool"}, @@ -118,24 +117,23 @@ func TestRunStandardSolutionSchematics(t *testing.T) { assert.Nil(t, err, "This should not have errored") } -func TestRunStandardUpgradeSolution(t *testing.T) { +func TestRunSecurityEnforcedUpgradeSolution(t *testing.T) { t.Parallel() options := testhelper.TestOptionsDefault(&testhelper.TestOptions{ Testing: t, - TerraformDir: standardSolutionTerraformDir, + TerraformDir: securityEnforcedSolutionTerraformDir, BestRegionYAMLPath: regionSelectionPath, Prefix: "els-st-da-upg", ResourceGroup: resourceGroup, CheckApplyResultForUpgrade: true, }) - options.TerraformVars = map[string]any{ - "access_tags": permanentResources["accessTags"], - "existing_kms_instance_crn": permanentResources["hpcs_south_crn"], - "kms_endpoint_type": "public", - "resource_group_name": options.Prefix, - "provider_visibility": "public", + options.TerraformVars = map[string]interface{}{ + "prefix": options.Prefix, + "access_tags": permanentResources["accessTags"], + "existing_kms_instance_crn": permanentResources["hpcs_south_crn"], + "existing_resource_group_name": resourceGroup, // Currently, we can not have upgrade test for elser model, because test provision private endpoint for ES (fscloud profile), and script can not connect to private ES API without schematics // "plan": "platinum", // "enable_elser_model": true, @@ -149,6 +147,61 @@ func TestRunStandardUpgradeSolution(t *testing.T) { } } +func TestRunSecurityEnforcedSolutionSchematics(t *testing.T) { + t.Parallel() + + options := testschematic.TestSchematicOptionsDefault(&testschematic.TestSchematicOptions{ + Testing: t, + TarIncludePatterns: []string{ + "*.tf", + fmt.Sprintf("%s/*.tf", securityEnforcedSolutionTerraformDir), + fmt.Sprintf("%s/*.tf", fullyConfigurableSolutionTerraformDir), + fmt.Sprintf("%s/*.sh", "scripts"), + }, + TemplateFolder: securityEnforcedSolutionTerraformDir, + BestRegionYAMLPath: regionSelectionPath, + Prefix: "els-se-da", + ResourceGroup: resourceGroup, + DeleteWorkspaceOnFail: false, + WaitJobCompleteMinutes: 60, + }) + + serviceCredentialSecrets := []map[string]interface{}{ + { + "secret_group_name": fmt.Sprintf("%s-secret-group", options.Prefix), + "service_credentials": []map[string]string{ + { + "secret_name": fmt.Sprintf("%s-cred-reader", options.Prefix), + "service_credentials_source_service_role_crn": "crn:v1:bluemix:public:iam::::role:Viewer", + }, + { + "secret_name": fmt.Sprintf("%s-cred-writer", options.Prefix), + "service_credentials_source_service_role_crn": "crn:v1:bluemix:public:iam::::role:Editor", + }, + }, + }, + } + + options.TerraformVars = []testschematic.TestSchematicTerraformVar{ + {Name: "ibmcloud_api_key", Value: options.RequiredEnvironmentVars["TF_VAR_ibmcloud_api_key"], DataType: "string", Secure: true}, + {Name: "kms_encryption_enabled", Value: true, DataType: "bool"}, + {Name: "access_tags", Value: permanentResources["accessTags"], DataType: "list(string)"}, + {Name: "existing_kms_instance_crn", Value: permanentResources["hpcs_south_crn"], DataType: "string"}, + {Name: "existing_resource_group_name", Value: resourceGroup, DataType: "string"}, + {Name: "plan", Value: "platinum", DataType: "string"}, + {Name: "enable_elser_model", Value: true, DataType: "bool"}, + {Name: "service_credential_names", Value: "{\"admin_test\": \"Administrator\", \"editor_test\": \"Editor\"}", DataType: "map(string)"}, + {Name: "existing_secrets_manager_instance_crn", Value: permanentResources["secretsManagerCRN"], DataType: "string"}, + {Name: "service_credential_secrets", Value: serviceCredentialSecrets, DataType: "list(object)"}, + {Name: "admin_pass_secrets_manager_secret_group", Value: options.Prefix, DataType: "string"}, + {Name: "admin_pass_secrets_manager_secret_name", Value: options.Prefix, DataType: "string"}, + {Name: "enable_kibana_dashboard", Value: true, DataType: "bool"}, + {Name: "prefix", Value: options.Prefix, DataType: "string"}, + } + err := options.RunSchematicTest() + assert.Nil(t, err, "This should not have errored") +} + func TestRunExistingInstance(t *testing.T) { t.Parallel() prefix := fmt.Sprintf("elastic-t-%s", strings.ToLower(random.UniqueId())) @@ -191,12 +244,12 @@ func TestRunExistingInstance(t *testing.T) { Testing: t, TarIncludePatterns: []string{ "*.tf", - fmt.Sprintf("%s/*.tf", standardSolutionTerraformDir), + fmt.Sprintf("%s/*.tf", fullyConfigurableSolutionTerraformDir), fmt.Sprintf("%s/*.tf", fscloudExampleTerraformDir), fmt.Sprintf("%s/*.tf", "modules/fscloud"), fmt.Sprintf("%s/*.sh", "scripts"), }, - TemplateFolder: standardSolutionTerraformDir, + TemplateFolder: fullyConfigurableSolutionTerraformDir, BestRegionYAMLPath: regionSelectionPath, Prefix: "els-sr-da", ResourceGroup: resourceGroup, @@ -205,11 +258,11 @@ func TestRunExistingInstance(t *testing.T) { }) options.TerraformVars = []testschematic.TestSchematicTerraformVar{ + {Name: "prefix", Value: options.Prefix, DataType: "string"}, {Name: "ibmcloud_api_key", Value: options.RequiredEnvironmentVars["TF_VAR_ibmcloud_api_key"], DataType: "string", Secure: true}, {Name: "existing_elasticsearch_instance_crn", Value: terraform.Output(t, existingTerraformOptions, "elasticsearch_crn"), DataType: "string"}, - {Name: "resource_group_name", Value: fmt.Sprintf("%s-resource-group", prefix), DataType: "string"}, + {Name: "existing_resource_group_name", Value: fmt.Sprintf("%s-resource-group", prefix), DataType: "string"}, {Name: "region", Value: region, DataType: "string"}, - {Name: "use_existing_resource_group", Value: true, DataType: "bool"}, {Name: "provider_visibility", Value: "public", DataType: "string"}, } err := options.RunSchematicTest() @@ -229,22 +282,23 @@ func TestRunExistingInstance(t *testing.T) { } // Test the DA when using IBM owned encryption keys -func TestRunStandardSolutionIBMKeys(t *testing.T) { +func TestFullyConfigurableSolutionIBMKeys(t *testing.T) { t.Parallel() options := testhelper.TestOptionsDefault(&testhelper.TestOptions{ Testing: t, - TerraformDir: standardSolutionTerraformDir, + TerraformDir: fullyConfigurableSolutionTerraformDir, Region: "us-south", - Prefix: "es-icd-key", + Prefix: "esicdkey", ResourceGroup: resourceGroup, }) options.TerraformVars = map[string]any{ "elasticsearch_version": "8.12", "provider_visibility": "public", - "resource_group_name": options.Prefix, - "use_ibm_owned_encryption_key": true, + "existing_resource_group_name": resourceGroup, + "kms_encryption_enabled": false, + "prefix": options.Prefix, } output, err := options.RunTestConsistency() @@ -255,8 +309,8 @@ func TestRunStandardSolutionIBMKeys(t *testing.T) { func TestPlanValidation(t *testing.T) { options := testhelper.TestOptionsDefault(&testhelper.TestOptions{ Testing: t, - TerraformDir: standardSolutionTerraformDir, - Prefix: "validate-plan", + TerraformDir: fullyConfigurableSolutionTerraformDir, + Prefix: "val-plan", ResourceGroup: resourceGroup, Region: "us-south", // skip VPC region picker }) @@ -264,37 +318,39 @@ func TestPlanValidation(t *testing.T) { options.TerraformOptions.NoColor = true options.TerraformOptions.Logger = logger.Discard options.TerraformOptions.Vars = map[string]interface{}{ - "prefix": options.Prefix, - "region": "us-south", - "elasticsearch_version": "8.10", - "provider_visibility": "public", - "resource_group_name": options.Prefix, + "prefix": options.Prefix, + "existing_resource_group_name": resourceGroup, + "region": "us-south", + "elasticsearch_version": "8.10", + "provider_visibility": "public", } // Test the DA when using Elser model - var standardSolutionWithElserModelVars = map[string]interface{}{ + var fullyConfigurableSolutionWithElserModelVars = map[string]interface{}{ + "kms_encryption_enabled": true, "existing_kms_instance_crn": permanentResources["hpcs_south_crn"], "enable_elser_model": true, "plan": "platinum", } // Test the DA when using Kibana dashboard and existing KMS instance - var standardSolutionWithKibanaDashboardVars = map[string]interface{}{ + var fullyConfigurableSolutionWithKibanaDashboardVars = map[string]interface{}{ "enable_kibana_dashboard": true, + "kms_encryption_enabled": true, "existing_kms_instance_crn": permanentResources["hpcs_south_crn"], "plan": "enterprise", } // Test the DA when using IBM owned encryption key - var standardSolutionWithUseIbmOwnedEncKey = map[string]interface{}{ - "use_ibm_owned_encryption_key": true, + var fullyConfigurableSolutionWithUseIbmOwnedEncKey = map[string]interface{}{ + "kms_encryption_enabled": false, } // Create a map of the variables tfVarsMap := map[string]map[string]interface{}{ - "standardSolutionWithElserModelVars": standardSolutionWithElserModelVars, - "standardSolutionWithKibanaDashboardVars": standardSolutionWithKibanaDashboardVars, - "standardSolutionWithUseIbmOwnedEncKey": standardSolutionWithUseIbmOwnedEncKey, + "fullyConfigurableSolutionWithElserModelVars": fullyConfigurableSolutionWithElserModelVars, + "fullyConfigurableSolutionWithKibanaDashboardVars": fullyConfigurableSolutionWithKibanaDashboardVars, + "fullyConfigurableSolutionWithUseIbmOwnedEncKey": fullyConfigurableSolutionWithUseIbmOwnedEncKey, } _, initErr := terraform.InitE(t, options.TerraformOptions) diff --git a/variables.tf b/variables.tf index 5762f1ec..4b72a8b8 100644 --- a/variables.tf +++ b/variables.tf @@ -60,14 +60,14 @@ variable "members" { # Validation is done in the Terraform plan phase by the IBM provider, so no need to add extra validation here. } -variable "member_cpu_count" { +variable "cpu_count" { type = number description = "The dedicated CPU per member that is allocated. For shared CPU, set to 0. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling)." default = 0 # Validation is done in the Terraform plan phase by the IBM provider, so no need to add extra validation here. } -variable "member_disk_mb" { +variable "disk_mb" { type = number description = "The disk that is allocated per member. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling)." default = 5120 @@ -81,7 +81,7 @@ variable "member_host_flavor" { # Validation is done in the Terraform plan phase by the IBM provider, so no need to add extra validation here. } -variable "member_memory_mb" { +variable "memory_mb" { type = number description = "The memory per member that is allocated. [Learn more](https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-resources-scaling)" default = 4096 diff --git a/version.tf b/version.tf index 5391ac4d..4437e809 100644 --- a/version.tf +++ b/version.tf @@ -1,10 +1,12 @@ terraform { required_version = ">= 1.9.0" - # Use "greater than or equal to" range in modules required_providers { ibm = { source = "ibm-cloud/ibm" version = ">= 1.79.1, <2.0.0" + source = "IBM-Cloud/ibm" + # Use "greater than or equal to" range in modules + version = ">= 1.79.2, <2.0.0" } null = { source = "hashicorp/null" @@ -12,7 +14,7 @@ terraform { } time = { source = "hashicorp/time" - version = ">= 0.9.1" + version = ">= 0.9.1, < 1.0.0" } } } From 7e6ffab8544c397f88bad85397fd2024d9b6f65a Mon Sep 17 00:00:00 2001 From: aatreyee257 Date: Wed, 2 Jul 2025 17:08:16 +0530 Subject: [PATCH 14/22] changes --- README.md | 1 + version.tf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7bf50d57..89603f28 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ You need the following permissions to run this module. | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.9.0 | +| [ibm](#requirement\_ibm) | >= 1.79.1, <2.0.0 | | [ibm](#requirement\_ibm) | >= 1.79.2, <2.0.0 | | [null](#requirement\_null) | >= 3.2.1, < 4.0.0 | | [time](#requirement\_time) | >= 0.9.1, < 1.0.0 | diff --git a/version.tf b/version.tf index 4437e809..3ac949cb 100644 --- a/version.tf +++ b/version.tf @@ -4,7 +4,7 @@ terraform { ibm = { source = "ibm-cloud/ibm" version = ">= 1.79.1, <2.0.0" - source = "IBM-Cloud/ibm" + source = "IBM-Cloud/ibm" # Use "greater than or equal to" range in modules version = ">= 1.79.2, <2.0.0" } From 25813343a9b8406f3030e0c181322251d665e190 Mon Sep 17 00:00:00 2001 From: shemau Date: Fri, 4 Jul 2025 12:37:10 +0100 Subject: [PATCH 15/22] fix: update secrets baseline --- .secrets.baseline | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 99241264..4dd3525d 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "go.sum|^.secrets.baseline$", "lines": null }, - "generated_at": "2025-07-03T10:08:09Z", + "generated_at": "2025-07-04T11:33:40Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -110,7 +110,7 @@ "hashed_secret": "8c7c51db5075ebd0369c51e9f14737d9b4c1c21d", "is_secret": false, "is_verified": false, - "line_number": 369, + "line_number": 368, "type": "Base64 High Entropy String", "verified_result": null } From 8dbcf05846e25e61f3e04fd7b9ca40d73e491e35 Mon Sep 17 00:00:00 2001 From: aatreyee257 Date: Mon, 14 Jul 2025 22:23:05 +0530 Subject: [PATCH 16/22] resolved comments --- solutions/fully-configurable/variables.tf | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/solutions/fully-configurable/variables.tf b/solutions/fully-configurable/variables.tf index 2a726d06..c98ce22e 100644 --- a/solutions/fully-configurable/variables.tf +++ b/solutions/fully-configurable/variables.tf @@ -474,6 +474,13 @@ variable "kibana_registry_server" { type = string description = "The server URL of the container registry used to pull the Kibana image." default = "https://index.docker.io/v1/" + validation { + condition = ( + !(var.use_private_registry && !var.use_existing_registry_secret) + || (var.kibana_registry_server != null && var.kibana_registry_server != "") + ) + error_message = "The `kibana_registry_server` must not be null or empty when `use_private_registry` is true and `use_existing_registry_secret` is false." + } } variable "kibana_image_digest" { @@ -484,8 +491,6 @@ variable "kibana_image_digest" { condition = var.kibana_image_digest == null || can(regex("^sha256:", var.kibana_image_digest)) error_message = "If provided, the value of kibana_image_digest must start with 'sha256:'." } - - } variable "kibana_image_port" { @@ -514,6 +519,13 @@ variable "kibana_registry_username" { description = "Username for the for the container registry." type = string default = null + validation { + condition = ( + !(var.use_private_registry && !var.use_existing_registry_secret) + || (var.kibana_registry_username != null && var.kibana_registry_username != "") + ) + error_message = "The `kibana_registry_username` must not be null or empty when `use_private_registry` is true and `use_existing_registry_secret` is false." + } } variable "kibana_registry_personal_access_token" { @@ -521,6 +533,13 @@ variable "kibana_registry_personal_access_token" { type = string default = null sensitive = true + validation { + condition = ( + !(var.use_private_registry && !var.use_existing_registry_secret) + || (var.kibana_registry_personal_access_token != null && var.kibana_registry_personal_access_token != "") + ) + error_message = "The `kibana_registry_personal_access_token` must not be null or empty when `use_private_registry` is true and `use_existing_registry_secret` is false." + } } ############################################################## From 455fe44b560acb02e83e5a6fd421d7c5595a7cb5 Mon Sep 17 00:00:00 2001 From: aatreyee257 Date: Tue, 15 Jul 2025 11:41:52 +0530 Subject: [PATCH 17/22] fixed test failure --- solutions/fully-configurable/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solutions/fully-configurable/variables.tf b/solutions/fully-configurable/variables.tf index c98ce22e..716de1ec 100644 --- a/solutions/fully-configurable/variables.tf +++ b/solutions/fully-configurable/variables.tf @@ -455,7 +455,7 @@ variable "existing_code_engine_project_id" { variable "enable_kibana_dashboard" { type = bool description = "Set to true to deploy Kibana in Code Engine. NOTE: By default, the Kibana image will be pulled from the official Elastic registry (docker.elastic.co) and is not certified by IBM, however this can be overridden using the `kibana_registry_namespace_image` and `kibana_image_digest` inputs." - default = true + default = false } variable "use_private_registry" { From ed61e9d9605957f0fb689506390329d4e7e694ea Mon Sep 17 00:00:00 2001 From: aatreyee257 Date: Wed, 23 Jul 2025 12:07:30 +0530 Subject: [PATCH 18/22] fixes --- .secrets.baseline | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 4c59d2d5..3b5f1d97 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "go.sum|^.secrets.baseline$", "lines": null }, - "generated_at": "2025-07-21T14:06:41Z", + "generated_at": "2025-07-23T06:27:24Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -110,7 +110,7 @@ "hashed_secret": "8c7c51db5075ebd0369c51e9f14737d9b4c1c21d", "is_secret": false, "is_verified": false, - "line_number": 375, + "line_number": 374, "type": "Base64 High Entropy String", "verified_result": null } From c04820a5ca73bc6e743b9d9378eb975e8effc78a Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Fri, 1 Aug 2025 00:51:02 +0530 Subject: [PATCH 19/22] fix error for secrets --- solutions/fully-configurable/main.tf | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/solutions/fully-configurable/main.tf b/solutions/fully-configurable/main.tf index 07dacca7..0faca682 100644 --- a/solutions/fully-configurable/main.tf +++ b/solutions/fully-configurable/main.tf @@ -422,23 +422,6 @@ data "http" "es_metadata" { ca_cert_pem = base64decode(local.elasticsearch_cert) } -module "secret" { - count = var.use_private_registry && !var.use_existing_registry_secret ? 1 : 0 - source = "terraform-ibm-modules/code-engine/ibm//modules/secret" - version = "4.2.4" - - name = var.kibana_image_secret - project_id = local.code_engine_project_id - format = "registry" - - data = { - username = var.kibana_registry_username - password = var.kibana_registry_personal_access_token - server = var.kibana_registry_server - } -} - - module "code_engine_kibana" { count = var.enable_kibana_dashboard ? 1 : 0 source = "terraform-ibm-modules/code-engine/ibm" @@ -452,6 +435,14 @@ module "code_engine_kibana" { data = { "ELASTICSEARCH_PASSWORD" = local.admin_pass } + }, + "registry-secret" = { + format = "registry" + data = { + username = var.kibana_registry_username + password = var.kibana_registry_personal_access_token + server = var.kibana_registry_server + } } } @@ -459,7 +450,7 @@ module "code_engine_kibana" { (local.code_engine_app_name) = { image_reference = var.kibana_image_digest != null ? "${var.kibana_registry_namespace_image}@${var.kibana_image_digest}" : "${var.kibana_registry_namespace_image}:${local.kibana_version}" image_port = var.kibana_image_port - image_secret = var.use_private_registry ? (var.use_existing_registry_secret ? var.kibana_image_secret : module.secret[0].name) : null + image_secret = var.use_private_registry ? (var.use_existing_registry_secret ? var.kibana_image_secret : "registry-secret") : null run_env_variables = [{ type = "literal" name = "ELASTICSEARCH_HOSTS" From fb98339d78b16059de95bd1828b37a74896370f8 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Fri, 1 Aug 2025 01:02:33 +0530 Subject: [PATCH 20/22] baseline --- .secrets.baseline | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index c62e04e1..45220eba 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,11 +3,7 @@ "files": "go.sum|^.secrets.baseline$", "lines": null }, -<<<<<<< HEAD - "generated_at": "2025-07-23T06:27:24Z", -======= "generated_at": "2025-07-24T21:11:38Z", ->>>>>>> origin/main "plugins_used": [ { "name": "AWSKeyDetector" @@ -114,11 +110,7 @@ "hashed_secret": "8c7c51db5075ebd0369c51e9f14737d9b4c1c21d", "is_secret": false, "is_verified": false, -<<<<<<< HEAD - "line_number": 374, -======= "line_number": 380, ->>>>>>> origin/main "type": "Base64 High Entropy String", "verified_result": null } From ff20c5de88ccaf70282d8147320b538435e88fcb Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Fri, 1 Aug 2025 01:03:21 +0530 Subject: [PATCH 21/22] baseline --- .secrets.baseline | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 45220eba..815f2904 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "go.sum|^.secrets.baseline$", "lines": null }, - "generated_at": "2025-07-24T21:11:38Z", + "generated_at": "2025-07-31T19:33:02Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -110,7 +110,7 @@ "hashed_secret": "8c7c51db5075ebd0369c51e9f14737d9b4c1c21d", "is_secret": false, "is_verified": false, - "line_number": 380, + "line_number": 379, "type": "Base64 High Entropy String", "verified_result": null } From 5a5f705c1b0d9c4022588342341b6b1bd2cbe954 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Fri, 1 Aug 2025 02:19:05 +0530 Subject: [PATCH 22/22] updated logic for image secret --- solutions/fully-configurable/main.tf | 30 ++++++++++++++++------------ 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/solutions/fully-configurable/main.tf b/solutions/fully-configurable/main.tf index 950d5e34..e47a0db8 100644 --- a/solutions/fully-configurable/main.tf +++ b/solutions/fully-configurable/main.tf @@ -432,22 +432,26 @@ module "code_engine_kibana" { resource_group_id = module.resource_group.resource_group_id project_name = local.code_engine_project_name existing_project_id = local.code_engine_project_id - secrets = { - "es-secret" = { - format = "generic" - data = { - "ELASTICSEARCH_PASSWORD" = local.admin_pass + secrets = merge( + { + "es-secret" = { + format = "generic" + data = { + "ELASTICSEARCH_PASSWORD" = local.admin_pass + } } }, - "registry-secret" = { - format = "registry" - data = { - username = var.kibana_registry_username - password = var.kibana_registry_personal_access_token - server = var.kibana_registry_server + var.use_private_registry && !var.use_existing_registry_secret ? { + "registry-secret" = { + format = "registry" + data = { + username = var.kibana_registry_username + password = var.kibana_registry_personal_access_token + server = var.kibana_registry_server + } } - } - } + } : {} + ) apps = { (local.code_engine_app_name) = {