feat: add support to use existing Secrets Manager instance in the DA solution (#109)

Author: akocbek

solutions/standard/README.md

@@ -6,4 +6,6 @@ This solution supports the following:
 - Optionally configure an IBM Secrets Manager IAM credentials engine to an IBM Secrets Manager instance.
 - Configuring KMS encryption using a newly created key, or passing an existing key.
 
+![secret-manager-deployable-architecture](../../reference-architecture/secrets_manager.svg)
+
 **NB:** This solution is not intended to be called by one or more other modules since it contains a provider configurations, meaning it is not compatible with the `for_each`, `count`, and `depends_on` arguments. For more information see [Providers Within Modules](https://developer.hashicorp.com/terraform/language/modules/develop/providers)

solutions/standard/main.tf

@@ -57,7 +57,15 @@ module "kms" {
 # Secrets Manager
 ########################################################################################################################
 
+locals {
+  parsed_existing_secrets_manager_crn = var.existing_secrets_manager_crn != null ? split(":", var.existing_secrets_manager_crn) : []
+  secrets_manager_guid                = var.existing_secrets_manager_crn != null ? (length(local.parsed_existing_secrets_manager_crn) > 0 ? local.parsed_existing_secrets_manager_crn[7] : null) : module.secrets_manager[0].secrets_manager_guid
+  secrets_manager_crn                 = var.existing_secrets_manager_crn != null ? var.existing_secrets_manager_crn : module.secrets_manager[0].secrets_manager_crn
+  secrets_manager_region              = var.existing_secrets_manager_crn != null ? (length(local.parsed_existing_secrets_manager_crn) > 0 ? local.parsed_existing_secrets_manager_crn[5] : null) : module.secrets_manager[0].secrets_manager_region
+}
+
 module "secrets_manager" {
+  count                = var.existing_secrets_manager_crn != null ? 0 : 1
   source               = "../.."
   resource_group_id    = module.resource_group.resource_group_id
   region               = var.region
@@ -84,7 +92,7 @@ module "iam_secrets_engine" {
   version              = "1.1.0"
   region               = var.region
   iam_engine_name      = var.prefix != null ? "${var.prefix}-${var.iam_engine_name}" : var.iam_engine_name
-  secrets_manager_guid = module.secrets_manager.secrets_manager_guid
+  secrets_manager_guid = local.secrets_manager_guid
   endpoint_type        = var.allowed_network == "private-only" ? "private" : "public"
 }
 
@@ -104,8 +112,8 @@ module "secrets_manager_public_cert_engine" {
     ibm              = ibm
     ibm.secret-store = ibm
   }
-  secrets_manager_guid         = module.secrets_manager.secrets_manager_guid
-  region                       = module.secrets_manager.secrets_manager_region
+  secrets_manager_guid         = local.secrets_manager_guid
+  region                       = local.secrets_manager_region
   internet_services_crn        = var.cis_id
   ibmcloud_cis_api_key         = var.ibmcloud_api_key
   dns_config_name              = var.dns_provider_name
@@ -120,7 +128,7 @@ module "private_secret_engine" {
   count                     = var.private_engine_enabled ? 1 : 0
   source                    = "terraform-ibm-modules/secrets-manager-private-cert-engine/ibm"
   version                   = "1.3.0"
-  secrets_manager_guid      = module.secrets_manager.secrets_manager_guid
+  secrets_manager_guid      = local.secrets_manager_guid
   region                    = var.region
   root_ca_name              = var.root_ca_name
   root_ca_common_name       = var.root_ca_common_name

solutions/standard/outputs.tf

@@ -10,25 +10,25 @@ output "resource_group_id" {
 
 output "secrets_manager_guid" {
   description = "GUID of Secrets Manager instance"
-  value       = module.secrets_manager.secrets_manager_guid
+  value       = local.secrets_manager_guid
 }
 
 output "secrets_manager_id" {
   description = "ID of Secrets Manager instance"
-  value       = module.secrets_manager.secrets_manager_id
+  value       = var.existing_secrets_manager_crn == null ? module.secrets_manager[0].secrets_manager_id : null
 }
 
 output "secrets_manager_name" {
-  value       = module.secrets_manager.secrets_manager_name
+  value       = var.existing_secrets_manager_crn == null ? module.secrets_manager[0].secrets_manager_name : null
   description = "Name of the Secrets Manager instance"
 }
 
 output "secrets_manager_crn" {
-  value       = module.secrets_manager.secrets_manager_crn
+  value       = local.secrets_manager_crn
   description = "CRN of the Secrets Manager instance"
 }
 
 output "secrets_manager_region" {
-  value       = module.secrets_manager.secrets_manager_region
+  value       = local.secrets_manager_region
   description = "Region of the Secrets Manager instance"
 }

solutions/standard/variables.tf

@@ -41,6 +41,12 @@ variable "secrets_manager_instance_name" {
   default     = "base-security-services-sm"
 }
 
+variable "existing_secrets_manager_crn" {
+  type        = string
+  description = "The CRN of an existing Secrets Manager instance. If not supplied, a new Secrets Manager instance will be created."
+  default     = null
+}
+
 variable "service_plan" {
   type        = string
   description = "The service/pricing plan to use when provisioning a new Secrets Manager instance. Allowed values: 'standard' and 'trial'. Only used if `provision_sm_instance` is set to true."

tests/existing-resources/main.tf

@@ -46,3 +46,22 @@ module "key_protect" {
     }
   ]
 }
+
+##############################################################################
+# Secrets Manager
+##############################################################################
+
+module "secrets_manager" {
+  source                     = "terraform-ibm-modules/secrets-manager/ibm"
+  version                    = "1.11.0"
+  resource_group_id          = module.resource_group.resource_group_id
+  region                     = var.region
+  secrets_manager_name       = "${var.prefix}-secrets-manager" #tfsec:ignore:general-secrets-no-plaintext-exposure
+  sm_service_plan            = "trial"
+  sm_tags                    = var.resource_tags
+  kms_encryption_enabled     = true
+  existing_kms_instance_guid = module.key_protect.kms_guid
+  kms_key_crn                = module.key_protect.keys["${var.prefix}-sm.${var.prefix}-sm-key"].crn
+  enable_event_notification  = true
+  existing_en_instance_crn   = module.event_notifications.crn
+}

tests/existing-resources/outputs.tf

@@ -22,3 +22,8 @@ output "event_notification_instance_crn" {
   value       = module.event_notifications.crn
   description = "CRN of created event notification"
 }
+
+output "secrets_manager_instance_crn" {
+  value       = module.secrets_manager.secrets_manager_crn
+  description = "CRN of created secret manager instance"
+}

tests/other_test.go

@@ -1,10 +1,12 @@
 package test
 
 import (
+	"math/rand"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testhelper"
+	"github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testschematic"
 )
 
 func TestRunBasicExample(t *testing.T) {
@@ -20,3 +22,44 @@ func TestRunBasicExample(t *testing.T) {
 	assert.Nil(t, err, "This should not have errored")
 	assert.NotNil(t, output, "Expected some output")
 }
+
+func TestRunCompleteExample(t *testing.T) {
+	t.Parallel()
+
+	options := setupOptions(t, "secrets-mgr")
+
+	output, err := options.RunTestConsistency()
+	assert.Nil(t, err, "This should not have errored")
+	assert.NotNil(t, output, "Expected some output")
+}
+
+func TestFSCloudInSchematics(t *testing.T) {
+	t.Parallel()
+
+	options := testschematic.TestSchematicOptionsDefault(&testschematic.TestSchematicOptions{
+		Testing: t,
+		Prefix:  "sm-fscloud",
+		TarIncludePatterns: []string{
+			"*.tf",
+			fscloudExampleTerraformDir + "/*.tf",
+			"modules/fscloud/*.tf",
+		},
+		// ResourceGroup:          resourceGroup,
+		TemplateFolder:         fscloudExampleTerraformDir,
+		Tags:                   []string{"test-schematic"},
+		DeleteWorkspaceOnFail:  false,
+		WaitJobCompleteMinutes: 60,
+	})
+
+	options.TerraformVars = []testschematic.TestSchematicTerraformVar{
+		{Name: "ibmcloud_api_key", Value: options.RequiredEnvironmentVars["TF_VAR_ibmcloud_api_key"], DataType: "string", Secure: true},
+		{Name: "region", Value: validRegions[rand.Intn(len(validRegions))], DataType: "string"},
+		{Name: "prefix", Value: options.Prefix, DataType: "string"},
+		{Name: "existing_kms_instance_guid", Value: permanentResources["hpcs_south"], DataType: "string"},
+		{Name: "kms_key_crn", Value: permanentResources["hpcs_south_root_key_crn"], DataType: "string"},
+		{Name: "sm_service_plan", Value: "trial", DataType: "string"},
+	}
+
+	err := options.RunSchematicTest()
+	assert.Nil(t, err, "This should not have errored")
+}

tests/pr_test.go

@@ -4,6 +4,7 @@ package test
 import (
 	"fmt"
 	"log"
+	"math/rand"
 	"os"
 	"strings"
 	"testing"
@@ -32,6 +33,14 @@ const yamlLocation = "../common-dev-assets/common-go-assets/common-permanent-res
 
 var permanentResources map[string]interface{}
 
+// Current supported Event Notification regions
+var validRegions = []string{
+	"us-south",
+	"eu-de",
+	"eu-gb",
+	"au-syd",
+}
+
 // TestMain will be run before any parallel tests, used to read data from yaml for use with tests
 func TestMain(m *testing.M) {
 
@@ -49,27 +58,17 @@ func setupOptions(t *testing.T, prefix string) *testhelper.TestOptions {
 		Testing:      t,
 		TerraformDir: completeExampleTerraformDir,
 		Prefix:       prefix,
+		Region:       validRegions[rand.Intn(len(validRegions))],
 		/*
 		 Comment out the 'ResourceGroup' input to force this tests to create a unique resource group. This is because
 		 there is a restriction with the Event Notification service, which allows only one Lite plan instance per resource group.
 		*/
 		// ResourceGroup:      resourceGroup,
-		BestRegionYAMLPath: "../common-dev-assets/common-go-assets/cloudinfo-region-secmgr-prefs.yaml",
 	})
 
 	return options
 }
 
-func TestRunCompleteExample(t *testing.T) {
-	t.Parallel()
-
-	options := setupOptions(t, "secrets-mgr")
-
-	output, err := options.RunTestConsistency()
-	assert.Nil(t, err, "This should not have errored")
-	assert.NotNil(t, output, "Expected some output")
-}
-
 func TestRunUpgradeExample(t *testing.T) {
 	t.Parallel()
 
@@ -82,38 +81,6 @@ func TestRunUpgradeExample(t *testing.T) {
 	}
 }
 
-func TestFSCloudInSchematics(t *testing.T) {
-	t.Parallel()
-
-	options := testschematic.TestSchematicOptionsDefault(&testschematic.TestSchematicOptions{
-		Testing: t,
-		Prefix:  "sm-fscloud",
-		TarIncludePatterns: []string{
-			"*.tf",
-			fscloudExampleTerraformDir + "/*.tf",
-			"modules/fscloud/*.tf",
-		},
-		BestRegionYAMLPath: "../common-dev-assets/common-go-assets/cloudinfo-region-secmgr-prefs.yaml",
-		// ResourceGroup:          resourceGroup,
-		TemplateFolder:         fscloudExampleTerraformDir,
-		Tags:                   []string{"test-schematic"},
-		DeleteWorkspaceOnFail:  false,
-		WaitJobCompleteMinutes: 60,
-	})
-
-	options.TerraformVars = []testschematic.TestSchematicTerraformVar{
-		{Name: "ibmcloud_api_key", Value: options.RequiredEnvironmentVars["TF_VAR_ibmcloud_api_key"], DataType: "string", Secure: true},
-		{Name: "region", Value: options.Region, DataType: "string"},
-		{Name: "prefix", Value: options.Prefix, DataType: "string"},
-		{Name: "existing_kms_instance_guid", Value: permanentResources["hpcs_south"], DataType: "string"},
-		{Name: "kms_key_crn", Value: permanentResources["hpcs_south_root_key_crn"], DataType: "string"},
-		{Name: "sm_service_plan", Value: "trial", DataType: "string"},
-	}
-
-	err := options.RunSchematicTest()
-	assert.Nil(t, err, "This should not have errored")
-}
-
 func TestRunDASolutionSchematics(t *testing.T) {
 	t.Parallel()
 
@@ -133,13 +100,12 @@ func TestRunDASolutionSchematics(t *testing.T) {
 		Tags:                   []string{"test-schematic"},
 		DeleteWorkspaceOnFail:  false,
 		WaitJobCompleteMinutes: 60,
-		BestRegionYAMLPath:     "../common-dev-assets/common-go-assets/cloudinfo-region-secmgr-prefs.yaml",
 	})
 
 	options.TerraformVars = []testschematic.TestSchematicTerraformVar{
 		{Name: "ibmcloud_api_key", Value: options.RequiredEnvironmentVars["TF_VAR_ibmcloud_api_key"], DataType: "string", Secure: true},
 		{Name: "prefix", Value: options.Prefix, DataType: "string"},
-		{Name: "region", Value: options.Region, DataType: "string"},
+		{Name: "region", Value: validRegions[rand.Intn(len(validRegions))], DataType: "string"},
 		{Name: "resource_group_name", Value: options.Prefix, DataType: "string"},
 		{Name: "service_plan", Value: "trial", DataType: "string"},
 		{Name: "allowed_network", Value: "private-only", DataType: "string"},
@@ -183,15 +149,6 @@ func GetSecretsManagerKey(sm_id string, sm_region string, sm_key_id string) *str
 func TestRunExistingResourcesInstances(t *testing.T) {
 	t.Parallel()
 
-	// Init test options for DA to get the region, which is used for provisioning the existing resources
-	options := testhelper.TestOptionsDefault(&testhelper.TestOptions{
-		Testing:      t,
-		TerraformDir: solutionsTerraformDir,
-		// Do not hard fail the test if the implicit destroy steps fail to allow a full destroy of resource to occur
-		ImplicitRequired:   false,
-		BestRegionYAMLPath: "../common-dev-assets/common-go-assets/cloudinfo-region-secmgr-prefs.yaml",
-	})
-
 	// ------------------------------------------------------------------------------------
 	// Provision Event Notification, KMS key and resource group first
 	// ------------------------------------------------------------------------------------
@@ -200,7 +157,6 @@ func TestRunExistingResourcesInstances(t *testing.T) {
 	realTerraformDir := "./existing-resources"
 	tempTerraformDir, _ := files.CopyTerraformFolderToTemp(realTerraformDir, fmt.Sprintf(prefix+"-%s", strings.ToLower(random.UniqueId())))
 	tags := common.GetTagsFromTravis()
-	region := "us-south"
 
 	// Verify ibmcloud_api_key variable is set
 	checkVariable := "TF_VAR_ibmcloud_api_key"
@@ -212,7 +168,7 @@ func TestRunExistingResourcesInstances(t *testing.T) {
 		TerraformDir: tempTerraformDir,
 		Vars: map[string]interface{}{
 			"prefix":        prefix,
-			"region":        options.Region,
+			"region":        validRegions[rand.Intn(len(validRegions))],
 			"resource_tags": tags,
 		},
 		// Set Upgrade to true to ensure latest version of providers and modules are used by terratest.
@@ -225,17 +181,25 @@ func TestRunExistingResourcesInstances(t *testing.T) {
 	if existErr != nil {
 		assert.True(t, existErr == nil, "Init and Apply of temp existing resource failed")
 	} else {
-		// add existing resources to previously created options
-		options.TerraformVars = map[string]interface{}{
-			"ibmcloud_api_key":                         os.Getenv("TF_VAR_ibmcloud_api_key"),
-			"region":                                   region,
-			"resource_group_name":                      terraform.Output(t, existingTerraformOptions, "resource_group_name"),
-			"use_existing_resource_group":              true,
-			"existing_event_notification_instance_crn": terraform.Output(t, existingTerraformOptions, "event_notification_instance_crn"),
-			"existing_secrets_manager_kms_key_crn":     terraform.Output(t, existingTerraformOptions, "secrets_manager_kms_key_crn"),
-			"existing_kms_instance_crn":                terraform.Output(t, existingTerraformOptions, "secrets_manager_kms_instance_crn"),
-			"service_plan":                             "trial",
-		}
+		options := testhelper.TestOptionsDefault(&testhelper.TestOptions{
+			Testing:      t,
+			TerraformDir: solutionsTerraformDir,
+			// Do not hard fail the test if the implicit destroy steps fail to allow a full destroy of resource to occur
+			ImplicitRequired: false,
+			TerraformVars: map[string]interface{}{
+				"ibmcloud_api_key":                         os.Getenv("TF_VAR_ibmcloud_api_key"),
+				"region":                                   validRegions[rand.Intn(len(validRegions))],
+				"resource_group_name":                      terraform.Output(t, existingTerraformOptions, "resource_group_name"),
+				"use_existing_resource_group":              true,
+				"existing_event_notification_instance_crn": terraform.Output(t, existingTerraformOptions, "event_notification_instance_crn"),
+				"existing_secrets_manager_kms_key_crn":     terraform.Output(t, existingTerraformOptions, "secrets_manager_kms_key_crn"),
+				"existing_kms_instance_crn":                terraform.Output(t, existingTerraformOptions, "secrets_manager_kms_instance_crn"),
+				"service_plan":                             "trial",
+				"existing_secrets_manager_crn":             terraform.Output(t, existingTerraformOptions, "secrets_manager_instance_crn"),
+				"iam_engine_enabled":                       true,
+				"private_engine_enabled":                   true,
+			},
+		})
 
 		output, err := options.RunTestConsistency()
 		assert.Nil(t, err, "This should not have errored")