feat: enable_log_archive variable has been replace by log_analysis_enable_archive and activity_tracker_enable_archive variables<br> - fixed some bugs related to using existing resources (#93)

Author: kierramarie

solutions/instances/main.tf

@@ -18,7 +18,7 @@ locals {
   at_cos_target_bucket_name   = var.prefix != null ? "${var.prefix}-${var.at_cos_target_bucket_name}" : var.at_cos_target_bucket_name
 
   cos_instance_crn            = var.existing_cos_instance_crn != null ? var.existing_cos_instance_crn : module.cos_instance[0].cos_instance_crn
-  existing_kms_guid           = var.existing_kms_instance_crn != null ? element(split(":", var.existing_kms_instance_crn), length(split(":", var.existing_kms_instance_crn)) - 3) : length(local.bucket_config_map) == 2 ? null : tobool("The CRN of the existing KMS is not provided.")
+  existing_kms_guid           = (var.existing_log_archive_cos_bucket_name != null && var.existing_at_cos_target_bucket_name != null) || (var.log_analysis_provision == false && var.enable_at_event_routing_to_cos_bucket == false) ? null : var.existing_kms_instance_crn != null ? element(split(":", var.existing_kms_instance_crn), length(split(":", var.existing_kms_instance_crn)) - 3) : tobool("The CRN of the existing KMS is not provided.")
   cos_instance_guid           = var.existing_cos_instance_crn == null ? module.cos_instance[0].cos_instance_guid : element(split(":", var.existing_cos_instance_crn), length(split(":", var.existing_cos_instance_crn)) - 3)
   archive_cos_bucket_name     = var.existing_log_archive_cos_bucket_name != null ? var.existing_log_archive_cos_bucket_name : module.cos_bucket[0].buckets[local.log_archive_cos_bucket_name].bucket_name
   archive_cos_bucket_endpoint = var.existing_log_archive_cos_bucket_endpoint != null ? var.existing_log_archive_cos_bucket_endpoint : module.cos_bucket[0].buckets[local.log_archive_cos_bucket_name].s3_endpoint_private
@@ -68,7 +68,7 @@ locals {
     )
   ) : null
 
-  kms_region = (length(local.bucket_config_map) != 0) ? (var.existing_cos_kms_key_crn == null ? element(split(":", var.existing_kms_instance_crn), length(split(":", var.existing_kms_instance_crn)) - 5) : null) : null
+  kms_region = (length(coalesce(local.bucket_config_map, [])) != 0) ? (var.existing_cos_kms_key_crn == null ? element(split(":", var.existing_kms_instance_crn), length(split(":", var.existing_kms_instance_crn)) - 5) : null) : null
   at_cos_route = var.enable_at_event_routing_to_cos_bucket ? [{
     route_name = "at-cos-route"
     locations  = ["*", "global"]
@@ -102,15 +102,16 @@ module "resource_group" {
 
 module "observability_instance" {
   source  = "terraform-ibm-modules/observability-instances/ibm"
-  version = "2.12.2"
+  version = "2.13.2"
   providers = {
     logdna.at = logdna.at
     logdna.ld = logdna.ld
   }
-  region            = var.region
-  resource_group_id = module.resource_group.resource_group_id
-  enable_archive    = var.enable_log_archive
-  ibmcloud_api_key  = local.archive_api_key
+  region                          = var.region
+  resource_group_id               = module.resource_group.resource_group_id
+  log_analysis_enable_archive     = var.log_analysis_enable_archive
+  activity_tracker_enable_archive = var.activity_tracker_enable_archive
+  ibmcloud_api_key                = local.archive_api_key
   # Log Analysis
   log_analysis_provision           = var.log_analysis_provision
   log_analysis_instance_name       = var.prefix != null ? "${var.prefix}-${var.log_analysis_instance_name}" : var.log_analysis_instance_name
@@ -164,7 +165,7 @@ module "kms" {
   providers = {
     ibm = ibm.kms
   }
-  count                       = (var.existing_cos_kms_key_crn != null || (length(local.bucket_config_map) == 0)) ? 0 : 1 # no need to create any KMS resources if passing an existing key, or bucket
+  count                       = (var.existing_cos_kms_key_crn != null || (length(coalesce(local.bucket_config_map, [])) == 0)) ? 0 : 1 # no need to create any KMS resources if passing an existing key, or bucket
   source                      = "terraform-ibm-modules/kms-all-inclusive/ibm"
   version                     = "4.13.2"
   create_key_protect_instance = false
@@ -205,7 +206,7 @@ resource "time_sleep" "wait_for_authorization_policy" {
 
 # Create IAM Authorization Policy to allow COS to access KMS for the encryption key
 resource "ibm_iam_authorization_policy" "policy" {
-  count                       = (var.skip_cos_kms_auth_policy || (length(local.bucket_config_map) == 0)) ? 0 : 1
+  count                       = (var.skip_cos_kms_auth_policy || (length(coalesce(local.bucket_config_map, [])) == 0)) ? 0 : 1
   source_service_name         = "cloud-object-storage"
   source_resource_instance_id = local.cos_instance_guid
   target_service_name         = local.kms_service
@@ -218,7 +219,7 @@ module "cos_instance" {
   providers = {
     ibm = ibm.cos
   }
-  count                    = (var.existing_cos_instance_crn == null) && length(local.bucket_config_map) != 0 ? 1 : 0 # no need to call COS module if consumer is using existing COS instance
+  count                    = (var.existing_cos_instance_crn == null) && length(coalesce(local.bucket_config_map, [])) != 0 ? 1 : 0 # no need to call COS module if consumer is using existing COS instance
   source                   = "terraform-ibm-modules/cos/ibm//modules/fscloud"
   version                  = "8.3.2"
   resource_group_id        = module.resource_group.resource_group_id
@@ -235,7 +236,7 @@ module "cos_bucket" {
   providers = {
     ibm = ibm.cos
   }
-  count   = (length(local.bucket_config_map) != 0) ? 1 : 0 # no need to call COS module if consumer is using existing COS bucket
+  count   = (length(coalesce(local.bucket_config_map, [])) != 0) ? 1 : 0 # no need to call COS module if consumer is using existing COS bucket
   source  = "terraform-ibm-modules/cos/ibm//modules/buckets"
   version = "8.3.2"
   bucket_configs = [

solutions/instances/variables.tf

@@ -79,9 +79,15 @@ variable "log_analysis_tags" {
   default     = []
 }
 
-variable "enable_log_archive" {
+variable "log_analysis_enable_archive" {
   type        = bool
-  description = "Enable the archive file for the IBM Log Analysis instance."
+  description = "Enable archive on log analysis instances"
+  default     = true
+}
+
+variable "activity_tracker_enable_archive" {
+  type        = bool
+  description = "Enable archive on activity tracker instances"
   default     = true
 }
 

tests/pr_test.go

@@ -207,3 +207,111 @@ func TestAgentsSolutionInSchematics(t *testing.T) {
 		logger.Log(t, "END: Destroy (existing resources)")
 	}
 }
+
+func TestRunExistingResourcesInstances(t *testing.T) {
+	t.Parallel()
+
+	// ------------------------------------------------------------------------------------
+	// Provision COS first
+	// ------------------------------------------------------------------------------------
+
+	prefix := fmt.Sprintf("obs-exist-%s", strings.ToLower(random.UniqueId()))
+	realTerraformDir := "./resources/existing-resources"
+	tempTerraformDir, _ := files.CopyTerraformFolderToTemp(realTerraformDir, fmt.Sprintf(prefix+"-%s", strings.ToLower(random.UniqueId())))
+	tags := common.GetTagsFromTravis()
+	region := "us-south"
+
+	// Verify ibmcloud_api_key variable is set
+	checkVariable := "TF_VAR_ibmcloud_api_key"
+	val, present := os.LookupEnv(checkVariable)
+	require.True(t, present, checkVariable+" environment variable not set")
+	require.NotEqual(t, "", val, checkVariable+" environment variable is empty")
+
+	logger.Log(t, "Tempdir: ", tempTerraformDir)
+	existingTerraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{
+		TerraformDir: tempTerraformDir,
+		Vars: map[string]interface{}{
+			"prefix":        prefix,
+			"region":        region,
+			"resource_tags": tags,
+		},
+		// Set Upgrade to true to ensure latest version of providers and modules are used by terratest.
+		// This is the same as setting the -upgrade=true flag with terraform.
+		Upgrade: true,
+	})
+
+	terraform.WorkspaceSelectOrNew(t, existingTerraformOptions, prefix)
+	_, existErr := terraform.InitAndApplyE(t, existingTerraformOptions)
+	if existErr != nil {
+		assert.True(t, existErr == nil, "Init and Apply of temp existing resource failed")
+	} else {
+
+		// ------------------------------------------------------------------------------------
+		// Deploy Observability instances DA passing in existing COS instance, and bucket details
+		// ------------------------------------------------------------------------------------
+
+		options := testhelper.TestOptionsDefault(&testhelper.TestOptions{
+			Testing:      t,
+			TerraformDir: solutionInstanceDADir,
+			// Do not hard fail the test if the implicit destroy steps fail to allow a full destroy of resource to occur
+			ImplicitRequired: false,
+			Region:           region,
+			TerraformVars: map[string]interface{}{
+				"cos_region":                               region,
+				"resource_group_name":                      terraform.Output(t, existingTerraformOptions, "resource_group_name"),
+				"use_existing_resource_group":              true,
+				"existing_log_archive_cos_bucket_name":     terraform.Output(t, existingTerraformOptions, "bucket_name"),
+				"existing_at_cos_target_bucket_name":       terraform.Output(t, existingTerraformOptions, "bucket_name_at"),
+				"existing_log_archive_cos_bucket_endpoint": terraform.Output(t, existingTerraformOptions, "bucket_endpoint"),
+				"existing_at_cos_target_bucket_endpoint":   terraform.Output(t, existingTerraformOptions, "bucket_endpoint_at"),
+				"existing_cos_instance_crn":                terraform.Output(t, existingTerraformOptions, "cos_crn"),
+				"management_endpoint_type_for_bucket":      "public",
+				"log_analysis_service_endpoints":           "public",
+				"enable_platform_metrics":                  "false",
+			},
+		})
+
+		output, err := options.RunTestConsistency()
+		assert.Nil(t, err, "This should not have errored")
+		assert.NotNil(t, output, "Expected some output")
+
+		// ------------------------------------------------------------------------------------
+		// Deploy Observability instance DA passing in existing COS instance (not bucket), and KMS key
+		// ------------------------------------------------------------------------------------
+
+		options2 := testhelper.TestOptionsDefault(&testhelper.TestOptions{
+			Testing:      t,
+			TerraformDir: solutionInstanceDADir,
+			// Do not hard fail the test if the implicit destroy steps fail to allow a full destroy of resource to occur
+			ImplicitRequired: false,
+			TerraformVars: map[string]interface{}{
+				"cos_region":                          region,
+				"resource_group_name":                 terraform.Output(t, existingTerraformOptions, "resource_group_name"),
+				"use_existing_resource_group":         true,
+				"existing_kms_instance_crn":           permanentResources["hpcs_south_crn"],
+				"kms_endpoint_type":                   "public",
+				"existing_cos_instance_crn":           terraform.Output(t, existingTerraformOptions, "cos_crn"),
+				"management_endpoint_type_for_bucket": "public",
+				"log_analysis_service_endpoints":      "public",
+				"enable_platform_metrics":             "false",
+			},
+		})
+
+		output2, err := options2.RunTestConsistency()
+		assert.Nil(t, err, "This should not have errored")
+		assert.NotNil(t, output2, "Expected some output")
+
+	}
+
+	// Check if "DO_NOT_DESTROY_ON_FAILURE" is set
+	envVal, _ := os.LookupEnv("DO_NOT_DESTROY_ON_FAILURE")
+	// Destroy the temporary existing resources if required
+	if t.Failed() && strings.ToLower(envVal) == "true" {
+		fmt.Println("Terratest failed. Debug the test and delete resources manually.")
+	} else {
+		logger.Log(t, "START: Destroy (existing resources)")
+		terraform.Destroy(t, existingTerraformOptions)
+		terraform.WorkspaceDelete(t, existingTerraformOptions, prefix)
+		logger.Log(t, "END: Destroy (existing resources)")
+	}
+}

tests/resources/existing-resources/README.md

@@ -0,0 +1 @@
+The terraform code in this directory is used by the existing resource test in tests/pr_test.go

tests/resources/existing-resources/main.tf

@@ -0,0 +1,37 @@
+##############################################################################
+# Resource Group
+##############################################################################
+
+module "resource_group" {
+  source  = "terraform-ibm-modules/resource-group/ibm"
+  version = "1.1.5"
+  # if an existing resource group is not set (null) create a new one using prefix
+  resource_group_name          = var.resource_group == null ? "${var.prefix}-resource-group" : null
+  existing_resource_group_name = var.resource_group
+}
+
+##############################################################################
+# Create Cloud Object Storage instance and buckets
+##############################################################################
+
+module "cos" {
+  source                 = "terraform-ibm-modules/cos/ibm"
+  version                = "8.2.13"
+  resource_group_id      = module.resource_group.resource_group_id
+  region                 = var.region
+  cos_instance_name      = "${var.prefix}-cos"
+  cos_tags               = var.resource_tags
+  bucket_name            = "${var.prefix}-bucket"
+  retention_enabled      = false # disable retention for test environments - enable for stage/prod
+  kms_encryption_enabled = false
+}
+
+module "additional_cos_bucket" {
+  source                   = "terraform-ibm-modules/cos/ibm"
+  version                  = "8.2.13"
+  region                   = var.region
+  create_cos_instance      = false
+  existing_cos_instance_id = module.cos.cos_instance_id
+  bucket_name              = "${var.prefix}-bucket-at"
+  kms_encryption_enabled   = false
+}

tests/resources/existing-resources/outputs.tf

@@ -0,0 +1,38 @@
+##############################################################################
+# Outputs
+##############################################################################
+
+output "resource_group_name" {
+  description = "Resource group name"
+  value       = module.resource_group.resource_group_name
+}
+
+output "prefix" {
+  description = "Prefix"
+  value       = var.prefix
+}
+
+output "cos_crn" {
+  description = "COS CRN"
+  value       = module.cos.cos_instance_crn
+}
+
+output "bucket_name" {
+  description = "Log Archive bucket name"
+  value       = module.cos.bucket_name
+}
+
+output "bucket_name_at" {
+  description = "Activity Tracker bucket name"
+  value       = module.additional_cos_bucket.bucket_name
+}
+
+output "bucket_endpoint" {
+  description = "Log Archive bucket endpoint"
+  value       = module.cos.s3_endpoint_public
+}
+
+output "bucket_endpoint_at" {
+  description = "Activity Tracker bucket endpoint"
+  value       = module.additional_cos_bucket.s3_endpoint_public
+}

tests/resources/existing-resources/provider.tf

@@ -0,0 +1,4 @@
+provider "ibm" {
+  ibmcloud_api_key = var.ibmcloud_api_key
+  region           = var.region
+}

tests/resources/existing-resources/variables.tf

@@ -0,0 +1,31 @@
+##############################################################################
+# Input variables
+##############################################################################
+
+variable "ibmcloud_api_key" {
+  type        = string
+  description = "The IBM Cloud API Key"
+  sensitive   = true
+}
+
+variable "region" {
+  type        = string
+  description = "Region"
+}
+
+variable "prefix" {
+  type        = string
+  description = "The prefix to add to all resources."
+}
+
+variable "resource_group" {
+  type        = string
+  description = "The name of an existing resource group to provision resources in. If not specified, a new resource group is created with the `prefix` variable."
+  default     = null
+}
+
+variable "resource_tags" {
+  type        = list(string)
+  description = "The tags to add to the created resources."
+  default     = []
+}

tests/resources/existing-resources/version.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = ">= 1.3.0"
+  required_providers {
+    ibm = {
+      source  = "ibm-cloud/ibm"
+      version = ">= 1.51.0"
+    }
+  }
+}