feat: add support for metrics bucket configuration for Cloud Logs in the instances variation (#180)

maheshwarishikha · web-flow · commit 2a5a804ceb42 · 2024-10-09T15:08:57.000+01:00
diff --git a/ibm_catalog.json b/ibm_catalog.json
@@ -418,6 +418,37 @@
             {
               "key": "cloud_log_data_bucket_access_tag"
             },
+            {
+              "key": "cloud_log_metrics_bucket_name"
+            },
+            {
+              "key": "cloud_log_metrics_bucket_class",
+              "options": [
+                {
+                  "displayname": "standard",
+                  "value": "standard"
+                },
+                {
+                  "displayname": "vault",
+                  "value": "vault"
+                },
+                {
+                  "displayname": "cold",
+                  "value": "cold"
+                },
+                {
+                  "displayname": "smart",
+                  "value": "smart"
+                },
+                {
+                  "displayname": "onerate_active",
+                  "value": "onerate_active"
+                }
+              ]
+            },
+            {
+              "key": "cloud_log_metrics_bucket_access_tag"
+            },
             {
               "key": "management_endpoint_type_for_bucket",
               "options": [
@@ -456,6 +487,12 @@
             {
               "key": "existing_cloud_logs_data_bucket_endpoint"
             },
+            {
+              "key": "existing_cloud_logs_metrics_bucket_crn"
+            },
+            {
+              "key": "existing_cloud_logs_metrics_bucket_endpoint"
+            },
             {
               "key": "skip_cos_kms_auth_policy"
             },
diff --git a/reference-architecture/deployable-architecture-observability-instances.svg b/reference-architecture/deployable-architecture-observability-instances.svg
diff --git a/solutions/instances/README.md b/solutions/instances/README.md
@@ -11,6 +11,7 @@ This deployable architecture creates observability instances in IBM Cloud and su
 * A KMS-encrypted Object Storage bucket to store archived logs, if one is not passed in.
 * A KMS-encrypted Object Storage bucket for Activity Tracker event routing, if one is not passed in. (Disabled by default as service is deprecated)
 * A KMS-encrypted Object Storage bucket for Cloud Logs data, if one is not passed in.
+* A KMS-encrypted Object Storage bucket for Cloud Logs metrics, if one is not passed in.
 * An Activity Tracker event route to an Object Storage bucket and Cloud Logs target.
 * An option to integrate Cloud Logs with existing event notification instance.
 
diff --git a/solutions/instances/main.tf b/solutions/instances/main.tf
@@ -18,11 +18,11 @@ locals {
   at_cos_target_bucket_name   = var.prefix != null ? "${var.prefix}-${var.at_cos_target_bucket_name}" : var.at_cos_target_bucket_name
 
   cos_instance_crn            = var.existing_cos_instance_crn != null ? var.existing_cos_instance_crn : length(module.cos_instance) != 0 ? module.cos_instance[0].cos_instance_crn : null
-  existing_kms_guid           = ((var.existing_cloud_logs_data_bucket_crn != null && var.existing_log_archive_cos_bucket_name != null && var.existing_at_cos_target_bucket_name != null) || (!var.log_analysis_provision && !var.enable_at_event_routing_to_cos_bucket && !var.cloud_logs_provision)) ? null : var.existing_kms_instance_crn != null ? element(split(":", var.existing_kms_instance_crn), length(split(":", var.existing_kms_instance_crn)) - 3) : tobool("The CRN of the existing KMS is not provided.")
+  existing_kms_guid           = ((var.existing_cloud_logs_metrics_bucket_crn != null && var.existing_cloud_logs_data_bucket_crn != null && var.existing_log_archive_cos_bucket_name != null && var.existing_at_cos_target_bucket_name != null) || (!var.log_analysis_provision && !var.enable_at_event_routing_to_cos_bucket && !var.cloud_logs_provision)) ? null : var.existing_kms_instance_crn != null ? element(split(":", var.existing_kms_instance_crn), length(split(":", var.existing_kms_instance_crn)) - 3) : tobool("The CRN of the existing KMS is not provided.")
   cos_instance_guid           = var.existing_cos_instance_crn == null ? length(module.cos_instance) != 0 ? module.cos_instance[0].cos_instance_guid : null : element(split(":", var.existing_cos_instance_crn), length(split(":", var.existing_cos_instance_crn)) - 3)
   archive_cos_bucket_name     = var.existing_log_archive_cos_bucket_name != null ? var.existing_log_archive_cos_bucket_name : var.log_analysis_provision ? module.cos_bucket[0].buckets[local.log_archive_cos_bucket_name].bucket_name : null
   archive_cos_bucket_endpoint = var.existing_log_archive_cos_bucket_endpoint != null ? var.existing_log_archive_cos_bucket_endpoint : var.log_analysis_provision ? module.cos_bucket[0].buckets[local.log_archive_cos_bucket_name].s3_endpoint_private : null
-  cos_kms_key_crn             = ((var.existing_cloud_logs_data_bucket_crn != null && var.existing_log_archive_cos_bucket_name != null && var.existing_at_cos_target_bucket_name != null) || (!var.log_analysis_provision && !var.enable_at_event_routing_to_cos_bucket && !var.cloud_logs_provision)) ? null : var.existing_cos_kms_key_crn != null ? var.existing_cos_kms_key_crn : module.kms[0].keys[format("%s.%s", local.cos_key_ring_name, local.cos_key_name)].crn
+  cos_kms_key_crn             = ((var.existing_cloud_logs_metrics_bucket_crn != null && var.existing_cloud_logs_data_bucket_crn != null && var.existing_log_archive_cos_bucket_name != null && var.existing_at_cos_target_bucket_name != null) || (!var.log_analysis_provision && !var.enable_at_event_routing_to_cos_bucket && !var.cloud_logs_provision)) ? null : var.existing_cos_kms_key_crn != null ? var.existing_cos_kms_key_crn : module.kms[0].keys[format("%s.%s", local.cos_key_ring_name, local.cos_key_name)].crn
 
   cos_target_bucket_name     = var.existing_at_cos_target_bucket_name != null ? var.existing_at_cos_target_bucket_name : var.enable_at_event_routing_to_cos_bucket ? module.cos_bucket[0].buckets[local.at_cos_target_bucket_name].bucket_name : null
   cos_target_bucket_endpoint = var.existing_at_cos_target_bucket_endpoint != null ? var.existing_at_cos_target_bucket_endpoint : var.enable_at_event_routing_to_cos_bucket ? module.cos_bucket[0].buckets[local.at_cos_target_bucket_name].s3_endpoint_private : null
@@ -51,10 +51,17 @@ locals {
     tag   = var.cloud_log_data_bucket_access_tag
   } : null
 
+  cloud_log_metrics_bucket_config = var.existing_cloud_logs_metrics_bucket_crn == null && var.cloud_logs_provision ? {
+    class = var.cloud_log_metrics_bucket_class
+    name  = local.cloud_log_metrics_bucket
+    tag   = var.cloud_log_metrics_bucket_access_tag
+  } : null
+
   buckets_config = concat(
     local.archive_bucket_config != null ? [local.archive_bucket_config] : [],
     local.at_bucket_config != null ? [local.at_bucket_config] : [],
-    local.cloud_log_data_bucket_config != null ? [local.cloud_log_data_bucket_config] : []
+    local.cloud_log_data_bucket_config != null ? [local.cloud_log_data_bucket_config] : [],
+    local.cloud_log_metrics_bucket_config != null ? [local.cloud_log_metrics_bucket_config] : []
   )
 
 
@@ -98,11 +105,18 @@ locals {
 
   apply_auth_policy = (var.skip_cos_kms_auth_policy || (length(coalesce(local.buckets_config, [])) == 0)) ? 0 : 1
 
+  # Cloud Logs data bucket
   cloud_log_data_bucket = var.prefix != null ? "${var.prefix}-${var.cloud_log_data_bucket_name}" : var.cloud_log_data_bucket_name
 
   parsed_log_data_bucket_name         = var.existing_cloud_logs_data_bucket_crn != null ? split(":", var.existing_cloud_logs_data_bucket_crn) : []
   existing_cloud_log_data_bucket_name = length(local.parsed_log_data_bucket_name) > 0 ? local.parsed_log_data_bucket_name[1] : null
 
+  # Cloud Logs metrics bucket
+  cloud_log_metrics_bucket = var.prefix != null ? "${var.prefix}-${var.cloud_log_metrics_bucket_name}" : var.cloud_log_metrics_bucket_name
+
+  parsed_log_metrics_bucket_name         = var.existing_cloud_logs_metrics_bucket_crn != null ? split(":", var.existing_cloud_logs_metrics_bucket_crn) : []
+  existing_cloud_log_metrics_bucket_name = length(local.parsed_log_metrics_bucket_name) > 0 ? local.parsed_log_metrics_bucket_name[1] : null
+
   # Event Notifications
   parsed_existing_en_instance_crn = var.existing_en_instance_crn != null ? split(":", var.existing_en_instance_crn) : []
   existing_en_guid                = length(local.parsed_existing_en_instance_crn) > 0 ? local.parsed_existing_en_instance_crn[7] : null
@@ -179,9 +193,9 @@ module "observability_instance" {
       bucket_endpoint = var.existing_cloud_logs_data_bucket_endpoint != null ? var.existing_cloud_logs_data_bucket_endpoint : module.cos_bucket[0].buckets[local.cloud_log_data_bucket].s3_endpoint_direct
     },
     metrics_data = {
-      enabled         = false # Support tracked in https://github.com/terraform-ibm-modules/terraform-ibm-observability-da/issues/170
-      bucket_crn      = null
-      bucket_endpoint = null
+      enabled         = true # Support of routing config is tracked in https://github.com/terraform-ibm-modules/terraform-ibm-observability-da/issues/170
+      bucket_crn      = var.existing_cloud_logs_metrics_bucket_crn != null ? var.existing_cloud_logs_metrics_bucket_crn : module.cos_bucket[0].buckets[local.cloud_log_metrics_bucket].bucket_crn
+      bucket_endpoint = var.existing_cloud_logs_metrics_bucket_endpoint != null ? var.existing_cloud_logs_metrics_bucket_endpoint : module.cos_bucket[0].buckets[local.cloud_log_metrics_bucket].s3_endpoint_direct
     }
   } : null
   cloud_logs_existing_en_instances = var.existing_en_instance_crn != null ? [{
diff --git a/solutions/instances/outputs.tf b/solutions/instances/outputs.tf
@@ -99,6 +99,11 @@ output "cloud_log_data_bucket_name" {
   description = "The name of the Cloud logs data COS bucket"
 }
 
+output "cloud_log_metrics_bucket_name" {
+  value       = var.existing_cloud_logs_metrics_bucket_crn == null && var.cloud_logs_provision ? module.cos_bucket[0].buckets[local.cloud_log_metrics_bucket].bucket_name : local.existing_cloud_log_metrics_bucket_name
+  description = "The name of the Cloud logs metrics COS bucket"
+}
+
 ## Activity Tracker
 output "at_targets" {
   value       = module.observability_instance.activity_tracker_targets
diff --git a/solutions/instances/variables.tf b/solutions/instances/variables.tf
@@ -142,6 +142,42 @@ variable "cloud_log_data_bucket_access_tag" {
   description = "A list of optional tags to add to the cloud log data object storage bucket."
 }
 
+variable "cloud_log_metrics_bucket_name" {
+  type        = string
+  default     = "cloud-logs-metrics-bucket"
+  description = "The name of the Cloud Object Storage bucket to create to store cloud logs metrics. Cloud Object Storage bucket names are globally unique. If the `add_bucket_name_suffix` variable is set to `true`, 4 random characters are added to this name to ensure that the name of the bucket is globally unique. If the prefix input variable is passed, the name of the bucket is prefixed to the value in the `<prefix>-value` format."
+}
+
+variable "existing_cloud_logs_metrics_bucket_crn" {
+  type        = string
+  nullable    = true
+  default     = null
+  description = "The crn of an existing bucket within the Cloud Object Storage instance to store IBM Cloud Logs metrics. If an existing Cloud Object Storage bucket is not specified, a bucket is created."
+}
+
+variable "existing_cloud_logs_metrics_bucket_endpoint" {
+  type        = string
+  nullable    = true
+  default     = null
+  description = "The endpoint of an existing Cloud Object Storage bucket to use for storing the IBM Cloud Logs metrics. If an existing Cloud Object Storage bucket is not specified, a bucket is created."
+}
+
+variable "cloud_log_metrics_bucket_class" {
+  type        = string
+  default     = "smart"
+  description = "The storage class of the newly provisioned cloud logs Cloud Object Storage bucket. Specify one of the following values for the storage class: `standard`, `vault`, `cold`, `smart` (default), or `onerate_active`."
+  validation {
+    condition     = contains(["standard", "vault", "cold", "smart", "onerate_active"], var.cloud_log_metrics_bucket_class)
+    error_message = "Specify one of the following values for the `cos_bucket_class`:  `standard`, `vault`, `cold`, `smart`, or `onerate_active`."
+  }
+}
+
+variable "cloud_log_metrics_bucket_access_tag" {
+  type        = list(string)
+  default     = []
+  description = "A list of optional tags to add to the cloud log metrics object storage bucket."
+}
+
 variable "skip_logs_routing_auth_policy" {
   description = "Whether to create an IAM authorization policy that permits Logs Routing Sender access to the IBM Cloud Logs."
   type        = bool
diff --git a/tests/pr_test.go b/tests/pr_test.go
@@ -261,22 +261,24 @@ func TestRunExistingResourcesInstances(t *testing.T) {
 			ImplicitRequired: false,
 			Region:           region,
 			TerraformVars: map[string]interface{}{
-				"cos_region":                               region,
-				"resource_group_name":                      terraform.Output(t, existingTerraformOptions, "resource_group_name"),
-				"use_existing_resource_group":              true,
-				"log_analysis_provision":                   true,
-				"existing_log_archive_cos_bucket_name":     terraform.Output(t, existingTerraformOptions, "bucket_name"),
-				"existing_at_cos_target_bucket_name":       terraform.Output(t, existingTerraformOptions, "bucket_name_at"),
-				"existing_log_archive_cos_bucket_endpoint": terraform.Output(t, existingTerraformOptions, "bucket_endpoint"),
-				"existing_at_cos_target_bucket_endpoint":   terraform.Output(t, existingTerraformOptions, "bucket_endpoint_at"),
-				"existing_cos_instance_crn":                terraform.Output(t, existingTerraformOptions, "cos_crn"),
-				"existing_cloud_logs_data_bucket_crn":      terraform.Output(t, existingTerraformOptions, "data_bucket_crn"),
-				"existing_cloud_logs_data_bucket_endpoint": terraform.Output(t, existingTerraformOptions, "data_bucket_endpoint"),
-				"existing_en_instance_crn":                 terraform.Output(t, existingTerraformOptions, "en_crn"),
-				"management_endpoint_type_for_bucket":      "public",
-				"log_analysis_service_endpoints":           "public",
-				"enable_platform_metrics":                  "false",
-				"enable_at_event_routing_to_log_analysis":  "true",
+				"cos_region":                                  region,
+				"resource_group_name":                         terraform.Output(t, existingTerraformOptions, "resource_group_name"),
+				"use_existing_resource_group":                 true,
+				"log_analysis_provision":                      true,
+				"existing_log_archive_cos_bucket_name":        terraform.Output(t, existingTerraformOptions, "bucket_name"),
+				"existing_at_cos_target_bucket_name":          terraform.Output(t, existingTerraformOptions, "bucket_name_at"),
+				"existing_log_archive_cos_bucket_endpoint":    terraform.Output(t, existingTerraformOptions, "bucket_endpoint"),
+				"existing_at_cos_target_bucket_endpoint":      terraform.Output(t, existingTerraformOptions, "bucket_endpoint_at"),
+				"existing_cos_instance_crn":                   terraform.Output(t, existingTerraformOptions, "cos_crn"),
+				"existing_cloud_logs_data_bucket_crn":         terraform.Output(t, existingTerraformOptions, "data_bucket_crn"),
+				"existing_cloud_logs_data_bucket_endpoint":    terraform.Output(t, existingTerraformOptions, "data_bucket_endpoint"),
+				"existing_cloud_logs_metrics_bucket_crn":      terraform.Output(t, existingTerraformOptions, "metrics_bucket_crn"),
+				"existing_cloud_logs_metrics_bucket_endpoint": terraform.Output(t, existingTerraformOptions, "metrics_bucket_endpoint"),
+				"existing_en_instance_crn":                    terraform.Output(t, existingTerraformOptions, "en_crn"),
+				"management_endpoint_type_for_bucket":         "public",
+				"log_analysis_service_endpoints":              "public",
+				"enable_platform_metrics":                     "false",
+				"enable_at_event_routing_to_log_analysis":     "true",
 			},
 		})
 
diff --git a/tests/resources/existing-resources/main.tf b/tests/resources/existing-resources/main.tf
@@ -47,6 +47,14 @@ module "cloud_log_buckets" {
       create_cos_instance    = false
       resource_instance_id   = module.cos.cos_instance_id
       kms_encryption_enabled = false
+    },
+    {
+      bucket_name            = "${var.prefix}-metrics-bucket"
+      add_bucket_name_suffix = true
+      region_location        = var.region
+      create_cos_instance    = false
+      resource_instance_id   = module.cos.cos_instance_id
+      kms_encryption_enabled = false
     }
   ]
 }
diff --git a/tests/resources/existing-resources/outputs.tf b/tests/resources/existing-resources/outputs.tf
@@ -47,6 +47,16 @@ output "data_bucket_endpoint" {
   value       = module.cloud_log_buckets.buckets["${var.prefix}-data-bucket"].s3_endpoint_public
 }
 
+output "metrics_bucket_crn" {
+  description = "Cloud Logs metrics bucket CRN"
+  value       = module.cloud_log_buckets.buckets["${var.prefix}-metrics-bucket"].bucket_crn
+}
+
+output "metrics_bucket_endpoint" {
+  description = "Cloud Logs metrics bucket endpoint"
+  value       = module.cloud_log_buckets.buckets["${var.prefix}-metrics-bucket"].s3_endpoint_public
+}
+
 output "en_crn" {
   description = "Event Notification CRN"
   value       = module.event_notification.crn

Original file line number	Diff line number	Diff line change
`@@ -47,6 +47,14 @@ module "cloud_log_buckets" {`
`47`	`47`	`create_cos_instance = false`
`48`	`48`	`resource_instance_id = module.cos.cos_instance_id`
`49`	`49`	`kms_encryption_enabled = false`
	`50`	`+ },`
	`51`	`+ {`
	`52`	`+ bucket_name = "${var.prefix}-metrics-bucket"`
	`53`	`+ add_bucket_name_suffix = true`
	`54`	`+ region_location = var.region`
	`55`	`+ create_cos_instance = false`
	`56`	`+ resource_instance_id = module.cos.cos_instance_id`
	`57`	`+ kms_encryption_enabled = false`
`50`	`58`	`}`
`51`	`59`	`]`
`52`	`60`	`}`