feat: updated the modules used by the instances DA. When upgrading from a previous version, the logs will show several messages about resource addresses being moved. This is expected and can be ignored.<br>- The Activity Tracker / COS service to service auth policy is now scoped to the exact COS bucket. When updating from an older version you will see the destroy and re-create of the auth policy, however the new policy will be created before the old one is deleted so there will be no disruption to every day services (#350)

Author: ocofaigh

solutions/instances/main.tf

@@ -64,7 +64,7 @@ locals {
     rules = [{
       action = "send"
       targets = [{
-        id = module.observability_instance.metrics_router_targets[local.metric_router_target_name].id
+        id = module.metrics_router.metrics_router_targets[local.metric_router_target_name].id
       }]
       inclusion_filters = []
     }]
@@ -127,13 +127,13 @@ locals {
   at_cos_route = var.enable_at_event_routing_to_cos_bucket ? [{
     route_name = local.at_cos_route_name
     locations  = ["*", "global"]
-    target_ids = [module.observability_instance.activity_tracker_targets[local.cos_target_name].id]
+    target_ids = [module.activity_tracker.activity_tracker_targets[local.cos_target_name].id]
   }] : []
 
   at_cloud_logs_route = var.enable_at_event_routing_to_cloud_logs ? [{
     route_name = local.at_cloud_logs_route_name
     locations  = ["*", "global"]
-    target_ids = [module.observability_instance.activity_tracker_targets[local.cloud_logs_target_name].id]
+    target_ids = [module.activity_tracker.activity_tracker_targets[local.cloud_logs_target_name].id]
   }] : []
 
   apply_auth_policy = (var.skip_cos_kms_auth_policy || (length(coalesce(local.buckets_config, [])) == 0)) ? 0 : 1
@@ -249,31 +249,33 @@ module "cloud_monitoring_crn_parser" {
   crn     = var.existing_cloud_monitoring_crn
 }
 
-module "observability_instance" {
-  depends_on        = [time_sleep.wait_for_atracker_cos_authorization_policy]
-  source            = "terraform-ibm-modules/observability-instances/ibm"
-  version           = "3.5.2"
+module "cloud_monitoring" {
+  count                   = var.cloud_monitoring_provision ? 1 : 0
+  source                  = "terraform-ibm-modules/cloud-monitoring/ibm"
+  version                 = "1.2.1"
+  region                  = var.region
+  resource_group_id       = module.resource_group.resource_group_id
+  instance_name           = local.cloud_monitoring_instance_name
+  plan                    = var.cloud_monitoring_plan
+  resource_tags           = var.cloud_monitoring_tags
+  access_tags             = [] # TODO: Add support
+  enable_platform_metrics = var.enable_platform_metrics
+  service_endpoints       = "public-and-private"
+}
+
+module "cloud_logs" {
+  count             = var.cloud_logs_provision ? 1 : 0
+  source            = "terraform-ibm-modules/cloud-logs/ibm"
+  version           = "1.3.0"
   region            = var.region
   resource_group_id = module.resource_group.resource_group_id
-
-  # IBM Cloud Monitoring
-  cloud_monitoring_provision         = var.cloud_monitoring_provision
-  cloud_monitoring_instance_name     = local.cloud_monitoring_instance_name
-  cloud_monitoring_plan              = var.cloud_monitoring_plan
-  cloud_monitoring_tags              = var.cloud_monitoring_tags
-  cloud_monitoring_service_endpoints = "public-and-private"
-  enable_platform_metrics            = var.enable_platform_metrics
-
-  # IBM Cloud Logs
-  cloud_logs_provision         = var.cloud_logs_provision
-  cloud_logs_instance_name     = local.cloud_logs_instance_name
-  cloud_logs_plan              = "standard"
-  cloud_logs_access_tags       = var.cloud_logs_access_tags
-  cloud_logs_tags              = var.cloud_logs_tags
-  cloud_logs_service_endpoints = "public-and-private"
-  cloud_logs_retention_period  = var.cloud_logs_retention_period
-  cloud_logs_policies          = var.cloud_logs_policies
-  cloud_logs_data_storage = var.cloud_logs_provision ? {
+  instance_name     = local.cloud_logs_instance_name
+  plan              = "standard"
+  resource_tags     = var.cloud_logs_tags
+  access_tags       = var.cloud_logs_access_tags
+  retention_period  = var.cloud_logs_retention_period
+  service_endpoints = "public-and-private"
+  data_storage = var.cloud_logs_provision ? {
     logs_data = {
       enabled         = true
       bucket_crn      = local.cloud_logs_data_bucket_crn
@@ -297,17 +299,37 @@ module "observability_instance" {
       skip_cos_auth_policy = nonsensitive(var.ibmcloud_cos_api_key) != null ? true : var.skip_cloud_logs_cos_auth_policy
     }
   } : null
-  cloud_logs_existing_en_instances = [for index, _ in local.cloud_logs_existing_en_instances : {
+  existing_event_notifications_instances = [for index, _ in local.cloud_logs_existing_en_instances : {
     en_instance_id      = module.en_crn_parser[index]["service_instance"]
     en_region           = module.en_crn_parser[index]["region"]
     en_integration_name = try("${local.prefix}-${local.cloud_logs_existing_en_instances[index]["integration_name"]}", local.cloud_logs_existing_en_instances[index]["integration_name"])
     skip_en_auth_policy = local.cloud_logs_existing_en_instances[index]["skip_en_auth_policy"]
   }]
-  skip_logs_routing_auth_policy = var.skip_logs_routing_auth_policy
   logs_routing_tenant_regions   = var.logs_routing_tenant_regions
+  skip_logs_routing_auth_policy = var.skip_logs_routing_auth_policy
+  policies                      = var.cloud_logs_policies
+}
+
+module "metrics_router" {
+  source  = "terraform-ibm-modules/cloud-monitoring/ibm//modules/metrics_routing"
+  version = "1.2.1"
+  metrics_router_targets = var.enable_metrics_routing_to_cloud_monitoring ? [
+    {
+      destination_crn                 = var.cloud_monitoring_provision ? module.cloud_monitoring[0].crn : var.existing_cloud_monitoring_crn
+      target_name                     = local.metric_router_target_name
+      target_region                   = var.cloud_monitoring_provision ? var.region : module.cloud_monitoring_crn_parser[0].region
+      skip_metrics_router_auth_policy = false
+    }
+  ] : []
+  metrics_router_routes   = var.enable_metrics_routing_to_cloud_monitoring ? (length(var.metrics_router_routes) != 0 ? var.metrics_router_routes : local.default_metrics_router_route) : []
+  metrics_router_settings = var.enable_metrics_routing_to_cloud_monitoring ? (var.metrics_router_settings != null ? var.metrics_router_settings : local.metrics_router_settings) : null
+}
 
-  # Activity Tracker
-  at_cos_targets = var.enable_at_event_routing_to_cos_bucket ? [
+module "activity_tracker" {
+  depends_on = [time_sleep.wait_for_atracker_cos_authorization_policy]
+  source     = "terraform-ibm-modules/activity-tracker/ibm"
+  version    = "1.0.0"
+  cos_targets = var.enable_at_event_routing_to_cos_bucket ? [
     {
       bucket_name                       = local.cos_target_bucket_name
       endpoint                          = local.cos_target_bucket_endpoint
@@ -318,32 +340,14 @@ module "observability_instance" {
       service_to_service_enabled        = true
     }
   ] : []
-
-  at_cloud_logs_targets = var.enable_at_event_routing_to_cloud_logs ? [
+  cloud_logs_targets = var.enable_at_event_routing_to_cloud_logs ? [
     {
-      instance_id   = module.observability_instance.cloud_logs_crn
+      instance_id   = module.cloud_logs[0].crn
       target_region = var.region
       target_name   = local.cloud_logs_target_name
     }
   ] : []
-
-  # Routes
   activity_tracker_routes = local.at_routes
-
-  # IBM Cloud Metrics Routing
-
-  metrics_router_targets = var.enable_metrics_routing_to_cloud_monitoring ? [
-    {
-      destination_crn                     = var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_crn : var.existing_cloud_monitoring_crn
-      target_name                         = local.metric_router_target_name
-      target_region                       = var.cloud_monitoring_provision ? var.region : module.cloud_monitoring_crn_parser[0].region
-      skip_mrouter_sysdig_iam_auth_policy = false
-    }
-  ] : []
-
-  metrics_router_routes = var.enable_metrics_routing_to_cloud_monitoring ? (length(var.metrics_router_routes) != 0 ? var.metrics_router_routes : local.default_metrics_router_route) : []
-
-  metrics_router_settings = var.enable_metrics_routing_to_cloud_monitoring ? (var.metrics_router_settings != null ? var.metrics_router_settings : local.metrics_router_settings) : null
 }
 
 resource "time_sleep" "wait_for_atracker_cos_authorization_policy" {
@@ -510,7 +514,7 @@ module "cos_bucket" {
         usage_metrics_enabled   = true
         request_metrics_enabled = true
         # if DA is creating monitoring instance, use that. If its passing existing instance, use that. If neither, pass null, meaning metrics are sent to the instance associated to the container's location unless otherwise specified in the Metrics Router service configuration.
-        metrics_monitoring_crn = var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_crn : var.existing_cloud_monitoring_crn != null ? var.existing_cloud_monitoring_crn : null
+        metrics_monitoring_crn = var.cloud_monitoring_provision ? module.cloud_monitoring[0].crn : var.existing_cloud_monitoring_crn != null ? var.existing_cloud_monitoring_crn : null
       }
       activity_tracking = {
         read_data_events  = true

solutions/instances/moved.tf

@@ -0,0 +1,29 @@
+moved {
+  from = module.observability_instance.module.metric_routing
+  to   = module.metrics_router
+}
+
+moved {
+  from = module.observability_instance.module.cloud_logs[0]
+  to   = module.cloud_logs[0]
+}
+
+moved {
+  from = module.observability_instance.module.cloud_monitoring[0].ibm_resource_instance.cloud_monitoring[0]
+  to   = module.cloud_monitoring[0].ibm_resource_instance.cloud_monitoring
+}
+
+moved {
+  from = module.observability_instance.module.cloud_monitoring[0].ibm_resource_key.resource_key[0]
+  to   = module.cloud_monitoring[0].ibm_resource_key.resource_key
+}
+
+moved {
+  from = module.observability_instance.module.cloud_monitoring[0].ibm_resource_tag.cloud_monitoring_tag
+  to   = module.cloud_monitoring[0].ibm_resource_tag.cloud_monitoring_tag
+}
+
+moved {
+  from = module.observability_instance.module.activity_tracker
+  to   = module.activity_tracker
+}

solutions/instances/outputs.tf

@@ -15,59 +15,59 @@ output "resource_group_id" {
 
 ## Cloud logs
 output "cloud_logs_crn" {
-  value       = var.cloud_logs_provision ? module.observability_instance.cloud_logs_crn : null
+  value       = var.cloud_logs_provision ? module.cloud_logs[0].crn : null
   description = "The id of the provisioned Cloud Logs instance."
 }
 
 output "cloud_logs_guid" {
-  value       = var.cloud_logs_provision ? module.observability_instance.cloud_logs_guid : null
+  value       = var.cloud_logs_provision ? module.cloud_logs[0].guid : null
   description = "The guid of the provisioned Cloud Logs instance."
 }
 
 output "cloud_logs_name" {
-  value       = var.cloud_logs_provision ? module.observability_instance.cloud_logs_name : null
+  value       = var.cloud_logs_provision ? module.cloud_logs[0].name : null
   description = "The name of the provisioned Cloud Logs instance."
 }
 
 output "cloud_logs_resource_group_id" {
-  value       = var.cloud_logs_provision ? module.observability_instance.cloud_logs_resource_group_id : null
+  value       = var.cloud_logs_provision ? module.cloud_logs[0].resource_group_id : null
   description = "The resource group where Cloud Logs instance resides."
 }
 
 output "cloud_logs_ingress_endpoint" {
-  value       = var.cloud_logs_provision ? module.observability_instance.cloud_logs_ingress_endpoint : null
+  value       = var.cloud_logs_provision ? module.cloud_logs[0].ingress_endpoint : null
   description = "The public ingress endpoint of the provisioned Cloud Logs instance."
 }
 
 output "cloud_logs_ingress_private_endpoint" {
-  value       = var.cloud_logs_provision ? module.observability_instance.cloud_logs_ingress_private_endpoint : null
+  value       = var.cloud_logs_provision ? module.cloud_logs[0].ingress_private_endpoint : null
   description = "The private ingress endpoint of the provisioned Cloud Logs instance."
 }
 
 ## Cloud logs policies
 output "logs_policies_details" {
-  value       = length(var.cloud_logs_policies) > 0 ? module.observability_instance.logs_policies_details : null
+  value       = length(var.cloud_logs_policies) > 0 ? module.cloud_logs[0].logs_policies_details : null
   description = "The details of the Cloud logs policies created."
 }
 
 ## Cloud Monitoring
 output "cloud_monitoring_name" {
-  value       = var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_name : (var.existing_cloud_monitoring_crn != null ? module.cloud_monitoring_crn_parser[0].service_name : null)
+  value       = var.cloud_monitoring_provision ? module.cloud_monitoring[0].name : (var.existing_cloud_monitoring_crn != null ? module.cloud_monitoring_crn_parser[0].service_name : null)
   description = "The name of the provisioned IBM cloud monitoring instance."
 }
 
 output "cloud_monitoring_crn" {
-  value       = var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_crn : (var.existing_cloud_monitoring_crn != null ? var.existing_cloud_monitoring_crn : null)
+  value       = var.cloud_monitoring_provision ? module.cloud_monitoring[0].crn : (var.existing_cloud_monitoring_crn != null ? var.existing_cloud_monitoring_crn : null)
   description = "The id of the provisioned IBM cloud monitoring instance."
 }
 
 output "cloud_monitoring_guid" {
-  value       = var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_guid : var.existing_cloud_monitoring_crn != null ? module.cloud_monitoring_crn_parser[0].service_instance : null
+  value       = var.cloud_monitoring_provision ? module.cloud_monitoring[0].guid : var.existing_cloud_monitoring_crn != null ? module.cloud_monitoring_crn_parser[0].service_instance : null
   description = "The guid of the provisioned IBM cloud monitoring instance."
 }
 
 output "cloud_monitoring_access_key" {
-  value       = var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_access_key : null
+  value       = var.cloud_monitoring_provision ? module.cloud_monitoring[0].access_key : null
   description = "IBM cloud monitoring access key for agents to use"
   sensitive   = true
 }
@@ -116,12 +116,12 @@ output "cloud_log_metrics_bucket_name" {
 
 ## Activity Tracker
 output "at_targets" {
-  value       = module.observability_instance.activity_tracker_targets
+  value       = module.activity_tracker.activity_tracker_targets
   description = "The map of created activity_tracker targets"
 }
 
 output "at_routes" {
-  value       = module.observability_instance.activity_tracker_routes
+  value       = module.activity_tracker.activity_tracker_routes
   description = "The map of created activity_tracker routes"
 }
 
@@ -140,10 +140,10 @@ output "kms_keys" {
 
 output "metrics_router_targets" {
   description = "The map of created metrics routing targets."
-  value       = var.enable_metrics_routing_to_cloud_monitoring ? module.observability_instance.metrics_router_targets : null
+  value       = var.enable_metrics_routing_to_cloud_monitoring ? module.metrics_router.metrics_router_targets : null
 }
 
 output "metrics_router_routes" {
   description = "The map of created metrics routing routes."
-  value       = var.enable_metrics_routing_to_cloud_monitoring ? module.observability_instance.metrics_router_routes : null
+  value       = var.enable_metrics_routing_to_cloud_monitoring ? module.metrics_router.metrics_router_routes : null
 }

solutions/instances/variables.tf

@@ -239,7 +239,7 @@ variable "metrics_router_settings" {
       id = string
     })))
   })
-  description = "Global settings for Metrics Routing. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-observability-da/blob/main/solutions/instances/DA-types.md#metrics-router-settings-)"
+  description = "Global settings for Metrics Routing. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-observability-da/blob/main/solutions/instances/DA-types.md#metrics-router-settings-)."
   default     = null
 }
 

tests/pr_test.go

@@ -29,6 +29,16 @@ const solutionAgentsDADir = "solutions/agents"
 const solutionTenantsDADir = "solutions/logs-routing"
 const agentsKubeconfigDir = "solutions/agents/kubeconfig"
 
+var IgnoreInstanceUpdates = []string{
+	// Need to ignore this since primary_metadata_region might be updating in the dev account due to tests using different regions
+	"module.metrics_router.ibm_metrics_router_settings.metrics_router_settings[0]",
+}
+
+var IgnoreAgentsUpdates = []string{
+	"module.observability_agents.module.logs_agent[0].helm_release.logs_agent",
+	"module.observability_agents.helm_release.cloud_monitoring_agent[0]",
+}
+
 // Currently only including regions that Event Notification support
 var validRegions = []string{
 	"au-syd",
@@ -73,10 +83,7 @@ func TestInstancesInSchematics(t *testing.T) {
 		DeleteWorkspaceOnFail:  false,
 		WaitJobCompleteMinutes: 60,
 		IgnoreUpdates: testhelper.Exemptions{
-			List: []string{
-				// Need to ignore this since primary_metadata_region might be updating in the dev account due to tests using different regions
-				"module.observability_instance.module.metric_routing.ibm_metrics_router_settings.metrics_router_settings[0]",
-			},
+			List: IgnoreInstanceUpdates,
 		},
 	})
 
@@ -110,10 +117,7 @@ func TestRunUpgradeSolutionInstances(t *testing.T) {
 		Region:       region,
 		Prefix:       "obs-ins-upg",
 		IgnoreUpdates: testhelper.Exemptions{
-			List: []string{
-				// Need to ignore this since primary_metadata_region might be updating in the dev account due to tests using different regions
-				"module.observability_instance.module.metric_routing.ibm_metrics_router_settings.metrics_router_settings[0]",
-			},
+			List: IgnoreInstanceUpdates,
 		},
 	})
 
@@ -199,10 +203,7 @@ func TestAgentsSolutionInSchematics(t *testing.T) {
 			WaitJobCompleteMinutes: 60,
 			Region:                 region,
 			IgnoreUpdates: testhelper.Exemptions{ // Ignore for consistency check
-				List: []string{
-					"module.observability_agents.module.logs_agent[0].helm_release.logs_agent",
-					"module.observability_agents.helm_release.cloud_monitoring_agent[0]",
-				},
+				List: IgnoreAgentsUpdates,
 			},
 		})
 
@@ -303,10 +304,7 @@ func TestRunExistingResourcesInstancesSchematics(t *testing.T) {
 			WaitJobCompleteMinutes: 60,
 			Region:                 region,
 			IgnoreUpdates: testhelper.Exemptions{
-				List: []string{
-					// Need to ignore this since primary_metadata_region might be updating in the dev account due to tests using different regions
-					"module.observability_instance.module.metric_routing.ibm_metrics_router_settings.metrics_router_settings[0]",
-				},
+				List: IgnoreInstanceUpdates,
 			},
 		})
 
@@ -350,10 +348,7 @@ func TestRunExistingResourcesInstancesSchematics(t *testing.T) {
 			WaitJobCompleteMinutes: 60,
 			Region:                 region,
 			IgnoreUpdates: testhelper.Exemptions{
-				List: []string{
-					// Need to ignore this since primary_metadata_region might be updating in the dev account due to tests using different regions
-					"module.observability_instance.module.metric_routing.ibm_metrics_router_settings.metrics_router_settings[0]",
-				},
+				List: IgnoreInstanceUpdates,
 			},
 		})