feat: added the ability to create a COS bucket in an instance that exists in a different account to the Clpud Logs instance (#65)

Author: kierramarie

ibm_catalog.json

@@ -197,6 +197,9 @@
                 }
               ]
             },
+            {
+              "key": "ibmcloud_cos_api_key"
+            },
             {
               "key": "cloud_logs_data_cos_bucket_name"
             },
@@ -233,6 +236,9 @@
                 }
               ]
             },
+            {
+              "key": "skip_cloud_logs_cos_auth_policy"
+            },
             {
               "key": "skip_cos_kms_iam_auth_policy"
             },
@@ -329,6 +335,13 @@
                 "crn:v1:bluemix:public:event-notifications::::serviceRole:Event-Source-Manager",
                 "crn:v1:bluemix:public:iam::::role:Viewer"
               ]
+            },
+            {
+              "role_crns": [
+                "crn:v1:bluemix:public:iam::::role:Administrator"
+              ],
+              "service_name": "iam-identity",
+              "notes": "[Optional] Required if creating COS buckets in a separate account from the account Cloud Logs is created in."
             }
           ],
           "architecture": {
@@ -521,6 +534,9 @@
                 }
               ]
             },
+            {
+              "key": "ibmcloud_cos_api_key"
+            },
             {
               "key": "cloud_logs_data_cos_bucket_name"
             },
@@ -552,6 +568,9 @@
                 }
               ]
             },
+            {
+              "key": "skip_cloud_logs_cos_auth_policy"
+            },
             {
               "key": "skip_cos_kms_iam_auth_policy"
             },
@@ -626,6 +645,13 @@
                 "crn:v1:bluemix:public:event-notifications::::serviceRole:Event-Source-Manager",
                 "crn:v1:bluemix:public:iam::::role:Viewer"
               ]
+            },
+            {
+              "role_crns": [
+                "crn:v1:bluemix:public:iam::::role:Administrator"
+              ],
+              "service_name": "iam-identity",
+              "notes": "[Optional] Required if creating COS buckets in a separate account from the account Cloud Logs is created in."
             }
           ],
           "architecture": {

solutions/fully-configurable/main.tf

@@ -16,9 +16,16 @@ locals {
   prefix            = var.prefix != null ? trimspace(var.prefix) != "" ? "${var.prefix}-" : "" : ""
   create_cloud_logs = var.existing_cloud_logs_crn == null
   cloud_logs_crn    = local.create_cloud_logs ? module.cloud_logs[0].crn : var.existing_cloud_logs_crn
+  # Even though we're only performing a comparison (var.ibmcloud_cos_api_key != null),
+  # Terraform treats the entire value as "tainted" due to sensitivity.
+  # Later, in the cloud_logs module, where the data_storage input variable is used in a for_each loop,
+  # the loop fails with the error: "Sensitive values, or values derived from sensitive values, cannot be used as for_each arguments."
+  # However, since we use nonsensitive() solely for logical comparison, we are not exposing any secret values to logs and it's safe to use. Issue https://github.ibm.com/GoldenEye/issues/issues/13562.
+  skip_cos_auth_policy = nonsensitive(var.ibmcloud_cos_api_key) != null ? true : var.skip_cloud_logs_cos_auth_policy
 }
 
 module "cloud_logs" {
+  depends_on                             = [time_sleep.wait_for_cos_authorization_policy[0]]
   count                                  = local.create_cloud_logs ? 1 : 0
   source                                 = "../.."
   resource_group_id                      = module.resource_group.resource_group_id
@@ -33,14 +40,16 @@ module "cloud_logs" {
   cbr_rules                              = var.cloud_logs_cbr_rules
   data_storage = {
     logs_data = {
-      enabled         = true
-      bucket_crn      = module.buckets.buckets[local.data_bucket_name].bucket_crn
-      bucket_endpoint = module.buckets.buckets[local.data_bucket_name].s3_endpoint_direct
+      enabled              = true
+      bucket_crn           = module.buckets.buckets[local.data_bucket_name].bucket_crn
+      bucket_endpoint      = module.buckets.buckets[local.data_bucket_name].s3_endpoint_direct
+      skip_cos_auth_policy = local.skip_cos_auth_policy
     },
     metrics_data = {
-      enabled         = true
-      bucket_crn      = module.buckets.buckets[local.metrics_bucket_name].bucket_crn
-      bucket_endpoint = module.buckets.buckets[local.metrics_bucket_name].s3_endpoint_direct
+      enabled              = true
+      bucket_crn           = module.buckets.buckets[local.metrics_bucket_name].bucket_crn
+      bucket_endpoint      = module.buckets.buckets[local.metrics_bucket_name].s3_endpoint_direct
+      skip_cos_auth_policy = local.skip_cos_auth_policy
     }
   }
   logs_routing_tenant_regions   = var.logs_routing_tenant_regions
@@ -68,7 +77,8 @@ locals {
   kms_key_crn   = var.kms_encryption_enabled_buckets ? var.existing_kms_key_crn != null ? var.existing_kms_key_crn : module.kms[0].keys[format("%s.%s", local.key_ring_name, local.key_name)].crn : null
   kms_key_id    = var.existing_kms_instance_crn != null ? module.kms[0].keys[format("%s.%s", local.key_ring_name, local.key_name)].key_id : var.existing_kms_key_crn != null ? module.existing_kms_key_crn_parser[0].resource : null
 
-  create_cross_account_auth_policy = var.existing_cloud_logs_crn == null ? !var.skip_cos_kms_iam_auth_policy && var.ibmcloud_kms_api_key == null ? false : true : false
+  create_cross_account_auth_policy     = var.existing_cloud_logs_crn == null ? !var.skip_cos_kms_iam_auth_policy && var.ibmcloud_kms_api_key == null ? false : true : false
+  create_cross_account_cos_auth_policy = var.existing_cloud_logs_crn == null && var.ibmcloud_cos_api_key != null && !var.skip_cloud_logs_cos_auth_policy
 }
 
 module "existing_cos_instance_crn_parser" {
@@ -131,6 +141,62 @@ module "buckets" {
   ]
 }
 
+module "bucket_crns" {
+  for_each = module.buckets.buckets
+  source   = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser"
+  version  = "1.1.0"
+  crn      = each.value.bucket_id
+}
+
+data "ibm_iam_account_settings" "iam_account_settings" {
+  count = local.create_cross_account_cos_auth_policy ? 1 : 0
+}
+
+resource "ibm_iam_authorization_policy" "cos_policy" {
+  provider               = ibm.cos
+  count                  = local.create_cross_account_cos_auth_policy ? length(module.buckets.bucket_configs) : 0
+  source_service_account = data.ibm_iam_account_settings.iam_account_settings[0].account_id
+  source_service_name    = "logs"
+  roles                  = ["Writer"]
+  description            = "Allow Cloud logs instances `Writer` access to the COS bucket with ID ${module.bucket_crns[module.buckets.bucket_configs[count.index].bucket_name].resource}, in the COS instance with ID ${module.existing_cos_instance_crn_parser.service_instance}."
+
+  resource_attributes {
+    name     = "serviceName"
+    operator = "stringEquals"
+    value    = "cloud-object-storage"
+  }
+
+  resource_attributes {
+    name     = "accountId"
+    operator = "stringEquals"
+    value    = module.existing_cos_instance_crn_parser.account_id
+  }
+
+  resource_attributes {
+    name     = "serviceInstance"
+    operator = "stringEquals"
+    value    = module.existing_cos_instance_crn_parser.service_instance
+  }
+
+  resource_attributes {
+    name     = "resourceType"
+    operator = "stringEquals"
+    value    = "bucket"
+  }
+
+  resource_attributes {
+    name     = "resource"
+    operator = "stringEquals"
+    value    = module.bucket_crns[module.buckets.bucket_configs[count.index].bucket_name].resource
+  }
+}
+
+resource "time_sleep" "wait_for_cos_authorization_policy" {
+  depends_on      = [ibm_iam_authorization_policy.cos_policy]
+  count           = var.ibmcloud_cos_api_key != null && !var.skip_cloud_logs_cos_auth_policy ? length(module.buckets.bucket_configs) : 0
+  create_duration = "30s"
+}
+
 module "existing_kms_crn_parser" {
   count   = var.existing_kms_instance_crn != null ? 1 : 0
   source  = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser"

solutions/fully-configurable/providers.tf

@@ -17,7 +17,7 @@ provider "ibm" {
 
 provider "ibm" {
   alias            = "cos"
-  ibmcloud_api_key = var.ibmcloud_api_key
+  ibmcloud_api_key = var.ibmcloud_cos_api_key != null ? var.ibmcloud_cos_api_key : var.ibmcloud_api_key
   region           = var.region
   visibility       = var.provider_visibility
 }

solutions/fully-configurable/variables.tf

@@ -50,6 +50,13 @@ variable "region" {
 # COS
 ########################################################################################################################
 
+variable "ibmcloud_cos_api_key" {
+  type        = string
+  description = "The IBM Cloud API key that can create Cloud Object Storage (COS) buckets. If not specified, the 'ibmcloud_api_key' variable is used. Specify this key if the COS instance is in an account that's different from the one associated with the cloud logs resources. Do not set if the same account owns all the instances."
+  sensitive   = true
+  default     = null
+}
+
 variable "existing_cos_instance_crn" {
   type        = string
   description = "The CRN of an existing Object Storage instance."
@@ -101,6 +108,12 @@ variable "skip_cos_kms_iam_auth_policy" {
   default     = false
 }
 
+variable "skip_cloud_logs_cos_auth_policy" {
+  type        = bool
+  description = "To skip creating an IAM authorization policy that allows the IBM Cloud logs to write to the Cloud Object Storage bucket, set this variable to `true`."
+  default     = false
+}
+
 variable "existing_monitoring_crn" {
   type        = string
   nullable    = true

solutions/security-enforced/main.tf

@@ -2,6 +2,7 @@ module "security_enforced" {
   source                                 = "../fully-configurable"
   ibmcloud_api_key                       = var.ibmcloud_api_key
   ibmcloud_kms_api_key                   = var.ibmcloud_kms_api_key
+  ibmcloud_cos_api_key                   = var.ibmcloud_cos_api_key
   provider_visibility                    = "private"
   prefix                                 = var.prefix
   region                                 = var.region
@@ -16,6 +17,7 @@ module "security_enforced" {
   cloud_logs_cos_buckets_class           = var.cloud_logs_cos_buckets_class
   management_endpoint_type_for_buckets   = "private"
   skip_cos_kms_iam_auth_policy           = var.skip_cos_kms_iam_auth_policy
+  skip_cloud_logs_cos_auth_policy        = var.skip_cloud_logs_cos_auth_policy
   existing_monitoring_crn                = var.existing_monitoring_crn
   kms_encryption_enabled_buckets         = true
   existing_kms_instance_crn              = var.existing_kms_instance_crn

solutions/security-enforced/variables.tf

@@ -39,6 +39,13 @@ variable "region" {
 # COS
 ########################################################################################################################
 
+variable "ibmcloud_cos_api_key" {
+  type        = string
+  description = "The IBM Cloud API key that can create Cloud Object Storage (COS) buckets. If not specified, the 'ibmcloud_api_key' variable is used. Specify this key if the COS instance is in an account that's different from the one associated with the cloud logs resources. Leave empty if the same account owns all the instances."
+  sensitive   = true
+  default     = null
+}
+
 variable "existing_cos_instance_crn" {
   type        = string
   description = "The CRN of an existing Object Storage instance."
@@ -75,6 +82,12 @@ variable "skip_cos_kms_iam_auth_policy" {
   default     = false
 }
 
+variable "skip_cloud_logs_cos_auth_policy" {
+  type        = bool
+  description = "To skip creating an IAM authorization policy that allows the IBM Cloud logs to write to the Cloud Object Storage bucket, set this variable to `true`."
+  default     = false
+}
+
 variable "existing_monitoring_crn" {
   type        = string
   nullable    = true