feat: added new inputs at_cos_bucket_retention_policy and cloud_log_data_bucket_retention_policy to the instances variation of the DA to allow retention to be configured on the COS buckets created by the solution. (#210)

Author: iamar7

ibm_catalog.json

@@ -244,6 +244,12 @@
             {
               "key": "cos_instance_access_tags"
             },
+            {
+              "key": "at_cos_bucket_retention_policy"
+            },
+            {
+              "key": "cloud_log_data_bucket_retention_policy"
+            },
             {
               "key": "add_bucket_name_suffix"
             },

solutions/instances/DA-types.md

@@ -5,6 +5,8 @@ Several optional input variables in the IBM Cloud [Observability instances deplo
 * Cloud Logs Event Notification Instances (`cloud_logs_existing_en_instances`)
 * Cloud Logs policies (`cloud_logs_policies`)
 * Metrics Router Routes (`metrics_router_routes`)
+* Activity Tracker Event Routing COS bucket retention policy (`at_cos_bucket_retention_policy`)
+* Cloud Logs data bucket retention policy(`cloud_log_data_bucket_retention_policy`)
 
 
 ## Cloud Logs Event Notification Instances <a name="cloud_logs_existing_en_instances"></a>
@@ -130,3 +132,57 @@ metrics_router_routes = {
 }
 ```
 Refer [here](https://cloud.ibm.com/docs/metrics-router?topic=metrics-router-route_rules_definitions&interface=ui) for more information about IBM Cloud Metrics Routing route.
+
+## at_cos_bucket_retention_policy <a name="at_cos_bucket_retention_policy"></a>
+
+The `at_cos_bucket_retention_policy` input variable allows you to provide the retention policy of the IBM Cloud Activity Tracker Event Routing COS target bucket that will be configured. Refer [here](https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-immutable) for more information.
+
+- Variable name: `at_cos_bucket_retention_policy`.
+- Type: An object representing a retention policy.
+- Default value: null (`null`).
+
+### Options for at_cos_bucket_retention_policy
+
+- `default` (optional): The number of days that an object can remain unmodified in an Object Storage bucket.
+- `maximum` (optional): The maximum number of days that an object can be kept unmodified in the bucket.
+- `minimum` (optional): The minimum number of days that an object must be kept unmodified in the bucket.
+- `permanent` (optional): Whether permanent retention status is enabled for the Object Storage bucket.
+
+### Example at_cos_bucket_retention_policy
+
+```hcl
+at_cos_bucket_retention_policy = {
+    default   = 90
+    maximum   = 350
+    minimum   = 90
+    permanent = false
+}
+```
+
+## cloud_log_data_bucket_retention_policy <a name="cloud_log_data_bucket_retention_policy"></a>
+
+The `cloud_log_data_bucket_retention_policy` input variable allows you to provide the retention policy of the IBM Cloud Logs data bucket that will be configured. Refer [here](https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-immutable) for more information.
+
+- Variable name: `cloud_log_data_bucket_retention_policy`.
+- Type: An object representing a retention policy.
+- Default value: null (`null`).
+
+### Options for cloud_log_data_bucket_retention_policy
+
+- `default` (optional): The number of days that an object can remain unmodified in an Object Storage bucket.
+- `maximum` (optional): The maximum number of days that an object can be kept unmodified in the bucket.
+- `minimum` (optional): The minimum number of days that an object must be kept unmodified in the bucket.
+- `permanent` (optional): Whether permanent retention status is enabled for the Object Storage bucket.
+
+
+
+### Example cloud_log_data_bucket_retention_policy
+
+```hcl
+cloud_log_data_bucket_retention_policy = {
+    default   = 90
+    maximum   = 350
+    minimum   = 90
+    permanent = false
+}
+```

solutions/instances/main.tf

@@ -91,20 +91,25 @@ locals {
     tag   = var.cloud_log_metrics_bucket_access_tag
   } : null
 
+  bucket_retention_configs = merge(
+    local.at_bucket_config != null ? { (local.at_cos_target_bucket_name) = var.at_cos_bucket_retention_policy } : null,
+    local.cloud_log_data_bucket_config != null ? { (local.cloud_log_data_bucket) = var.cloud_log_data_bucket_retention_policy } : null
+  )
+
   buckets_config = concat(
     local.archive_bucket_config != null ? [local.archive_bucket_config] : [],
     local.at_bucket_config != null ? [local.at_bucket_config] : [],
     local.cloud_log_data_bucket_config != null ? [local.cloud_log_data_bucket_config] : [],
     local.cloud_log_metrics_bucket_config != null ? [local.cloud_log_metrics_bucket_config] : []
   )
 
-  archive_rule = var.existing_at_cos_target_bucket_name == null ? {
+  archive_rule = length(local.buckets_config) != 0 ? {
     enable = true
     days   = 90
     type   = "Glacier"
   } : null
 
-  expire_rule = var.existing_at_cos_target_bucket_name == null ? {
+  expire_rule = length(local.buckets_config) != 0 ? {
     enable = true
     days   = 366
   } : null
@@ -120,6 +125,7 @@ locals {
     locations  = ["*", "global"]
     target_ids = [module.observability_instance.activity_tracker_targets[local.cloud_logs_target_name].id]
   }] : []
+
   apply_auth_policy = (var.skip_cos_kms_auth_policy || (length(coalesce(local.buckets_config, [])) == 0)) ? 0 : 1
   at_routes         = concat(local.at_cos_route, local.at_cloud_logs_route)
 
@@ -478,7 +484,7 @@ module "cos_bucket" {
       force_delete                  = true
       archive_rule                  = local.archive_rule
       expire_rule                   = local.expire_rule
-      retention_rule                = null
+      retention_rule                = lookup(local.bucket_retention_configs, value.name, null)
       metrics_monitoring = {
         usage_metrics_enabled   = true
         request_metrics_enabled = true

solutions/instances/variables.tf

@@ -97,7 +97,7 @@ variable "cloud_logs_access_tags" {
 }
 # https://github.ibm.com/GoldenEye/issues/issues/10928#issuecomment-93550079
 variable "cloud_logs_existing_en_instances" {
-  description = "A list of existing Event Notification instances to be integrated with the Cloud Logging service. Each object in the list represents an Event Notification instance, including its CRN, an optional name for the integration, and an optional flag to skip the authentication policy creation for the  Event Notification instance [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-observability-da/tree/main/solutions/standard/DA-types.md#cloud_logs_existing_en_instances). This variable is intended for integrating a multiple Event Notifications instance to Cloud Logs. If you need to integrate only one instance, you may also use the `existing_en_instance_crn`, `en_integration_name` and `skip_en_auth_policy` variables instead."
+  description = "A list of existing Event Notification instances to be integrated with the Cloud Logging service. Each object in the list represents an Event Notification instance, including its CRN, an optional name for the integration, and an optional flag to skip the authentication policy creation for the  Event Notification instance [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-observability-da/blob/main/solutions/instances/DA-types.md#cloud-logs-event-notification-instances-). This variable is intended for integrating a multiple Event Notifications instance to Cloud Logs. If you need to integrate only one instance, you may also use the `existing_en_instance_crn`, `en_integration_name` and `skip_en_auth_policy` variables instead."
   type = list(object({
     instance_crn        = string
     integration_name    = optional(string, "cloud-logs-en-integration")
@@ -135,78 +135,6 @@ variable "cloud_logs_retention_period" {
   }
 }
 
-variable "cloud_log_data_bucket_name" {
-  type        = string
-  default     = "cloud-logs-data-bucket"
-  description = "The name of the Cloud Object Storage bucket to create to store cloud log data. Cloud Object Storage bucket names are globally unique. If the `add_bucket_name_suffix` variable is set to `true`, 4 random characters are added to this name to ensure that the name of the bucket is globally unique. If the prefix input variable is passed, the name of the bucket is prefixed to the value in the `<prefix>-value` format."
-}
-
-variable "existing_cloud_logs_data_bucket_crn" {
-  type        = string
-  nullable    = true
-  default     = null
-  description = "The crn of an existing bucket within the Cloud Object Storage instance to store IBM Cloud Logs data. If an existing Cloud Object Storage bucket is not specified, a bucket is created."
-}
-
-variable "existing_cloud_logs_data_bucket_endpoint" {
-  type        = string
-  nullable    = true
-  default     = null
-  description = "The endpoint of an existing Cloud Object Storage bucket to use for storing the IBM Cloud Logs data. If an existing Cloud Object Storage bucket is not specified, a bucket is created."
-}
-
-variable "cloud_log_data_bucket_class" {
-  type        = string
-  default     = "smart"
-  description = "The storage class of the newly provisioned cloud logs Cloud Object Storage bucket. Specify one of the following values for the storage class: `standard`, `vault`, `cold`, `smart` (default), or `onerate_active`."
-  validation {
-    condition     = contains(["standard", "vault", "cold", "smart", "onerate_active"], var.cloud_log_data_bucket_class)
-    error_message = "Specify one of the following values for the `cos_bucket_class`:  `standard`, `vault`, `cold`, `smart`, or `onerate_active`."
-  }
-}
-
-variable "cloud_log_data_bucket_access_tag" {
-  type        = list(string)
-  default     = []
-  description = "A list of optional tags to add to the cloud log data object storage bucket."
-}
-
-variable "cloud_log_metrics_bucket_name" {
-  type        = string
-  default     = "cloud-logs-metrics-bucket"
-  description = "The name of the Cloud Object Storage bucket to create to store cloud logs metrics. Cloud Object Storage bucket names are globally unique. If the `add_bucket_name_suffix` variable is set to `true`, 4 random characters are added to this name to ensure that the name of the bucket is globally unique. If the prefix input variable is passed, the name of the bucket is prefixed to the value in the `<prefix>-value` format."
-}
-
-variable "existing_cloud_logs_metrics_bucket_crn" {
-  type        = string
-  nullable    = true
-  default     = null
-  description = "The crn of an existing bucket within the Cloud Object Storage instance to store IBM Cloud Logs metrics. If an existing Cloud Object Storage bucket is not specified, a bucket is created."
-}
-
-variable "existing_cloud_logs_metrics_bucket_endpoint" {
-  type        = string
-  nullable    = true
-  default     = null
-  description = "The endpoint of an existing Cloud Object Storage bucket to use for storing the IBM Cloud Logs metrics. If an existing Cloud Object Storage bucket is not specified, a bucket is created."
-}
-
-variable "cloud_log_metrics_bucket_class" {
-  type        = string
-  default     = "smart"
-  description = "The storage class of the newly provisioned cloud logs Cloud Object Storage bucket. Specify one of the following values for the storage class: `standard`, `vault`, `cold`, `smart` (default), or `onerate_active`."
-  validation {
-    condition     = contains(["standard", "vault", "cold", "smart", "onerate_active"], var.cloud_log_metrics_bucket_class)
-    error_message = "Specify one of the following values for the `cos_bucket_class`:  `standard`, `vault`, `cold`, `smart`, or `onerate_active`."
-  }
-}
-
-variable "cloud_log_metrics_bucket_access_tag" {
-  type        = list(string)
-  default     = []
-  description = "A list of optional tags to add to the cloud log metrics object storage bucket."
-}
-
 variable "skip_logs_routing_auth_policy" {
   description = "Whether to create an IAM authorization policy that permits Logs Routing Sender access to the IBM Cloud Logs."
   type        = bool
@@ -378,6 +306,28 @@ variable "cos_instance_tags" {
   default     = []
 }
 
+variable "at_cos_bucket_retention_policy" {
+  type = object({
+    default   = optional(number, 90)
+    maximum   = optional(number, 350)
+    minimum   = optional(number, 90)
+    permanent = optional(bool, false)
+  })
+  description = "The retention policy of the IBM Cloud Activity Tracker Event Routing COS target bucket. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-observability-da/blob/main/solutions/instances/DA-types.md#at_cos_bucket_retention_policy-)"
+  default     = null
+}
+
+variable "cloud_log_data_bucket_retention_policy" {
+  type = object({
+    default   = optional(number, 90)
+    maximum   = optional(number, 350)
+    minimum   = optional(number, 90)
+    permanent = optional(bool, false)
+  })
+  description = "The retention policy of the IBM Cloud Logs data bucket.[Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-observability-da/blob/main/solutions/instances/DA-types.md#cloud_log_data_bucket_retention_policy-)"
+  default     = null
+}
+
 variable "cos_instance_access_tags" {
   type        = list(string)
   description = "A list of access tags to apply to a new Cloud Object Storage instance."
@@ -396,6 +346,18 @@ variable "at_cos_target_bucket_name" {
   description = "The name of the Cloud Object Storage bucket to create for the Cloud Object Storage target to store AT events. Cloud Object Storage bucket names are globally unique. If the `add_bucket_name_suffix` variable is set to `true`, 4 random characters are added to this name to ensure that the name of the bucket is globally unique. If the prefix input variable is passed, the name of the instance is prefixed to the value in the `<prefix>-value` format."
 }
 
+variable "cloud_log_data_bucket_name" {
+  type        = string
+  default     = "cloud-logs-data-bucket"
+  description = "The name of the Cloud Object Storage bucket to create to store cloud log data. Cloud Object Storage bucket names are globally unique. If the `add_bucket_name_suffix` variable is set to `true`, 4 random characters are added to this name to ensure that the name of the bucket is globally unique. If the prefix input variable is passed, the name of the bucket is prefixed to the value in the `<prefix>-value` format."
+}
+
+variable "cloud_log_metrics_bucket_name" {
+  type        = string
+  default     = "cloud-logs-metrics-bucket"
+  description = "The name of the Cloud Object Storage bucket to create to store cloud logs metrics. Cloud Object Storage bucket names are globally unique. If the `add_bucket_name_suffix` variable is set to `true`, 4 random characters are added to this name to ensure that the name of the bucket is globally unique. If the prefix input variable is passed, the name of the bucket is prefixed to the value in the `<prefix>-value` format."
+}
+
 variable "archive_bucket_access_tags" {
   type        = list(string)
   default     = []
@@ -408,6 +370,18 @@ variable "at_cos_bucket_access_tags" {
   description = "A list of optional access tags to add to the IBM Cloud Activity Tracker Event Routing Cloud Object Storage bucket."
 }
 
+variable "cloud_log_data_bucket_access_tag" {
+  type        = list(string)
+  default     = []
+  description = "A list of optional tags to add to the cloud log data object storage bucket."
+}
+
+variable "cloud_log_metrics_bucket_access_tag" {
+  type        = list(string)
+  default     = []
+  description = "A list of optional tags to add to the cloud log metrics object storage bucket."
+}
+
 variable "log_archive_cos_bucket_class" {
   type        = string
   default     = "smart"
@@ -428,13 +402,47 @@ variable "at_cos_target_bucket_class" {
   }
 }
 
+variable "cloud_log_data_bucket_class" {
+  type        = string
+  default     = "smart"
+  description = "The storage class of the newly provisioned cloud logs Cloud Object Storage bucket. Specify one of the following values for the storage class: `standard` or `smart` (default)."
+  validation {
+    condition     = contains(["standard", "smart"], var.cloud_log_data_bucket_class)
+    error_message = "Specify one of the following values for the `cos_bucket_class`:  `standard` or `smart`. See more at https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-configure-data-bucket"
+  }
+}
+
+variable "cloud_log_metrics_bucket_class" {
+  type        = string
+  default     = "smart"
+  description = "The storage class of the newly provisioned cloud logs Cloud Object Storage bucket. Specify one of the following values for the storage class: `standard` or `smart` (default)."
+  validation {
+    condition     = contains(["standard", "smart"], var.cloud_log_metrics_bucket_class)
+    error_message = "Specify one of the following values for the `cos_bucket_class`:  `standard`, or `smart`. See more at https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-configure-data-bucket"
+  }
+}
+
 variable "existing_cos_instance_crn" {
   type        = string
   nullable    = true
   default     = null
   description = "The CRN of an existing Cloud Object Storage instance. If a CRN is not specified, a new instance of Cloud Object Storage is created."
 }
 
+variable "existing_cloud_logs_data_bucket_crn" {
+  type        = string
+  nullable    = true
+  default     = null
+  description = "The crn of an existing bucket within the Cloud Object Storage instance to store IBM Cloud Logs data. If an existing Cloud Object Storage bucket is not specified, a bucket is created."
+}
+
+variable "existing_cloud_logs_metrics_bucket_crn" {
+  type        = string
+  nullable    = true
+  default     = null
+  description = "The crn of an existing bucket within the Cloud Object Storage instance to store IBM Cloud Logs metrics. If an existing Cloud Object Storage bucket is not specified, a bucket is created."
+}
+
 variable "existing_at_cos_target_bucket_name" {
   type        = string
   nullable    = true
@@ -455,6 +463,20 @@ variable "skip_cos_kms_auth_policy" {
   default     = false
 }
 
+variable "existing_cloud_logs_data_bucket_endpoint" {
+  type        = string
+  nullable    = true
+  default     = null
+  description = "The endpoint of an existing Cloud Object Storage bucket to use for storing the IBM Cloud Logs data. If an existing Cloud Object Storage bucket is not specified, a bucket is created."
+}
+
+variable "existing_cloud_logs_metrics_bucket_endpoint" {
+  type        = string
+  nullable    = true
+  default     = null
+  description = "The endpoint of an existing Cloud Object Storage bucket to use for storing the IBM Cloud Logs metrics. If an existing Cloud Object Storage bucket is not specified, a bucket is created."
+}
+
 variable "skip_cloud_logs_cos_auth_policy" {
   type        = bool
   description = "To skip creating an IAM authorization policy that allows the IBM Cloud logs to write to the Cloud Object Storage bucket, set this variable to `true`."