Merge pull request #5 from RohitSquareops/patch-1

Log bucket can now have CW logs and added retention period on bucket

Author: RohitSquareops

README.md

@@ -11,12 +11,17 @@ Terraform module to create Remote State Storage resources for workload deploymen
 
 ```hcl
 module "backend" {
-  source = "squareops/tfstate/aws"
-  logging                                         = true
-  environment                                     = "Production"
-  bucket_name                                     = "tfstate"
-  force_destroy                                   = true
-  versioning_enabled                              = true
+  source                       = "squareops/tfstate/aws"
+  logging                      = true
+  bucket_name                  = "production-tfstate-bucket" #unique global s3 bucket name
+  environment                  = "prod"
+  force_destroy                = true
+  versioning_enabled           = true
+  cloudwatch_logging_enabled   = true
+  log_retention_in_days        = 90
+  log_bucket_lifecycle_enabled = true
+  s3_ia_retention_in_days      = 90
+  s3_galcier_retention_in_days = 180
 }
 
 ```
@@ -30,6 +35,9 @@ Terraform state locking is a mechanism used to prevent multiple users from simul
 
 An Amazon S3 bucket and a DynamoDB table can be used as a remote backend to store and manage the Terraform state file, and also to implement state locking. The S3 bucket is used to store the state file, while the DynamoDB table is used to store the lock information, such as who acquired the lock and when. Terraform will check the lock state in the DynamoDB table before making changes to the state file in the S3 bucket, and will wait or retry if the lock is already acquired by another instance. This provides a centralized and durable mechanism for managing the Terraform state and ensuring that changes are made in a controlled and safe manner.
 
+Additionally, you may have a log bucket configured to store CloudTrail and CloudWatch logs. This log bucket can have a bucket lifecycle policy in place to automatically manage the lifecycle of log data. For example, log data can be transitioned to Amazon S3 Glacier for long-term storage after a certain period, and eventually to Amazon S3 Infrequent Access storage. This helps in optimizing storage costs and ensures that log data is retained according to your organization's compliance requirements.
+
+
 ## Security & Compliance [<img src="	https://prowler.pro/wp-content/themes/prowler-pro/assets/img/logo.svg" width="250" align="right" />](https://prowler.pro/)
 
 Security scanning is graciously provided by Prowler. Proowler is the leading fully hosted, cloud-native solution providing continuous cluster security and compliance.
@@ -85,17 +93,23 @@ In this module, we have implemented the following CIS Compliance checks for S3:
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_bucket_name"></a> [bucket\_name](#input\_bucket\_name) | Name of the S3 bucket to be created. | `string` | `""` | no |
-| <a name="input_environment"></a> [environment](#input\_environment) | Specify the type of environment(dev, demo, prod) in which the S3 bucket will be created. | `string` | `"demo"` | no |
+| <a name="input_cloudwatch_logging_enabled"></a> [cloudwatch\_logging\_enabled](#input\_cloudwatch\_logging\_enabled) | Enable or disable CloudWatch log group logging. | `bool` | `true` | no |
+| <a name="input_environment"></a> [environment](#input\_environment) | Specify the type of environment(dev, demo, prod) in which the S3 bucket will be created. | `string` | `""` | no |
 | <a name="input_force_destroy"></a> [force\_destroy](#input\_force\_destroy) | Whether or not to delete all objects from the bucket to allow for destruction of the bucket without error. | `bool` | `false` | no |
+| <a name="input_log_bucket_lifecycle_enabled"></a> [log\_bucket\_lifecycle\_enabled](#input\_log\_bucket\_lifecycle\_enabled) | Enable or disable the S3 bucket's lifecycle rule for log data. | `bool` | `true` | no |
+| <a name="input_log_retention_in_days"></a> [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | Retention period (in days) for CloudWatch log groups. | `number` | `90` | no |
 | <a name="input_logging"></a> [logging](#input\_logging) | Configuration for S3 bucket access logging. | `bool` | `true` | no |
+| <a name="input_s3_galcier_retention_in_days"></a> [s3\_galcier\_retention\_in\_days](#input\_s3\_galcier\_retention\_in\_days) | Retention period (in days) for moving S3 log data to Glacier storage. | `number` | `180` | no |
+| <a name="input_s3_ia_retention_in_days"></a> [s3\_ia\_retention\_in\_days](#input\_s3\_ia\_retention\_in\_days) | Retention period (in days) for moving S3 log data to Infrequent Access storage. | `number` | `90` | no |
 | <a name="input_versioning_enabled"></a> [versioning\_enabled](#input\_versioning\_enabled) | Whether or not to enable versioning for the S3 bucket, which allows multiple versions of an object to be stored in the same bucket. | `bool` | `false` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| <a name="output_dynamodb_table_name"></a> [dynamodb\_table\_name](#output\_dynamodb\_table\_name) | Name of the DynamoDB table that will be used to manage locking and unlocking of the Terraform state file. |
-| <a name="output_log_bucket_name"></a> [log\_bucket\_name](#output\_log\_bucket\_name) | Name of the S3 bucket that will be used to store logs for this module. |
+| <a name="output_dynamodb_table_name"></a> [dynamodb\_table\_name](#output\_dynamodb\_table\_name) | Name of the DynamoDB table that will be used to manage locking and unlocking of the terraform state file. |
+| <a name="output_log_bucket_name"></a> [log\_bucket\_name](#output\_log\_bucket\_name) | Name of the S3 bucket that will be used to store logs. |
+| <a name="output_region"></a> [region](#output\_region) | Name of the region in which Cloudtrail is created |
 | <a name="output_state_bucket_name"></a> [state\_bucket\_name](#output\_state\_bucket\_name) | Specify the region in which an S3 bucket will be created by the module. |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 

examples/state-storage-backend/main.tf

@@ -9,10 +9,15 @@ locals {
 }
 
 module "backend" {
-  source             = "squareops/tfstate/aws"
-  logging            = true
-  environment        = local.environment
-  bucket_name        = "production-tfstate-bucket" #unique global s3 bucket name
-  force_destroy      = true
-  versioning_enabled = true
+  source                       = "squareops/tfstate/aws"
+  logging                      = true
+  bucket_name                  = "production-tfstate-bucket" #unique global s3 bucket name
+  environment                  = local.environment
+  force_destroy                = true
+  versioning_enabled           = true
+  cloudwatch_logging_enabled   = true
+  log_retention_in_days        = 90
+  log_bucket_lifecycle_enabled = true
+  s3_ia_retention_in_days      = 90
+  s3_galcier_retention_in_days = 180
 }

examples/state-storage-backend/outputs.tf

@@ -1,6 +1,6 @@
 output "bucket_region" {
   description = "Specify the region in which an S3 bucket will be created by the module."
-  value       = local.region
+  value       = module.backend.region
 }
 
 output "state_bucket_name" {

logging.tf

@@ -7,8 +7,8 @@ resource "aws_cloudtrail" "s3_cloudtrail" {
   include_global_service_events = false
   enable_logging                = true
   enable_log_file_validation    = true
-  cloud_watch_logs_role_arn     = aws_iam_role.s3_cloudtrail_cloudwatch_role[0].arn
-  cloud_watch_logs_group_arn    = "${aws_cloudwatch_log_group.s3_cloudwatch[0].arn}:*"
+  cloud_watch_logs_role_arn     = var.cloudwatch_logging_enabled ? aws_iam_role.s3_cloudtrail_cloudwatch_role[0].arn : null
+  cloud_watch_logs_group_arn    = var.cloudwatch_logging_enabled ? "${aws_cloudwatch_log_group.s3_cloudwatch[0].arn}:*" : null
   kms_key_id                    = module.kms_key[0].key_arn
   event_selector {
     read_write_type           = "All"
@@ -17,6 +17,10 @@ resource "aws_cloudtrail" "s3_cloudtrail" {
       type   = "AWS::S3::Object"
       values = ["arn:aws:s3"]
     }
+    data_resource {
+      type   = "AWS::Lambda::Function"
+      values = ["arn:aws:lambda"]
+    }
   }
   tags = merge(
     { "Name" = format("%s-%s-S3", var.bucket_name, data.aws_caller_identity.current.account_id) },
@@ -25,20 +29,18 @@ resource "aws_cloudtrail" "s3_cloudtrail" {
 }
 
 resource "aws_cloudwatch_log_group" "s3_cloudwatch" {
-  count      = var.logging ? 1 : 0
-  name       = format("%s-%s-S3", var.bucket_name, data.aws_caller_identity.current.account_id)
-  kms_key_id = module.kms_key[0].key_arn
-  provisioner "local-exec" {
-    command = "sleep 10"
-  }
+  count             = var.logging && var.cloudwatch_logging_enabled ? 1 : 0
+  name              = format("%s-%s-S3", var.bucket_name, data.aws_caller_identity.current.account_id)
+  kms_key_id        = module.kms_key[0].key_arn
+  retention_in_days = var.log_retention_in_days
   tags = merge(
     { "Name" = format("%s-%s-S3", var.bucket_name, data.aws_caller_identity.current.account_id) },
     local.tags,
   )
 }
 
 resource "aws_iam_role" "s3_cloudtrail_cloudwatch_role" {
-  count              = var.logging ? 1 : 0
+  count              = var.logging && var.cloudwatch_logging_enabled ? 1 : 0
   name               = format("%s-cloudtrail-cloudwatch-S3", var.bucket_name)
   assume_role_policy = data.aws_iam_policy_document.cloudtrail_assume_role[0].json
   tags = merge(
@@ -61,7 +63,7 @@ data "aws_iam_policy_document" "cloudtrail_assume_role" {
 }
 
 resource "aws_iam_policy" "s3_cloudtrail_cloudwatch_policy" {
-  count  = var.logging ? 1 : 0
+  count  = var.logging && var.cloudwatch_logging_enabled ? 1 : 0
   name   = format("%s-cloudtrail-cloudwatch-S3", var.bucket_name)
   policy = <<EOF
 {
@@ -99,14 +101,11 @@ EOF
 
 
 resource "aws_iam_role_policy_attachment" "s3_cloudtrail_policy_attachment" {
-  count      = var.logging ? 1 : 0
+  count      = var.logging && var.cloudwatch_logging_enabled ? 1 : 0
   role       = aws_iam_role.s3_cloudtrail_cloudwatch_role[0].name
   policy_arn = aws_iam_policy.s3_cloudtrail_cloudwatch_policy[0].arn
 }
 
-
-
-
 module "log_bucket" {
   count                                 = var.logging ? 1 : 0
   source                                = "terraform-aws-modules/s3-bucket/aws"
@@ -116,13 +115,32 @@ module "log_bucket" {
   attach_elb_log_delivery_policy        = true
   attach_lb_log_delivery_policy         = true
   attach_deny_insecure_transport_policy = true
+  versioning = {
+    enabled = var.versioning_enabled
+  }
   # S3 bucket-level Public Access Block configuration
   block_public_acls       = true
   block_public_policy     = true
   ignore_public_acls      = true
   restrict_public_buckets = true
-  attach_policy           = true
-  policy                  = <<POLICY
+  lifecycle_rule = [
+    {
+      id      = "log"
+      enabled = var.log_bucket_lifecycle_enabled
+
+      transition = [
+        {
+          days          = var.s3_ia_retention_in_days
+          storage_class = "ONEZONE_IA"
+          }, {
+          days          = var.s3_galcier_retention_in_days
+          storage_class = "GLACIER"
+        }
+      ]
+    }
+  ]
+  attach_policy = true
+  policy        = <<POLICY
 {
     "Version": "2012-10-17",
     "Statement": [
@@ -189,7 +207,7 @@ data "aws_iam_policy_document" "default" {
     condition {
       test     = "StringLike"
       variable = "kms:EncryptionContext:aws:cloudtrail:arn"
-      values   = ["arn:aws:cloudtrail:*:XXXXXXXXXXXX:trail/*"]
+      values   = ["arn:aws:cloudtrail:*:${data.aws_caller_identity.current.account_id}:trail/*"]
     }
   }
 
@@ -220,12 +238,12 @@ data "aws_iam_policy_document" "default" {
       test     = "StringEquals"
       variable = "kms:CallerAccount"
       values = [
-      "XXXXXXXXXXXX"]
+      "${data.aws_caller_identity.current.account_id}"]
     }
     condition {
       test     = "StringLike"
       variable = "kms:EncryptionContext:aws:cloudtrail:arn"
-      values   = ["arn:aws:cloudtrail:*:XXXXXXXXXXXX:trail/*"]
+      values   = ["arn:aws:cloudtrail:*:${data.aws_caller_identity.current.account_id}:trail/*"]
     }
   }
 

outputs.tf

@@ -4,11 +4,16 @@ output "state_bucket_name" {
 }
 
 output "dynamodb_table_name" {
-  description = "Name of the DynamoDB table that will be used to manage locking and unlocking of the Terraform state file."
+  description = "Name of the DynamoDB table that will be used to manage locking and unlocking of the terraform state file."
   value       = aws_dynamodb_table.dynamodb_table.id
 }
 
 output "log_bucket_name" {
-  description = "Name of the S3 bucket that will be used to store logs for this module."
+  description = "Name of the S3 bucket that will be used to store logs."
   value       = var.logging ? module.log_bucket[0].s3_bucket_id : null
 }
+
+output "region" {
+  description = "Name of the region in which Cloudtrail is created"
+  value       = var.logging ? aws_cloudtrail.s3_cloudtrail[0].home_region : null
+}

variables.tf

@@ -24,6 +24,36 @@ variable "logging" {
 
 variable "environment" {
   description = "Specify the type of environment(dev, demo, prod) in which the S3 bucket will be created. "
-  default     = "demo"
+  default     = ""
   type        = string
 }
+
+variable "cloudwatch_logging_enabled" {
+  description = "Enable or disable CloudWatch log group logging."
+  default     = true
+  type        = bool
+}
+
+variable "log_retention_in_days" {
+  description = "Retention period (in days) for CloudWatch log groups."
+  default     = 90
+  type        = number
+}
+
+variable "s3_galcier_retention_in_days" {
+  description = "Retention period (in days) for moving S3 log data to Glacier storage."
+  default     = 180
+  type        = number
+}
+
+variable "s3_ia_retention_in_days" {
+  description = "Retention period (in days) for moving S3 log data to Infrequent Access storage."
+  default     = 90
+  type        = number
+}
+
+variable "log_bucket_lifecycle_enabled" {
+  description = "Enable or disable the S3 bucket's lifecycle rule for log data."
+  default     = true
+  type        = bool
+}