DL-5 grant data and insight department permission to use cloudtrail bucket (#2362)

* DL-5 grant data and insight department permission to use cloudtrail bucket

* grant the bucket permission to airflow instead of ecs role

* make the cloudtrail access become part of the s3 department access

* remove the cloudtrail from airflow policy which is included in the s3_access

Author: Tian-2017

terraform/core/05-departments.tf

@@ -139,6 +139,7 @@ module "department_data_and_insight" {
   departmental_airflow_user       = true
   mwaa_etl_scripts_bucket_arn     = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
   mwaa_key_arn                    = aws_kms_key.mwaa_key.arn
+  cloudtrail_bucket               = module.cloudtrail_storage
 }
 
 module "department_env_enforcement" {

terraform/modules/department/02-inputs-optional.tf

@@ -58,3 +58,12 @@ variable "additional_s3_access" {
   default = []
 }
 
+variable "cloudtrail_bucket" {
+  description = "CloudTrail storage S3 bucket"
+  type = object({
+    bucket_id   = string
+    bucket_arn  = string
+    kms_key_arn = string
+  })
+  default = null
+}

terraform/modules/department/50-aws-iam-policies.tf

@@ -3,6 +3,11 @@
 
 // S3 read only access policy
 data "aws_iam_policy_document" "read_only_s3_department_access" {
+  # Include CloudTrail bucket access for data-and-insight department
+  source_policy_documents = local.department_identifier == "data-and-insight" && var.cloudtrail_bucket != null ? [
+    data.aws_iam_policy_document.cloudtrail_access[0].json
+  ] : []
+
   statement {
     sid    = "ListAllS3AndKmsKeys"
     effect = "Allow"
@@ -32,6 +37,8 @@ data "aws_iam_policy_document" "read_only_s3_department_access" {
     ]
   }
 
+
+
   dynamic "statement" {
     for_each = var.additional_s3_access
     iterator = additional_access_item
@@ -90,6 +97,8 @@ data "aws_iam_policy_document" "read_only_s3_department_access" {
     ]
   }
 
+
+
   dynamic "statement" {
     for_each = var.additional_s3_access
     iterator = additional_access_item
@@ -180,6 +189,11 @@ resource "aws_iam_policy" "read_only_glue_access" {
 
 // Full departmental S3 access policy
 data "aws_iam_policy_document" "s3_department_access" {
+  # Include CloudTrail bucket access for data-and-insight department
+  source_policy_documents = local.department_identifier == "data-and-insight" && var.cloudtrail_bucket != null ? [
+    data.aws_iam_policy_document.cloudtrail_access[0].json
+  ] : []
+
   statement {
     sid    = "ListAllS3AndKmsKeys"
     effect = "Allow"
@@ -215,6 +229,8 @@ data "aws_iam_policy_document" "s3_department_access" {
     ]
   }
 
+
+
   dynamic "statement" {
     for_each = var.additional_s3_access
     iterator = additional_access_item
@@ -277,10 +293,12 @@ data "aws_iam_policy_document" "s3_department_access" {
       var.mwaa_etl_scripts_bucket_arn,
       "${var.mwaa_etl_scripts_bucket_arn}/${replace(local.department_identifier, "-", "_")}/*",
       "${var.mwaa_etl_scripts_bucket_arn}/unrestricted/*",
-      "${var.mwaa_etl_scripts_bucket_arn}/shared/*",
+      "${var.mwaa_etl_scripts_bucket_arn}/shared/*"
     ]
   }
 
+
+
   dynamic "statement" {
     for_each = var.additional_s3_access
     iterator = additional_access_item
@@ -1079,3 +1097,40 @@ resource "aws_iam_policy" "mtfh_access_policy" {
   description = "Allows ${local.department_identifier} department access for ecs tasks to mtfh/ subdirectory in landing zone"
   policy      = data.aws_iam_policy_document.mtfh_access[0].json
 }
+
+// Read-only CloudTrail access for Data and Insight department only
+data "aws_iam_policy_document" "cloudtrail_access" {
+  count = local.department_identifier == "data-and-insight" && var.cloudtrail_bucket != null ? 1 : 0
+
+  statement {
+    sid    = "CloudTrailKmsReadAccess"
+    effect = "Allow"
+    actions = [
+      "kms:Decrypt",
+      "kms:GenerateDataKey*",
+      "kms:DescribeKey"
+    ]
+    resources = [var.cloudtrail_bucket.kms_key_arn]
+  }
+
+  statement {
+    sid    = "CloudTrailS3ReadAccess"
+    effect = "Allow"
+    actions = [
+      "s3:GetObject",
+      "s3:GetObjectVersion",
+      "s3:ListBucket"
+    ]
+    resources = [
+      var.cloudtrail_bucket.bucket_arn,
+      "${var.cloudtrail_bucket.bucket_arn}/*"
+    ]
+  }
+}
+
+resource "aws_iam_policy" "cloudtrail_access_policy" {
+  count       = local.department_identifier == "data-and-insight" && var.cloudtrail_bucket != null ? 1 : 0
+  name        = lower("${var.identifier_prefix}-${local.department_identifier}-cloudtrail-access-policy")
+  description = "Allows ${local.department_identifier} department read-only access to CloudTrail bucket"
+  policy      = data.aws_iam_policy_document.cloudtrail_access[0].json
+}

terraform/modules/department/50-aws-iam-roles.tf

@@ -96,7 +96,7 @@ resource "aws_iam_role_policy_attachment" "glue_runner_pass_role_to_glue_for_not
   policy_arn = aws_iam_policy.glue_runner_pass_role_to_glue_for_notebook_use.arn
 }
 
-# Define a map for the departmentalairflow policies
+# Define a map for the departmental airflow policies
 locals {
   airflow_policy_map = {
     s3_access                 = aws_iam_policy.s3_access.arn,