Allow-all-deparments-to-create-catalog-tables-via-the-manual-bucket (#2583)

* allow all departments to create catalog tables via the manual bucket

* correct the parking database module name and add the bucket and kms key arn to redshift module

* add revenues

Author: Tian-2017

terraform/etl/42-redshift.tf

@@ -10,10 +10,12 @@ module "redshift" {
   refined_zone_bucket_arn  = module.refined_zone_data_source.bucket_arn
   trusted_zone_bucket_arn  = module.trusted_zone_data_source.bucket_arn
   raw_zone_bucket_arn      = module.raw_zone_data_source.bucket_arn
+  user_uploads_bucket_arn  = module.user_uploads_data_source.bucket_arn
   landing_zone_kms_key_arn = module.landing_zone_data_source.kms_key_arn
   raw_zone_kms_key_arn     = module.raw_zone_data_source.kms_key_arn
   refined_zone_kms_key_arn = module.refined_zone_data_source.kms_key_arn
   trusted_zone_kms_key_arn = module.trusted_zone_data_source.kms_key_arn
+  user_uploads_kms_key_arn = module.user_uploads_data_source.kms_key_arn
   secrets_manager_key      = data.aws_kms_key.secrets_manager_key.arn
   additional_iam_roles     = local.is_production_environment ? [] : [aws_iam_role.parking_redshift_copier[0].arn]
 }
@@ -48,7 +50,7 @@ locals {
     replace(module.department_parking_data_source.refined_zone_catalog_database_name, "-", "_") = module.department_parking_data_source.refined_zone_catalog_database_name,
     replace(module.department_parking_data_source.trusted_zone_catalog_database_name, "-", "_") = module.department_parking_data_source.trusted_zone_catalog_database_name,
     replace("parking-ringgo-sftp-raw-zone", "-", "_")                                           = "parking-ringgo-sftp-raw-zone",
-    replace(aws_glue_catalog_database.parking_user_uploads.name, "-", "_")                      = aws_glue_catalog_database.parking_user_uploads.name,
+    replace(aws_glue_catalog_database.department_user_uploads["parking"].name, "-", "_")        = aws_glue_catalog_database.department_user_uploads["parking"].name,
 
     replace(module.department_finance_data_source.raw_zone_catalog_database_name, "-", "_")     = module.department_finance_data_source.raw_zone_catalog_database_name,
     replace(module.department_finance_data_source.refined_zone_catalog_database_name, "-", "_") = module.department_finance_data_source.refined_zone_catalog_database_name,
@@ -127,7 +129,7 @@ locals {
         "liberator_refined_zone",
         replace(module.department_parking_data_source.trusted_zone_catalog_database_name, "-", "_"),
         replace("parking-ringgo-sftp-raw-zone", "-", "_"),
-        replace(aws_glue_catalog_database.parking_user_uploads.name, "-", "_"),
+        replace(aws_glue_catalog_database.department_user_uploads["parking"].name, "-", "_"),
       ], local.unrestricted_schemas)
     },
     {
@@ -311,7 +313,7 @@ locals {
         "liberator_refined_zone",
         replace(module.department_parking_data_source.trusted_zone_catalog_database_name, "-", "_"),
         replace("parking-ringgo-sftp-raw-zone", "-", "_"),
-        replace(aws_glue_catalog_database.parking_user_uploads.name, "-", "_")
+        replace(aws_glue_catalog_database.department_user_uploads["parking"].name, "-", "_")
       ]
       roles_to_inherit_permissions_from = [
         local.unrestricted_data_role_name

terraform/etl/61-aws-glue-catalog-database.tf

@@ -86,8 +86,22 @@ resource "aws_glue_catalog_database" "arcus_archive" {
   }
 }
 
-resource "aws_glue_catalog_database" "parking_user_uploads" {
-  name = "parking_user_uploads_db"
+locals {
+  department_user_uploads_databases = {
+    parking            = "parking_user_uploads_db"
+    housing            = "housing_user_uploads_db"
+    data_and_insight   = "data_and_insight_user_uploads_db"
+    child_fam_services = "child_fam_services_user_uploads_db"
+    unrestricted       = "unrestricted_user_uploads_db"
+    env_services       = "env_services_user_uploads_db"
+    revenues           = "revenues_user_uploads_db"
+  }
+}
+
+resource "aws_glue_catalog_database" "department_user_uploads" {
+  for_each = local.department_user_uploads_databases
+
+  name = each.value
 
   lifecycle {
     prevent_destroy = true

terraform/etl/62-lambda-csv-to-glue-catalog.tf

@@ -1,6 +1,18 @@
 # Lambda function to automatically create/delete Glue Catalog tables
 # Workflow: S3 CSV upload/delete → SQS → Lambda → Glue Catalog table create/delete (retry once on failure → DLQ)
 
+locals {
+  department_user_uploads_prefixes = {
+    parking            = "parking/"
+    housing            = "housing/"
+    data_and_insight   = "data-and-insight/"
+    child_fam_services = "child-fam-services/"
+    unrestricted       = "unrestricted/"
+    env_services       = "env-services/"
+    revenues           = "revenues/"
+  }
+}
+
 data "aws_iam_policy_document" "csv_to_glue_catalog_lambda_assume_role" {
   statement {
     actions = ["sts:AssumeRole"]
@@ -34,12 +46,11 @@ data "aws_iam_policy_document" "csv_to_glue_catalog_lambda_execution" {
       "glue:GetPartitions",
       "glue:DeletePartition",
     ]
-    # Currently only scoped to parking
-    resources = [
-      "arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.data_platform.account_id}:catalog",
-      "arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.data_platform.account_id}:database/parking_user_uploads_db",
-      "arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.data_platform.account_id}:table/parking_user_uploads_db/*",
-    ]
+    resources = concat(
+      ["arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.data_platform.account_id}:catalog"],
+      [for db_name in values(local.department_user_uploads_databases) : "arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.data_platform.account_id}:database/${db_name}"],
+      [for db_name in values(local.department_user_uploads_databases) : "arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.data_platform.account_id}:table/${db_name}/*"]
+    )
   }
 
   statement {
@@ -177,11 +188,14 @@ resource "aws_sqs_queue_policy" "csv_to_glue_catalog_events" {
 resource "aws_s3_bucket_notification" "user_uploads_csv_notification" {
   bucket = module.user_uploads_data_source.bucket_id
 
-  queue {
-    queue_arn     = aws_sqs_queue.csv_to_glue_catalog_events.arn
-    events        = ["s3:ObjectCreated:*", "s3:ObjectRemoved:*"]
-    filter_prefix = "parking/" # Currently only scoped to parking
-    filter_suffix = ".csv"
+  dynamic "queue" {
+    for_each = local.department_user_uploads_prefixes
+    content {
+      queue_arn     = aws_sqs_queue.csv_to_glue_catalog_events.arn
+      events        = ["s3:ObjectCreated:*", "s3:ObjectRemoved:*"]
+      filter_prefix = queue.value
+      filter_suffix = ".csv"
+    }
   }
 
   depends_on = [aws_sqs_queue_policy.csv_to_glue_catalog_events]

terraform/modules/redshift/01-inputs-required.tf

@@ -62,3 +62,13 @@ variable "secrets_manager_key" {
   description = "ARN of secrets manager KMS key"
   type        = string
 }
+
+variable "user_uploads_bucket_arn" {
+  description = "ARN of user uploads bucket"
+  type        = string
+}
+
+variable "user_uploads_kms_key_arn" {
+  description = "ARN of user uploads KMS key"
+  type        = string
+}

terraform/modules/redshift/10-redshift.tf

@@ -36,7 +36,9 @@ data "aws_iam_policy_document" "redshift" {
       "${var.trusted_zone_bucket_arn}/*",
       var.trusted_zone_bucket_arn,
       "${var.raw_zone_bucket_arn}/*",
-      var.raw_zone_bucket_arn
+      var.raw_zone_bucket_arn,
+      "${var.user_uploads_bucket_arn}/*",
+      var.user_uploads_bucket_arn
     ]
   }
   statement {
@@ -55,6 +57,7 @@ data "aws_iam_policy_document" "redshift" {
       var.raw_zone_kms_key_arn,
       var.refined_zone_kms_key_arn,
       var.trusted_zone_kms_key_arn,
+      var.user_uploads_kms_key_arn,
     ]
   }
 }
@@ -165,11 +168,11 @@ resource "aws_security_group" "redshift_cluster_security_group" {
   }
 
   ingress {
-    description     = "Allows inbound traffic from the Qlik EC2 data gateway"
-    from_port       = 5439
-    to_port         = 5439
-    protocol        = "tcp"
-    cidr_blocks     = ["10.120.32.49/32"]
+    description = "Allows inbound traffic from the Qlik EC2 data gateway"
+    from_port   = 5439
+    to_port     = 5439
+    protocol    = "tcp"
+    cidr_blocks = ["10.120.32.49/32"]
   }
   ingress {
     description     = "Allows security group based inbound traffic"