allow all departments to create catalog tables via the manual bucket

Author: Tian-2017

terraform/etl/61-aws-glue-catalog-database.tf

@@ -86,8 +86,21 @@ resource "aws_glue_catalog_database" "arcus_archive" {
   }
 }
 
-resource "aws_glue_catalog_database" "parking_user_uploads" {
-  name = "parking_user_uploads_db"
+locals {
+  department_user_uploads_databases = {
+    parking            = "parking_user_uploads_db"
+    housing            = "housing_user_uploads_db"
+    data_and_insight   = "data_and_insight_user_uploads_db"
+    child_fam_services = "child_fam_services_user_uploads_db"
+    unrestricted       = "unrestricted_user_uploads_db"
+    env_services       = "env_services_user_uploads_db"
+  }
+}
+
+resource "aws_glue_catalog_database" "department_user_uploads" {
+  for_each = local.department_user_uploads_databases
+
+  name = each.value
 
   lifecycle {
     prevent_destroy = true

terraform/etl/62-lambda-csv-to-glue-catalog.tf

@@ -1,6 +1,17 @@
 # Lambda function to automatically create/delete Glue Catalog tables
 # Workflow: S3 CSV upload/delete → SQS → Lambda → Glue Catalog table create/delete (retry once on failure → DLQ)
 
+locals {
+  department_user_uploads_prefixes = {
+    parking            = "parking/"
+    housing            = "housing/"
+    data_and_insight   = "data-and-insight/"
+    child_fam_services = "child-fam-services/"
+    unrestricted       = "unrestricted/"
+    env_services       = "env-services/"
+  }
+}
+
 data "aws_iam_policy_document" "csv_to_glue_catalog_lambda_assume_role" {
   statement {
     actions = ["sts:AssumeRole"]
@@ -34,12 +45,11 @@ data "aws_iam_policy_document" "csv_to_glue_catalog_lambda_execution" {
       "glue:GetPartitions",
       "glue:DeletePartition",
     ]
-    # Currently only scoped to parking
-    resources = [
-      "arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.data_platform.account_id}:catalog",
-      "arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.data_platform.account_id}:database/parking_user_uploads_db",
-      "arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.data_platform.account_id}:table/parking_user_uploads_db/*",
-    ]
+    resources = concat(
+      ["arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.data_platform.account_id}:catalog"],
+      [for db_name in values(local.department_user_uploads_databases) : "arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.data_platform.account_id}:database/${db_name}"],
+      [for db_name in values(local.department_user_uploads_databases) : "arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.data_platform.account_id}:table/${db_name}/*"]
+    )
   }
 
   statement {
@@ -177,11 +187,14 @@ resource "aws_sqs_queue_policy" "csv_to_glue_catalog_events" {
 resource "aws_s3_bucket_notification" "user_uploads_csv_notification" {
   bucket = module.user_uploads_data_source.bucket_id
 
-  queue {
-    queue_arn     = aws_sqs_queue.csv_to_glue_catalog_events.arn
-    events        = ["s3:ObjectCreated:*", "s3:ObjectRemoved:*"]
-    filter_prefix = "parking/" # Currently only scoped to parking
-    filter_suffix = ".csv"
+  dynamic "queue" {
+    for_each = local.department_user_uploads_prefixes
+    content {
+      queue_arn     = aws_sqs_queue.csv_to_glue_catalog_events.arn
+      events        = ["s3:ObjectCreated:*", "s3:ObjectRemoved:*"]
+      filter_prefix = queue.value
+      filter_suffix = ".csv"
+    }
   }
 
   depends_on = [aws_sqs_queue_policy.csv_to_glue_catalog_events]