Merge pull request #840 from NHSDigital/feature/jale13-nrl-1288-reporting-allow-multiple-log-groups

NRL 1288/1304 Reporting allow multiple log groups

Author: jackleary

terraform/account-wide-infrastructure/modules/glue/glue.tf

@@ -1,7 +1,7 @@
 # Create Glue Data Catalog Database
 resource "aws_glue_catalog_database" "log_database" {
   name         = "${var.name_prefix}-reporting"
-  location_uri = "${aws_s3_bucket.target-data-bucket.id}/logs/"
+  location_uri = "${aws_s3_bucket.target-data-bucket.id}/"
 }
 
 # Create Glue Crawler
@@ -10,7 +10,37 @@ resource "aws_glue_crawler" "log_crawler" {
   database_name = aws_glue_catalog_database.log_database.name
   role          = aws_iam_role.glue_service_role.name
   s3_target {
-    path = "${aws_s3_bucket.target-data-bucket.id}/logs/"
+    path = "${aws_s3_bucket.target-data-bucket.id}/consumer_countDocumentReference/"
+  }
+  s3_target {
+    path = "${aws_s3_bucket.target-data-bucket.id}/consumer_readDocumentReference/"
+  }
+  s3_target {
+    path = "${aws_s3_bucket.target-data-bucket.id}/consumer_searchDocumentReference/"
+  }
+  s3_target {
+    path = "${aws_s3_bucket.target-data-bucket.id}/consumer_searchPostDocumentReference/"
+  }
+  s3_target {
+    path = "${aws_s3_bucket.target-data-bucket.id}/producer_createDocumentReference/"
+  }
+  s3_target {
+    path = "${aws_s3_bucket.target-data-bucket.id}/producer_deleteDocumentReference/"
+  }
+  s3_target {
+    path = "${aws_s3_bucket.target-data-bucket.id}/producer_readDocumentReference/"
+  }
+  s3_target {
+    path = "${aws_s3_bucket.target-data-bucket.id}/producer_searchDocumentReference/"
+  }
+  s3_target {
+    path = "${aws_s3_bucket.target-data-bucket.id}/producer_searchPostDocumentReference/"
+  }
+  s3_target {
+    path = "${aws_s3_bucket.target-data-bucket.id}/producer_updateDocumentReference/"
+  }
+  s3_target {
+    path = "${aws_s3_bucket.target-data-bucket.id}/producer_upsertDocumentReference//"
   }
   schema_change_policy {
     delete_behavior = "LOG"
@@ -49,8 +79,8 @@ resource "aws_glue_job" "glue_job" {
     "--enable-auto-scaling"             = "true"
     "--enable-continous-cloudwatch-log" = "true"
     "--datalake-formats"                = "delta"
-    "--source_path"                     = "s3://${aws_s3_bucket.source-data-bucket.id}/"     # Specify the source S3 path
-    "--target_path"                     = "s3://${aws_s3_bucket.target-data-bucket.id}/logs" # Specify the destination S3 path
+    "--source_path"                     = "s3://${aws_s3_bucket.source-data-bucket.id}/" # Specify the source S3 path
+    "--target_path"                     = "s3://${aws_s3_bucket.target-data-bucket.id}/" # Specify the destination S3 path
     "--job_name"                        = "${var.name_prefix}-glue-job"
     "--partition_cols"                  = "date"
     "--enable-continuous-log-filter"    = "true"