PoC for 902: to be re-engineered IV (with test bucket in TF)

Author: JamesW1-NHS

infrastructure/instance/ecs_batch_processor_config.tf

@@ -207,6 +207,10 @@ resource "aws_ecs_task_definition" "ecs_task" {
         name  = "ACK_BUCKET_NAME"
         value = aws_s3_bucket.batch_data_destination_bucket.bucket
       },
+      {
+        name  = "EA_BUCKET_NAME"
+        value = aws_s3_bucket.batch_data_ea_bucket.bucket
+      },
       {
         name  = "KINESIS_STREAM_ARN"
         value = local.kinesis_arn

infrastructure/instance/file_name_processor.tf

@@ -277,6 +277,7 @@ resource "aws_lambda_function" "file_processor_lambda" {
 
   environment {
     variables = {
+      ACCOUNT_ID           = var.immunisation_account_id
       SOURCE_BUCKET_NAME   = aws_s3_bucket.batch_data_source_bucket.bucket
       ACK_BUCKET_NAME      = aws_s3_bucket.batch_data_destination_bucket.bucket
       QUEUE_URL            = aws_sqs_queue.batch_file_created.url

infrastructure/instance/s3_config.tf

@@ -231,3 +231,78 @@ resource "aws_s3_bucket_policy" "batch_config_bucket_policy" {
     ]
   })
 }
+
+# ---
+# temp: test output bucket for EA files. pending DPS bucket.
+
+resource "aws_s3_bucket" "batch_data_ea_bucket" {
+  bucket        = "${local.batch_prefix}-data-ea"
+  force_destroy = local.is_temp
+}
+
+resource "aws_s3_bucket_public_access_block" "batch_data_ea_bucket_public_access_block" {
+  bucket = aws_s3_bucket.batch_data_ea_bucket.id
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+resource "aws_s3_bucket_policy" "batch_data_ea_bucket_policy" {
+  bucket = aws_s3_bucket.batch_data_ea_bucket.id
+  policy = jsonencode({
+    Version : "2012-10-17",
+    Statement : [
+      {
+        Effect : "Allow",
+        Principal : {
+          AWS : "arn:aws:iam::${var.dspp_core_account_id}:root"
+        },
+        Action : var.environment == "prod" ? [
+          "s3:ListBucket",
+          "s3:GetObject",
+          ] : [
+          "s3:ListBucket",
+          "s3:GetObject",
+          "s3:DeleteObject"
+        ],
+        Resource : [
+          aws_s3_bucket.batch_data_ea_bucket.arn,
+          "${aws_s3_bucket.batch_data_ea_bucket.arn}/*"
+        ]
+      },
+      {
+        Sid    = "HTTPSOnly"
+        Effect = "Deny"
+        Principal = {
+          "AWS" : "*"
+        }
+        Action = "s3:*"
+        Resource = [
+          aws_s3_bucket.batch_data_ea_bucket.arn,
+          "${aws_s3_bucket.batch_data_ea_bucket.arn}/*",
+        ]
+        Condition = {
+          Bool = {
+            "aws:SecureTransport" = "false"
+          }
+        }
+      },
+    ]
+  })
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "s3_batch_ea_encryption" {
+  bucket = aws_s3_bucket.batch_data_ea_bucket.id
+
+  rule {
+    apply_server_side_encryption_by_default {
+      kms_master_key_id = data.aws_kms_key.existing_s3_encryption_key.arn
+      sse_algorithm     = "aws:kms"
+    }
+  }
+}
+
+# ---
+

lambdas/filenameprocessor/src/file_name_processor.py

@@ -18,6 +18,7 @@
 from common.log_decorator import logging_decorator
 from common.models.errors import UnhandledAuditTableError
 from constants import (
+    EA_BUCKET_NAME,
     ERROR_TYPE_TO_STATUS_CODE_MAP,
     SOURCE_BUCKET_NAME,
     FileNotProcessedReason,
@@ -42,7 +43,6 @@
 # We could implement a new lambda triggered on it BUT if it's never triggered, we never get the upsert.
 
 EXPECTED_BUCKET_OWNER_ACCOUNT = os.getenv("ACCOUNT_ID")
-TEST_EA_BUCKET = "902-test-ea-bucket"
 TEST_EA_FILENAME = "Vaccination_Extended_Attributes"
 
 
@@ -103,7 +103,16 @@ def handle_record(record) -> dict:
         # here: if it's an EA file, move it, and upsert it to PROCESSING; use the bucket name as the queue name
         if TEST_EA_FILENAME in file_key:
             queue_name = "TEST_COVID"
-            dest_bucket_name = TEST_EA_BUCKET
+            dest_bucket_name = EA_BUCKET_NAME
+
+            upsert_audit_table(
+                message_id,
+                file_key,
+                created_at_formatted_string,
+                expiry_timestamp,
+                queue_name,
+                FileStatus.PROCESSING,
+            )
 
             s3_client = get_s3_client()
             s3_client.copy_object(
@@ -114,14 +123,6 @@ def handle_record(record) -> dict:
                 ExpectedSourceBucketOwner=EXPECTED_BUCKET_OWNER_ACCOUNT,
             )
 
-            upsert_audit_table(
-                message_id,
-                file_key,
-                created_at_formatted_string,
-                expiry_timestamp,
-                dest_bucket_name,
-                FileStatus.PROCESSING,
-            )
             logger.info("Lambda invocation successful for file '%s'", file_key)
 
             # TODO: check the file is in the dest bucket, upsert again accordingly.
@@ -144,7 +145,7 @@ def handle_record(record) -> dict:
                     file_key,
                     created_at_formatted_string,
                     expiry_timestamp,
-                    dest_bucket_name,
+                    queue_name,
                     file_status,
                 )
                 s3_client.delete_object(
@@ -156,6 +157,14 @@ def handle_record(record) -> dict:
                 status_code = 400
                 message = (f"Failed to send to {dest_bucket_name} for further processing",)
                 file_status = FileStatus.FAILED
+                upsert_audit_table(
+                    message_id,
+                    file_key,
+                    created_at_formatted_string,
+                    expiry_timestamp,
+                    queue_name,
+                    file_status,
+                )
                 move_file(bucket_name, file_key, f"archive/{file_key}")
 
             # Return details for logs