Initial tf setup

Author: dlzhry2nhs

filenameprocessor/src/file_name_processor.py

@@ -163,6 +163,7 @@ def lambda_handler(event: dict, context) -> None:  # pylint: disable=unused-argu
 
     logger.info("Filename processor lambda task started")
     for record in event["Records"]:
+        print(record)
         handle_record(record)
 
     logger.info("Filename processor lambda task completed")

terraform/cw_event_rule_batch_file_created.tf

@@ -0,0 +1,41 @@
+resource "aws_cloudwatch_event_rule" "batch_file_created" {
+  name        = "${local.short_prefix}-data-source-batch-file-created"
+  description = "Batch file was added to the data sources S3 bucket"
+
+  event_pattern = jsonencode({
+    source = [
+        "aws.s3"
+    ],
+    detail-type = [
+      "Object Created"
+    ],
+    detail = {
+        bucket = {
+            name = ["${aws_s3_bucket.batch_data_source_bucket.bucket}"]
+        },
+        object = {
+            key = [
+                {
+                    anything-but = {
+                        prefix = "archive/"
+                    }
+                },
+                {
+                    anything-but = {
+                        prefix = "processing/"
+                    }
+                }
+            ]
+        }
+    }
+  })
+}
+
+resource "aws_cloudwatch_event_target" "batch_file_created_sqs_queue" {
+  rule = aws_cloudwatch_event_rule.batch_file_created.name
+  arn  = aws_sqs_queue.batch_file_created_queue.arn
+
+  sqs_target {
+    message_group_id = "new_batch_file"
+  }
+}

terraform/file_name_processor.tf

@@ -179,13 +179,24 @@ resource "aws_iam_policy" "filenameprocessor_lambda_sqs_policy" {
 
   policy = jsonencode({
     Version = "2012-10-17",
-    Statement = [{
-      Effect = "Allow",
-      Action = [
-        "sqs:SendMessage"
-      ],
-      Resource = aws_sqs_queue.supplier_fifo_queue.arn
-    }]
+    Statement = [
+      {
+        Effect = "Allow",
+        Action = [
+          "sqs:SendMessage"
+        ],
+        Resource = aws_sqs_queue.supplier_fifo_queue.arn
+      },
+      {
+        Effect = "Allow",
+        Action = [
+          "sqs:ReceiveMessage",
+          "sqs:DeleteMessage",
+          "sqs:GetQueueAttributes"
+        ],
+        Resource = aws_sqs_queue.batch_file_created_queue.arn
+      }
+    ]
   })
 }
 
@@ -266,6 +277,7 @@ resource "aws_iam_role_policy_attachment" "filenameprocessor_lambda_dynamo_acces
   role       = aws_iam_role.filenameprocessor_lambda_exec_role.name
   policy_arn = aws_iam_policy.filenameprocessor_dynamo_access_policy.arn
 }
+
 # Lambda Function with Security Group and VPC.
 resource "aws_lambda_function" "file_processor_lambda" {
   function_name = "${local.short_prefix}-filenameproc_lambda"
@@ -304,24 +316,11 @@ resource "aws_lambda_function" "file_processor_lambda" {
 
 }
 
-
-# Permission for S3 to invoke Lambda function
-resource "aws_lambda_permission" "s3_invoke_permission" {
-  statement_id  = "AllowExecutionFromS3"
-  action        = "lambda:InvokeFunction"
-  function_name = aws_lambda_function.file_processor_lambda.function_name
-  principal     = "s3.amazonaws.com"
-  source_arn    = aws_s3_bucket.batch_data_source_bucket.arn
-}
-
-# S3 Bucket notification to trigger Lambda function
-resource "aws_s3_bucket_notification" "datasources_lambda_notification" {
-  bucket = aws_s3_bucket.batch_data_source_bucket.bucket
-
-  lambda_function {
-    lambda_function_arn = aws_lambda_function.file_processor_lambda.arn
-    events              = ["s3:ObjectCreated:*"]
-  }
+resource "aws_lambda_event_source_mapping" "batch_file_created_sqs_to_lambda" {
+  event_source_arn = aws_sqs_queue.batch_file_created_queue.arn
+  function_name    = aws_lambda_function.file_processor_lambda.arn
+  batch_size       = 1
+  enabled          = true
 }
 
 resource "aws_cloudwatch_log_group" "file_name_processor_log_group" {

terraform/s3_config.tf

@@ -3,6 +3,11 @@ resource "aws_s3_bucket" "batch_data_source_bucket" {
   force_destroy = local.is_temp
 }
 
+resource "aws_s3_bucket_notification" "datasources_file_created" {
+  bucket      = aws_s3_bucket.batch_data_source_bucket.id
+  eventbridge = true
+}
+
 resource "aws_s3_bucket_public_access_block" "batch_data_source_bucket_public_access_block" {
   bucket = aws_s3_bucket.batch_data_source_bucket.id
 

terraform/sqs_batch_file_created.tf

@@ -0,0 +1,24 @@
+data "aws_iam_policy_document" "batch_file_created_queue_policy" {
+  statement {
+    effect = "Allow"
+
+		principals {
+      type        = "Service"
+      identifiers = ["events.amazonaws.com"]
+    }
+
+    actions   = ["sqs:SendMessage"]
+    resources = [
+			aws_sqs_queue.batch_file_created_queue.arn
+		]
+  }
+}
+
+# FIFO SQS Queue - targetted by Event Bridge for new objects created in the data-sources S3 bucket
+resource "aws_sqs_queue" "batch_file_created_queue" {
+  name                        = "${local.short_prefix}-batch-file-created-queue.fifo"
+	policy 											= data.aws_iam_policy_document.batch_file_created_queue_policy.json
+  fifo_queue                  = true
+  content_based_deduplication = true # Optional, helps with deduplication
+  visibility_timeout_seconds  = 60
+}