Self review

Author: dlzhry2nhs

batch_processor_filter/README.md

@@ -1,3 +1,31 @@
-batch-processor-filter-lambda
+# batch-processor-filter-lambda
 
-TODO
+## Contributing to this project
+You will first need to have the required setup as specified in the root README.
+
+Then, getting set up to contribute to this Lambda is the same for any others. Refer to 
+the `Setting up a virtual environment with poetry` section in the root README. Assuming 1 and 2 have already been done,
+then you will just need to follow steps 3-5 for this specific directory.
+
+Then run:
+```
+make test
+```
+to verify your setup.
+
+## Overview
+The context of this Lambda function should be understood within the following architecture diagram: 
+https://nhsd-confluence.digital.nhs.uk/spaces/Vacc/pages/1161762503/Immunisation+FHIR+API+-+Batch+Ingestion+Improvements
+
+The purpose of the batch-processor-filter Lambda function is to ensure that there is only ever one batch file event
+processing at any given time for a given combination of `supplier` + `vaccination type`. This is because the order in
+which updates are applied is vital; for the same supplier/vacc type combination, there may be changes to the same
+patient's vaccination record across 2 different files. Therefore, the order in which they arrive must be preserved.
+
+The Lambda function consumes one event at a time on a per `message group id` (composed of the supplier name + vacc type)
+basis from an SQS FIFO queue. At a high-level, the pseudocode is as follows:
+
+- check the Audit Table for any duplicate named files and process accordingly if that is the case
+- check the Audit Table if there is already an event processing for the given supplier + vacc type
+- if there is not, then update the event's status in the Audit Table and forward to SQS for processing by ECS
+- if there is, then throw an error so that the event is returned to the queue and will be tried again later

batch_processor_filter/src/batch_file_repository.py

@@ -9,7 +9,7 @@
 
 
 class BatchFileRepository:
-    """Repository class to handle interactions with batch file interactions e.g. managing the source and ack files"""
+    """Repository class to handle interactions with batch files e.g. management of the source and ack files"""
     _ARCHIVE_FILE_DIR: str = "archive"
     _SOURCE_BUCKET_NAME: str = SOURCE_BUCKET_NAME
     _ACK_BUCKET_NAME: str = ACK_BUCKET_NAME
@@ -29,8 +29,8 @@ def _create_ack_failure_data(batch_file_created_event: BatchFileCreatedEvent) ->
             "RESPONSE_CODE": "10002",
             "RESPONSE_DISPLAY": "Infrastructure Level Response Value - Processing Error",
             "RECEIVED_TIME": batch_file_created_event["created_at_formatted_string"],
-            "MAILBOX_FROM": "",  # TODO: Leave blank for DPS, add mailbox if from mesh mailbox
-            "LOCAL_ID": "",  # TODO: Leave blank for DPS, add from ctl file if data picked up from MESH mailbox
+            "MAILBOX_FROM": "",  # VED-197 TODO: Leave blank for DPS, add mailbox if from mesh mailbox
+            "LOCAL_ID": "",  # VED-197 TODO: Leave blank for DPS, add from ctl file if data picked up from MESH mailbox
             "MESSAGE_DELIVERY": False,
         }
 

batch_processor_filter/src/lambda_handler.py

@@ -12,9 +12,9 @@
 def lambda_handler(event: events.SQSEvent, _: context):
     event_records = event.get("Records", [])
 
-    # Terraform is configured so this Lambda will get a batch size of 1. We are using SQS FIFO with the message group
-    # id set to {supplier}_{vaccine_type} so we will only want batches of 1 at a time.
-    # Lambda will scale out to handle multiple message groups in parallel:
+    # Terraform is configured so the Lambda will get a max batch size of 1. We are using SQS FIFO with the message group
+    # id set to {supplier}_{vaccine_type} so it makes sense to only process batches of 1 at a time.
+    # Lambda will scale out to handle multiple message groups in parallel so this will not block other groups:
     # https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/fifo-queue-lambda-behavior.html
     if len(event_records) != 1:
         raise InvalidBatchSizeError(f"Received {len(event_records)} records, expected 1")

filenameprocessor/tests/test_lambda_handler.py

@@ -194,11 +194,6 @@ def test_lambda_handler_new_file_success_and_first_in_queue(self):
                 self.assert_sqs_message(file_details)
                 self.assert_no_ack_file(file_details)
 
-                # Reset audit table
-                # TODO - is this needed? Check when running all
-                # for item in self.get_audit_table_items():
-                #     dynamodb_client.delete_item(TableName=AUDIT_TABLE_NAME, Key=dict(item.items()))
-
     def test_lambda_handler_non_root_file(self):
         """
         Tests that when the file is not in the root of the source bucket, no action is taken: