VED-806 (Follow-up) Fix batch filter lambda bug (#963)

Author: dlzhry2nhs

lambdas/batch_processor_filter/src/batch_processor_filter_service.py

@@ -7,7 +7,7 @@
 from batch_file_repository import BatchFileRepository
 from common.clients import get_sqs_client, logger
 from common.log_firehose import send_log_to_firehose
-from constants import QUEUE_URL, FileNotProcessedReason, FileStatus
+from constants import QUEUE_URL, SPLUNK_FIREHOSE_STREAM_NAME, FileNotProcessedReason, FileStatus
 from exceptions import EventAlreadyProcessingForSupplierAndVaccTypeError
 
 BATCH_AUDIT_REPOSITORY = BatchAuditRepository()
@@ -72,4 +72,7 @@ def apply_filter(self, batch_file_created_event: BatchFileCreatedEvent) -> None:
 
         successful_log_message = f"File forwarded for processing by ECS. Filename: {filename}"
         logger.info(successful_log_message)
-        send_log_to_firehose({**batch_file_created_event, "message": successful_log_message})
+        send_log_to_firehose(
+            stream_name=SPLUNK_FIREHOSE_STREAM_NAME,
+            log_data={**batch_file_created_event, "message": successful_log_message},
+        )

lambdas/batch_processor_filter/tests/test_lambda_handler.py

@@ -26,6 +26,7 @@
         AUDIT_TABLE_FILENAME_GSI,
         AUDIT_TABLE_NAME,
         AUDIT_TABLE_QUEUE_NAME_GSI,
+        SPLUNK_FIREHOSE_STREAM_NAME,
         AuditTableKeys,
         FileStatus,
     )
@@ -299,7 +300,8 @@ def test_lambda_handler_processes_event_successfully(self):
             ]
         )
         self.mock_firehose_send_log.assert_called_once_with(
-            {**self.default_batch_file_event, "message": expected_success_log_message}
+            stream_name=SPLUNK_FIREHOSE_STREAM_NAME,
+            log_data={**self.default_batch_file_event, "message": expected_success_log_message},
         )
 
     def test_lambda_handler_processes_event_successfully_when_event_for_same_supplier_and_vacc_already_processed(
@@ -345,4 +347,6 @@ def test_lambda_handler_processes_event_successfully_when_event_for_same_supplie
                 call(expected_success_log_message),
             ]
         )
-        self.mock_firehose_send_log.assert_called_once_with({**test_event, "message": expected_success_log_message})
+        self.mock_firehose_send_log.assert_called_once_with(
+            stream_name=SPLUNK_FIREHOSE_STREAM_NAME, log_data={**test_event, "message": expected_success_log_message}
+        )

lambdas/batch_processor_filter/tests/testing_utils.py

@@ -10,6 +10,7 @@
     "FILE_NAME_GSI": "filename_index",
     "QUEUE_NAME_GSI": "queue_name_index",
     "SOURCE_BUCKET_NAME": "immunisation-batch-internal-dev-data-sources",
+    "SPLUNK_FIREHOSE_STREAM_NAME": "immunisation-batch-internal-dev-splunk-stream",
     "ACK_BUCKET_NAME": "immunisation-batch-internal-dev-data-destinations",
 }
 

lambdas/shared/src/common/log_firehose.py

@@ -6,6 +6,9 @@
 )
 
 
+# Not keen on including blocking calls in function code to forward log data to Splunk (via Firehose)
+# Consider simply logging and setting up CW subscription filters to forward to Firehose
+# https://docs.aws.amazon.com/firehose/latest/dev/writing-with-cloudwatch-logs.html
 def send_log_to_firehose(stream_name: str, log_data: dict) -> None:
     """Sends the log_message to Firehose"""
     try: