Merge branch 'master' into VED-789-Schema-Validation

Author: Akol125

backend/src/models/fhir_immunization_pre_validators.py

@@ -98,6 +98,7 @@ def validate(self):
             self.pre_validate_vaccination_procedure_code,
             self.pre_validate_vaccination_procedure_display,
             self.pre_validate_vaccine_code,
+            self.pre_validate_vaccine_display,
         ]
 
         for method in validation_methods:
@@ -976,3 +977,16 @@ def pre_validate_vaccine_code(self, values: dict) -> dict:
             PreValidation.for_snomed_code(field_value, field_location)
         except (KeyError, IndexError):
             pass
+
+    def pre_validate_vaccine_display(self, values: dict) -> dict:
+        """
+        Pre-validate that, if vaccineCode.coding[?(@.system=='http://snomed.info/sct')].display
+        (legacy CSV field : VACCINE_PRODUCT_TERM) exists, then it is a non-empty string
+        """
+        url = Urls.snomed
+        field_location = f"vaccineCode.coding[?(@.system=='{url}')].display"
+        try:
+            field_value = [x for x in values["vaccineCode"]["coding"] if x.get("system") == url][0]["display"]
+            PreValidation.for_string(field_value, field_location)
+        except (KeyError, IndexError):
+            pass

backend/tests/test_immunization_pre_validator.py

@@ -1404,6 +1404,14 @@ def test_pre_validate_vaccine_code(self):
                 actual_error_messages,
             )
 
+    def test_pre_validate_vaccine_display(self):
+        """Test test_pre_validate_vaccine_display accepts valid values and rejects invalid values"""
+        ValidatorModelTests.test_string_value(
+            self,
+            field_location="vaccineCode.coding[?(@.system=='http://snomed.info/sct')].display",
+            valid_strings_to_test=["DUMMY"],
+        )
+
     def test_pre_validate_vaccination_procedure_display(self):
         """Test test_pre_validate_vaccination_procedure_display accepts valid values and rejects invalid values"""
         ValidatorModelTests.test_string_value(

lambdas/batch_processor_filter/src/batch_processor_filter_service.py

@@ -7,7 +7,7 @@
 from batch_file_repository import BatchFileRepository
 from common.clients import get_sqs_client, logger
 from common.log_firehose import send_log_to_firehose
-from constants import QUEUE_URL, FileNotProcessedReason, FileStatus
+from constants import QUEUE_URL, SPLUNK_FIREHOSE_STREAM_NAME, FileNotProcessedReason, FileStatus
 from exceptions import EventAlreadyProcessingForSupplierAndVaccTypeError
 
 BATCH_AUDIT_REPOSITORY = BatchAuditRepository()
@@ -72,4 +72,7 @@ def apply_filter(self, batch_file_created_event: BatchFileCreatedEvent) -> None:
 
         successful_log_message = f"File forwarded for processing by ECS. Filename: {filename}"
         logger.info(successful_log_message)
-        send_log_to_firehose({**batch_file_created_event, "message": successful_log_message})
+        send_log_to_firehose(
+            stream_name=SPLUNK_FIREHOSE_STREAM_NAME,
+            log_data={**batch_file_created_event, "message": successful_log_message},
+        )

lambdas/batch_processor_filter/tests/test_lambda_handler.py

@@ -26,6 +26,7 @@
         AUDIT_TABLE_FILENAME_GSI,
         AUDIT_TABLE_NAME,
         AUDIT_TABLE_QUEUE_NAME_GSI,
+        SPLUNK_FIREHOSE_STREAM_NAME,
         AuditTableKeys,
         FileStatus,
     )
@@ -299,7 +300,8 @@ def test_lambda_handler_processes_event_successfully(self):
             ]
         )
         self.mock_firehose_send_log.assert_called_once_with(
-            {**self.default_batch_file_event, "message": expected_success_log_message}
+            stream_name=SPLUNK_FIREHOSE_STREAM_NAME,
+            log_data={**self.default_batch_file_event, "message": expected_success_log_message},
         )
 
     def test_lambda_handler_processes_event_successfully_when_event_for_same_supplier_and_vacc_already_processed(
@@ -345,4 +347,6 @@ def test_lambda_handler_processes_event_successfully_when_event_for_same_supplie
                 call(expected_success_log_message),
             ]
         )
-        self.mock_firehose_send_log.assert_called_once_with({**test_event, "message": expected_success_log_message})
+        self.mock_firehose_send_log.assert_called_once_with(
+            stream_name=SPLUNK_FIREHOSE_STREAM_NAME, log_data={**test_event, "message": expected_success_log_message}
+        )

lambdas/batch_processor_filter/tests/testing_utils.py

@@ -10,6 +10,7 @@
     "FILE_NAME_GSI": "filename_index",
     "QUEUE_NAME_GSI": "queue_name_index",
     "SOURCE_BUCKET_NAME": "immunisation-batch-internal-dev-data-sources",
+    "SPLUNK_FIREHOSE_STREAM_NAME": "immunisation-batch-internal-dev-splunk-stream",
     "ACK_BUCKET_NAME": "immunisation-batch-internal-dev-data-destinations",
 }
 

lambdas/shared/src/common/log_firehose.py

@@ -4,6 +4,9 @@
 from common.clients import logger
 
 
+# Not keen on including blocking calls in function code to forward log data to Splunk (via Firehose)
+# Consider simply logging and setting up CW subscription filters to forward to Firehose
+# https://docs.aws.amazon.com/firehose/latest/dev/writing-with-cloudwatch-logs.html
 def send_log_to_firehose(stream_name: str, log_data: dict) -> None:
     """Sends the log_message to Firehose"""
     try: