From 1580a4bf0f371d723183b61e86767fe4d1362161 Mon Sep 17 00:00:00 2001 From: nhsdevws Date: Wed, 14 May 2025 15:25:03 +0100 Subject: [PATCH 1/3] Rebase. Unit Tests Pass --- delta_backend/src/common/mappings.py | 27 ++ delta_backend/src/delta.py | 21 +- .../tests/test_convert_to_flat_json.py | 17 +- delta_backend/tests/test_delta.py | 210 +++++++++----- .../tests/utils_for_converter_tests.py | 258 +++++------------- 5 files changed, 257 insertions(+), 276 deletions(-) create mode 100644 delta_backend/src/common/mappings.py diff --git a/delta_backend/src/common/mappings.py b/delta_backend/src/common/mappings.py new file mode 100644 index 0000000000..5f9988cb3c --- /dev/null +++ b/delta_backend/src/common/mappings.py @@ -0,0 +1,27 @@ +""" + Define enums for event names, operations, and action flags. + + # case eventName operation actionFlag + ----------------- --------- --------- ---------- + create INSERT CREATE NEW + update MODIFY UPDATE UPDATE + logically delete MODIFY DELETE DELETE + physically delete REMOVE REMOVE N/A +""" + +class EventName(): + CREATE = "INSERT" + UPDATE = "MODIFY" + DELETE_LOGICAL = "MODIFY" + DELETE_PHYSICAL = "REMOVE" + +class Operation(): + CREATE = "CREATE" + UPDATE = "UPDATE" + DELETE_LOGICAL = "DELETE" + DELETE_PHYSICAL = "REMOVE" + +class ActionFlag(): + CREATE = "NEW" + UPDATE = "UPDATE" + DELETE_LOGICAL = "DELETE" diff --git a/delta_backend/src/delta.py b/delta_backend/src/delta.py index a47d465664..ed94da6763 100644 --- a/delta_backend/src/delta.py +++ b/delta_backend/src/delta.py @@ -8,6 +8,7 @@ from botocore.exceptions import ClientError from log_firehose import FirehoseLogger from Converter import Converter +from common.mappings import ActionFlag, Operation, EventName failure_queue_url = os.environ["AWS_SQS_QUEUE_URL"] delta_table_name = os.environ["DELTA_TABLE_NAME"] @@ -37,6 +38,7 @@ def get_vaccine_type(patientsk) -> str: def handler(event, context): + ret = True logger.info("Starting Delta Handler") log_data = dict() firehose_log = dict() @@ -61,14 +63,14 @@ def handler(event, context): response = str() imms_id = str() operation = str() - if record["eventName"] != "REMOVE": + if record["eventName"] != EventName.DELETE_PHYSICAL: new_image = record["dynamodb"]["NewImage"] imms_id = new_image["PK"]["S"].split("#")[1] vaccine_type = get_vaccine_type(new_image["PatientSK"]["S"]) supplier_system = new_image["SupplierSystem"]["S"] if supplier_system not in ("DPSFULL", "DPSREDUCED"): operation = new_image["Operation"]["S"] - action_flag = "NEW" if operation == "CREATE" else operation + action_flag = ActionFlag.CREATE if operation == Operation.CREATE else operation resource_json = json.loads(new_image["Resource"]["S"]) FHIRConverter = Converter(json.dumps(resource_json)) flat_json = FHIRConverter.runConversion(resource_json) # Get the flat JSON @@ -94,9 +96,9 @@ def handler(event, context): firehose_log["event"] = log_data firehose_logger.send_log(firehose_log) logger.info(f"Record from DPS skipped for {imms_id}") - return {"statusCode": 200, "body": f"Record from DPS skipped for {imms_id}"} + continue else: - operation = "REMOVE" + operation = Operation.DELETE_PHYSICAL new_image = record["dynamodb"]["Keys"] logger.info(f"Record to delta:{new_image}") imms_id = new_image["PK"]["S"].split("#")[1] @@ -104,7 +106,7 @@ def handler(event, context): Item={ "PK": str(uuid.uuid4()), "ImmsID": imms_id, - "Operation": "REMOVE", + "Operation": Operation.DELETE_PHYSICAL, "VaccineType": "default", "SupplierSystem": "default", "DateTimeStamp": approximate_creation_time.isoformat(), @@ -131,7 +133,6 @@ def handler(event, context): firehose_log["event"] = log_data firehose_logger.send_log(firehose_log) logger.info(log) - return {"statusCode": 200, "body": "Records processed successfully"} else: log = f"Record NOT created for {imms_id}" operation_outcome["statusCode"] = "500" @@ -140,7 +141,7 @@ def handler(event, context): firehose_log["event"] = log_data firehose_logger.send_log(firehose_log) logger.info(log) - return {"statusCode": 500, "body": "Records not processed successfully"} + ret = False except Exception as e: operation_outcome["statusCode"] = "500" @@ -155,7 +156,5 @@ def handler(event, context): log_data["operation_outcome"] = operation_outcome firehose_log["event"] = log_data firehose_logger.send_log(firehose_log) - return { - "statusCode": 500, - "body": "Records not processed", - } + ret = False + return ret diff --git a/delta_backend/tests/test_convert_to_flat_json.py b/delta_backend/tests/test_convert_to_flat_json.py index 88d420b472..9667398f7d 100644 --- a/delta_backend/tests/test_convert_to_flat_json.py +++ b/delta_backend/tests/test_convert_to_flat_json.py @@ -8,6 +8,7 @@ from SchemaParser import SchemaParser from Converter import Converter from ConversionChecker import ConversionChecker, RecordError +from common.mappings import ActionFlag, Operation, EventName import ExceptionMessages MOCK_ENV_VARS = { @@ -100,7 +101,7 @@ def tearDown(self): self.mock_firehose_logger.stop() @staticmethod - def get_event(event_name="INSERT", operation="operation", supplier="EMIS"): + def get_event(event_name=EventName.CREATE, operation="operation", supplier="EMIS"): """Returns test event data.""" return ValuesForTests.get_event(event_name, operation, supplier) @@ -110,8 +111,7 @@ def assert_dynamodb_record(self, operation_flag, action_flag, items, expected_va Ignores dynamically generated fields like PK, DateTimeStamp, and ExpiresAt. Ensures that the 'Imms' field matches exactly. """ - self.assertEqual(response["statusCode"], 200) - self.assertEqual(response["body"], "Records processed successfully") + self.assertTrue(response) filtered_items = [ {k: v for k, v in item.items() if k not in ["PK", "DateTimeStamp", "ExpiresAt"]} @@ -126,6 +126,11 @@ def assert_dynamodb_record(self, operation_flag, action_flag, items, expected_va self.assertIsInstance(imms_data, dict) self.assertGreater(len(imms_data), 0) + for key, expected_value in expected_values.items(): + self.assertIn(key, filtered_items[0], f"{key} is missing") + if (filtered_items[0][key] != expected_value): + print (f"{key} mismatch {filtered_items[0][key]} != {expected_value}") + # Check Imms JSON structure matches exactly self.assertEqual(imms_data, expected_imms, "Imms data does not match expected JSON structure") @@ -167,9 +172,9 @@ def test_fhir_converter_json_error_scenario(self): def test_handler_imms_convert_to_flat_json(self): """Test that the Imms field contains the correct flat JSON data for CREATE, UPDATE, and DELETE operations.""" expected_action_flags = [ - {"Operation": "CREATE", "EXPECTED_ACTION_FLAG": "NEW"}, - {"Operation": "UPDATE", "EXPECTED_ACTION_FLAG": "UPDATE"}, - {"Operation": "DELETE", "EXPECTED_ACTION_FLAG": "DELETE"}, + {"Operation": Operation.CREATE, "EXPECTED_ACTION_FLAG": ActionFlag.CREATE}, + {"Operation": Operation.UPDATE, "EXPECTED_ACTION_FLAG": ActionFlag.UPDATE}, + {"Operation": Operation.DELETE_LOGICAL, "EXPECTED_ACTION_FLAG": ActionFlag.DELETE_LOGICAL}, ] for test_case in expected_action_flags: diff --git a/delta_backend/tests/test_delta.py b/delta_backend/tests/test_delta.py index 0cb556286f..c4883b3e48 100644 --- a/delta_backend/tests/test_delta.py +++ b/delta_backend/tests/test_delta.py @@ -3,6 +3,7 @@ from botocore.exceptions import ClientError import os import json +from common.mappings import EventName, Operation, ActionFlag # Set environment variables before importing the module ## @TODO: # Note: Environment variables shared across tests, thus aligned @@ -11,7 +12,7 @@ os.environ["SOURCE"] = "my_source" from delta import send_message, handler # Import after setting environment variables -from tests.utils_for_converter_tests import ValuesForTests +from utils_for_converter_tests import ValuesForTests, RecordConfig class DeltaTestCase(unittest.TestCase): @@ -52,51 +53,6 @@ def setUp_mock_resources(self, mock_boto_resource, mock_boto_client): mock_table.put_item.side_effect = Exception("Test Exception") return mock_table - @staticmethod - def get_event(event_name="INSERT", operation="CREATE", supplier="EMIS", n_records=1): - """Create test event for the handler function.""" - return { - "Records": [ - DeltaTestCase.get_event_record(f"covid#{i+1}2345", event_name, operation, supplier) - for i in range(n_records) - ] - } - - @staticmethod - def get_event_record(pk, event_name="INSERT", operation="CREATE", supplier="EMIS"): - if operation != "DELETE": - return{ - "eventName": event_name, - "dynamodb": { - "ApproximateCreationDateTime": 1690896000, - "NewImage": { - "PK": {"S": pk}, - "PatientSK": {"S": pk}, - "IdentifierPK": {"S": "system#1"}, - "Operation": {"S": operation}, - "SupplierSystem": {"S": supplier}, - "Resource": { - "S": json.dumps(ValuesForTests.get_test_data_resource()), - } - } - } - } - else: - return { - "eventName": "REMOVE", - "dynamodb": { - "ApproximateCreationDateTime": 1690896000, - "Keys": { - "PK": {"S": pk}, - "PatientSK": {"S": pk}, - "SupplierSystem": {"S": "EMIS"}, - "Resource": { - "S": json.dumps(ValuesForTests.get_test_data_resource()), - } - } - } - } - @patch("boto3.client") def test_send_message_success(self, mock_boto_client): # Arrange @@ -134,52 +90,70 @@ def test_send_message_client_error(self, mock_logger_error, mock_boto_client): @patch("boto3.resource") def test_handler_success_insert(self, mock_boto_resource): # Arrange - self.setup_mock_dynamodb(mock_boto_resource) + mock_table = self.setup_mock_dynamodb(mock_boto_resource) suppilers = ["DPS", "EMIS"] for supplier in suppilers: - event = self.get_event(supplier=supplier) + imms_id = f"test-insert-imms-{supplier}-id" + event = ValuesForTests.get_event(event_name=EventName.CREATE, operation=Operation.CREATE, imms_id=imms_id, supplier=supplier) # Act result = handler(event, self.context) # Assert - self.assertEqual(result["statusCode"], 200) + self.assertTrue(result) + mock_table.put_item.assert_called() + self.mock_firehose_logger.send_log.assert_called() # check logged + put_item_call_args = mock_table.put_item.call_args # check data written to DynamoDB + put_item_data = put_item_call_args.kwargs["Item"] + self.assertIn("Imms", put_item_data) + self.assertEqual(put_item_data["Imms"]["ACTION_FLAG"], ActionFlag.CREATE) + self.assertEqual(put_item_data["Operation"], Operation.CREATE) + self.assertEqual(put_item_data["SupplierSystem"], supplier) @patch("boto3.resource") def test_handler_failure(self, mock_boto_resource): # Arrange self.setup_mock_dynamodb(mock_boto_resource, status_code=500) - event = self.get_event() + event = ValuesForTests.get_event() # Act result = handler(event, self.context) # Assert - self.assertEqual(result["statusCode"], 500) + self.assertFalse(result) @patch("boto3.resource") def test_handler_success_update(self, mock_boto_resource): # Arrange self.setup_mock_dynamodb(mock_boto_resource) - event = self.get_event(event_name="UPDATE", operation="UPDATE") + event = ValuesForTests.get_event(event_name=EventName.UPDATE, operation=Operation.UPDATE) # Act result = handler(event, self.context) # Assert - self.assertEqual(result["statusCode"], 200) + self.assertTrue(result) @patch("boto3.resource") def test_handler_success_remove(self, mock_boto_resource): # Arrange - self.setup_mock_dynamodb(mock_boto_resource) - event = self.get_event(event_name="REMOVE", operation="DELETE") + mock_table = self.setup_mock_dynamodb(mock_boto_resource) + imms_id = "test-update-imms-id" + event = ValuesForTests.get_event(event_name=EventName.UPDATE, operation=Operation.UPDATE, imms_id=imms_id) # Act result = handler(event, self.context) # Assert - self.assertEqual(result["statusCode"], 200) + self.assertTrue(result) + mock_table.put_item.assert_called() + self.mock_firehose_logger.send_log.assert_called() # check logged + put_item_call_args = mock_table.put_item.call_args # check data written to DynamoDB + put_item_data = put_item_call_args.kwargs["Item"] + self.assertIn("Imms", put_item_data) + self.assertEqual(put_item_data["Imms"]["ACTION_FLAG"], ActionFlag.UPDATE) + self.assertEqual(put_item_data["Operation"], Operation.UPDATE) + self.assertEqual(put_item_data["ImmsID"], imms_id) @patch("boto3.resource") @patch("boto3.client") @@ -187,19 +161,19 @@ def test_handler_exception_intrusion_check(self, mock_boto_resource, mock_boto_c # Arrange self.setup_mock_dynamodb(mock_boto_resource, status_code=500) mock_boto_client.return_value = MagicMock() - event = self.get_event() + event = ValuesForTests.get_event() # Act & Assert result = handler(event, self.context) - self.assertEqual(result["statusCode"], 500) + self.assertFalse(result) @patch("boto3.resource") @patch("boto3.client") def test_handler_exception_intrusion(self, mock_boto_client, mock_boto_resource): # Arrange self.setUp_mock_resources(mock_boto_resource, mock_boto_client) - event = self.get_event() + event = ValuesForTests.get_event() context = {} # Act & Assert @@ -213,23 +187,22 @@ def test_handler_exception_intrusion(self, mock_boto_client, mock_boto_resource) def test_handler_exception_intrusion_check_false(self, mocked_intrusion, mock_boto_client): # Arrange self.setUp_mock_resources(mocked_intrusion, mock_boto_client) - event = self.get_event() + event = ValuesForTests.get_event() context = {} # Act & Assert response = handler(event, context) - self.assertEqual(response["statusCode"], 500) + self.assertFalse(response) @patch("delta.logger.info") # Mock logging def test_dps_record_skipped(self, mock_logger_info): - event = self.get_event(supplier="DPSFULL") + event = ValuesForTests.get_event(supplier="DPSFULL") context = {} response = handler(event, context) - self.assertEqual(response["statusCode"], 200) - self.assertEqual(response["body"], "Record from DPS skipped for 12345") + self.assertTrue(response) # Check logging and Firehose were called mock_logger_info.assert_called_with("Record from DPS skipped for 12345") @@ -249,7 +222,7 @@ def test_partial_success_with_errors(self, mock_dynamodb, mock_converter, mock_l mock_dynamodb.return_value.Table.return_value = mock_table mock_table.put_item.return_value = {"ResponseMetadata": {"HTTPStatusCode": 200}} - event = self.get_event() + event = ValuesForTests.get_event() context = {} response = handler(event, context) @@ -260,3 +233,110 @@ def test_partial_success_with_errors(self, mock_dynamodb, mock_converter, mock_l # Check logging and Firehose were called # mock_logger_info.assert_called() # mock_firehose_send_log.assert_called() + + @patch("boto3.resource") + def test_send_message_multi_records_diverse(self, mock_boto_resource): + # Arrange + mock_table = self.setup_mock_dynamodb(mock_boto_resource) + + records_config = [ + RecordConfig(EventName.CREATE, Operation.CREATE, "id1", ActionFlag.CREATE), + RecordConfig(EventName.UPDATE, Operation.UPDATE, "id2", ActionFlag.UPDATE), + RecordConfig(EventName.DELETE_LOGICAL, Operation.DELETE_LOGICAL, "id3", ActionFlag.DELETE_LOGICAL), + RecordConfig(EventName.DELETE_PHYSICAL, Operation.DELETE_PHYSICAL, "id4"), + ] + # Generate the event using ValuesForTests.get_multi_record_event + event = ValuesForTests.get_multi_record_event(records_config) + + # Act + result = handler(event, self.context) + + # Assert + self.assertTrue(result) + self.assertEqual(mock_table.put_item.call_count, len(records_config)) + self.assertEqual(self.mock_firehose_logger.send_log.call_count, len(records_config)) + + @patch("boto3.resource") + def test_send_message_multi_create(self, mock_boto_resource): + # Arrange + mock_table = self.setup_mock_dynamodb(mock_boto_resource) + + records_config = [ + RecordConfig(EventName.CREATE, Operation.CREATE, "create-id1", ActionFlag.CREATE), + RecordConfig(EventName.CREATE, Operation.CREATE, "create-id2", ActionFlag.CREATE), + RecordConfig(EventName.CREATE, Operation.CREATE, "create-id3", ActionFlag.CREATE) + ] + # Generate the event using ValuesForTests.get_multi_record_event + event = ValuesForTests.get_multi_record_event(records_config) + + # Act + result = handler(event, self.context) + + # Assert + self.assertTrue(result) + self.assertEqual(mock_table.put_item.call_count, 3) + self.assertEqual(self.mock_firehose_logger.send_log.call_count, 3) + + + @patch("boto3.resource") + def test_send_message_multi_update(self, mock_boto_resource): + # Arrange + mock_table = self.setup_mock_dynamodb(mock_boto_resource) + + records_config = [ + RecordConfig(EventName.UPDATE, Operation.UPDATE, "update-id1", ActionFlag.UPDATE), + RecordConfig(EventName.UPDATE, Operation.UPDATE, "update-id2", ActionFlag.UPDATE), + RecordConfig(EventName.UPDATE, Operation.UPDATE, "update-id3", ActionFlag.UPDATE) + ] + # Generate the event using ValuesForTests.get_multi_record_event + event = ValuesForTests.get_multi_record_event(records_config) + + # Act + result = handler(event, self.context) + + # Assert + self.assertTrue(result) + self.assertEqual(mock_table.put_item.call_count, 3) + self.assertEqual(self.mock_firehose_logger.send_log.call_count, 3) + + @patch("boto3.resource") + def test_send_message_multi_logical_delete(self, mock_boto_resource): + # Arrange + mock_table = self.setup_mock_dynamodb(mock_boto_resource) + + records_config = [ + RecordConfig(EventName.DELETE_LOGICAL, Operation.DELETE_LOGICAL, "update-id1", ActionFlag.DELETE_LOGICAL), + RecordConfig(EventName.DELETE_LOGICAL, Operation.DELETE_LOGICAL, "update-id2", ActionFlag.DELETE_LOGICAL), + RecordConfig(EventName.DELETE_LOGICAL, Operation.DELETE_LOGICAL, "update-id3", ActionFlag.DELETE_LOGICAL) + ] + # Generate the event using ValuesForTests.get_multi_record_event + event = ValuesForTests.get_multi_record_event(records_config) + + # Act + result = handler(event, self.context) + + # Assert + self.assertTrue(result) + self.assertEqual(mock_table.put_item.call_count, 3) + self.assertEqual(self.mock_firehose_logger.send_log.call_count, 3) + + @patch("boto3.resource") + def test_send_message_multi_physical_delete(self, mock_boto_resource): + # Arrange + mock_table = self.setup_mock_dynamodb(mock_boto_resource) + + records_config = [ + RecordConfig(EventName.DELETE_PHYSICAL, Operation.DELETE_PHYSICAL, "update-id1"), + RecordConfig(EventName.DELETE_PHYSICAL, Operation.DELETE_PHYSICAL, "update-id2"), + RecordConfig(EventName.DELETE_PHYSICAL, Operation.DELETE_PHYSICAL, "update-id3") + ] + # Generate the event using ValuesForTests.get_multi_record_event + event = ValuesForTests.get_multi_record_event(records_config) + + # Act + result = handler(event, self.context) + + # Assert + self.assertTrue(result) + self.assertEqual(mock_table.put_item.call_count, 3) + self.assertEqual(self.mock_firehose_logger.send_log.call_count, 3) diff --git a/delta_backend/tests/utils_for_converter_tests.py b/delta_backend/tests/utils_for_converter_tests.py index d361ac241c..e2873023b9 100644 --- a/delta_backend/tests/utils_for_converter_tests.py +++ b/delta_backend/tests/utils_for_converter_tests.py @@ -1,7 +1,17 @@ from decimal import Decimal import json +from common.mappings import EventName, Operation +from typing import List +class RecordConfig: + def __init__(self, event_name, operation, imms_id, expected_action_flag=None, supplier="EMIS"): + self.event_name = event_name + self.operation = operation + self.supplier = supplier + self.imms_id = imms_id + self.expected_action_flag = expected_action_flag + class ValuesForTests: MOCK_ENVIRONMENT_DICT = { @@ -118,43 +128,65 @@ class ValuesForTests: json_value_for_test = json.dumps(json_data) @staticmethod - def get_event(event_name="INSERT", operation="CREATE", supplier="EMIS"): - if operation != "REMOVE": - return { - "Records": [ - { - "eventName": event_name, - "dynamodb": { - "ApproximateCreationDateTime": 1690896000, - "NewImage": { - "PK": {"S": "covid#12345"}, - "PatientSK": {"S": "COVID19#ca8ba2c6-2383-4465-b456-c1174c21cf31"}, - "IdentifierPK": {"S": "system#1"}, - "Operation": {"S": operation}, - "SupplierSystem": {"S": supplier}, - "Resource": {"S": ValuesForTests.json_value_for_test}, - }, - }, + def get_event(event_name=EventName.CREATE, operation=Operation.CREATE, supplier="EMIS", imms_id="12345"): + """Create test event for the handler function.""" + return { + "Records": [ + ValuesForTests.get_event_record(imms_id, event_name, operation, supplier) + ] + } + + @staticmethod + def get_multi_record_event(records_config: List[RecordConfig]): + records = [] + for config in records_config: + # Extract values from the config dictionary + imms_id = config.imms_id + event_name = config.event_name + operation = config.operation + supplier = config.supplier + + # Generate record using the provided configuration + records.append( + ValuesForTests.get_event_record( + imms_id=imms_id, + event_name=event_name, + operation=operation, + supplier=supplier, + ) + ) + return {"Records": records} + + @staticmethod + def get_event_record(imms_id, event_name, operation, supplier="EMIS"): + pk = f"covid#{imms_id}" + if operation != Operation.DELETE_PHYSICAL: + return{ + "eventName": event_name, + "dynamodb": { + "ApproximateCreationDateTime": 1690896000, + "NewImage": { + "PK": {"S": pk}, + "PatientSK": {"S": "COVID19#ca8ba2c6-2383-4465-b456-c1174c21cf31"}, + "IdentifierPK": {"S": "system#1"}, + "Operation": {"S": operation}, + "SupplierSystem": {"S": supplier}, + "Resource": {"S": ValuesForTests.json_value_for_test}, } - ] + } } else: return { - "Records": [ - { - "eventName": "REMOVE", - "dynamodb": { - "ApproximateCreationDateTime": 1690896000, - "Keys": { - "PK": {"S": "covid#12345"}, - "PatientSK": {"S": "covid#12345"}, - "SupplierSystem": {"S": "EMIS"}, - "Resource": {"S": ValuesForTests.json_value_for_test}, - "PatientSK": {"S": "COVID19#ca8ba2c6-2383-4465-b456-c1174c21cf31"}, - }, - }, + "eventName": event_name, + "dynamodb": { + "ApproximateCreationDateTime": 1690896000, + "Keys": { + "PK": {"S": pk}, + "PatientSK": {"S": "COVID19#ca8ba2c6-2383-4465-b456-c1174c21cf31"}, + "SupplierSystem": {"S": supplier}, + "Resource": {"S": ValuesForTests.json_value_for_test}, } - ] + } } expected_static_values = { @@ -280,168 +312,6 @@ def get_expected_imms(expected_action_flag): "LOCATION_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code", "CONVERSION_ERRORS": [] } - - @staticmethod - def get_test_data_resource(): - """ - The returned resource includes details about the practitioner, patient, - vaccine code, location, and other relevant fields. - """ - return { - "resourceType": "Immunization", - "contained": [ - { - "resourceType": "Practitioner", - "id": "Pract1", - "name": [ - { - "family": "O'Reilly", - "given": ["Ellena"] - } - ] - }, - { - "resourceType": "Patient", - "id": "Pat1", - "identifier": [ - { - "system": "https://fhir.nhs.uk/Id/nhs-number", - "value": "9674963871" - } - ], - "name": [ - { - "family": "GREIR", - "given": ["SABINA"] - } - ], - "gender": "female", - "birthDate": "2019-01-31", - "address": [ - { - "postalCode": "GU14 6TU" - } - ] - } - ], - "extension": [ - { - "url": "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure", - "valueCodeableConcept": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "1303503001", - "display": - "Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)" - } - ] - } - } - ], - "identifier": [ - { - "system": "https://www.ravs.england.nhs.uk/", - "value": "0001_RSV_v5_RUN_2_CDFDPS-742_valid_dose_1" - } - ], - "status": "completed", - "vaccineCode": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "42605811000001109", - "display": - "Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product)" - } - ] - }, - "patient": { - "reference": "#Pat1" - }, - "occurrenceDateTime": "2024-06-10T18:33:25+00:00", - "recorded": "2024-06-10T18:33:25+00:00", - "primarySource": True, - "manufacturer": { - "display": "Pfizer" - }, - "location": { - "type": "Location", - "identifier": { - "value": "J82067", - "system": "https://fhir.nhs.uk/Id/ods-organization-code" - } - }, - "lotNumber": "RSVTEST", - "expirationDate": "2024-12-31", - "site": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "368208006", - "display": "Left upper arm structure (body structure)" - } - ] - }, - "route": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "78421000", - "display": "Intramuscular route (qualifier value)" - } - ] - }, - "doseQuantity": { - "value": 0.5, - "unit": "Milliliter (qualifier value)", - "system": "http://unitsofmeasure.org", - "code": "258773002" - }, - "performer": [ - { - "actor": { - "reference": "#Pract1" - } - }, - { - "actor": { - "type": "Organization", - "identifier": { - "system": "https://fhir.nhs.uk/Id/ods-organization-code", - "value": "X0X0X" - } - } - } - ], - "reasonCode": [ - { - "coding": [ - { - "code": "Test", - "system": "http://snomed.info/sct" - } - ] - } - ], - "protocolApplied": [ - { - "targetDisease": [ - { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "840539006", - "display": "Disease caused by severe acute respiratory syndrome coronavirus 2" - } - ] - } - ], - "doseNumberPositiveInt": 1 - } - ], - "id": "ca8ba2c6-2383-4465-b456-c1174c21cf31" - } class ErrorValuesForTests: From aa8035fc6f19b0e5f326d601fb393b508d966fdb Mon Sep 17 00:00:00 2001 From: nhsdevws Date: Thu, 15 May 2025 11:14:08 +0100 Subject: [PATCH 2/3] Update Tests --- delta_backend/src/common/mappings.py | 6 +- .../tests/test_convert_to_flat_json.py | 5 -- delta_backend/tests/test_delta.py | 61 +++++++++++++------ 3 files changed, 47 insertions(+), 25 deletions(-) diff --git a/delta_backend/src/common/mappings.py b/delta_backend/src/common/mappings.py index 5f9988cb3c..ad92f9bab3 100644 --- a/delta_backend/src/common/mappings.py +++ b/delta_backend/src/common/mappings.py @@ -1,12 +1,12 @@ """ - Define enums for event names, operations, and action flags. + Enums for event names, operations, and action flags. # case eventName operation actionFlag ----------------- --------- --------- ---------- create INSERT CREATE NEW update MODIFY UPDATE UPDATE - logically delete MODIFY DELETE DELETE - physically delete REMOVE REMOVE N/A + logical delete MODIFY DELETE DELETE + physical delete REMOVE REMOVE N/A """ class EventName(): diff --git a/delta_backend/tests/test_convert_to_flat_json.py b/delta_backend/tests/test_convert_to_flat_json.py index 9667398f7d..6db4d84553 100644 --- a/delta_backend/tests/test_convert_to_flat_json.py +++ b/delta_backend/tests/test_convert_to_flat_json.py @@ -126,11 +126,6 @@ def assert_dynamodb_record(self, operation_flag, action_flag, items, expected_va self.assertIsInstance(imms_data, dict) self.assertGreater(len(imms_data), 0) - for key, expected_value in expected_values.items(): - self.assertIn(key, filtered_items[0], f"{key} is missing") - if (filtered_items[0][key] != expected_value): - print (f"{key} mismatch {filtered_items[0][key]} != {expected_value}") - # Check Imms JSON structure matches exactly self.assertEqual(imms_data, expected_imms, "Imms data does not match expected JSON structure") diff --git a/delta_backend/tests/test_delta.py b/delta_backend/tests/test_delta.py index c4883b3e48..718e140148 100644 --- a/delta_backend/tests/test_delta.py +++ b/delta_backend/tests/test_delta.py @@ -125,21 +125,31 @@ def test_handler_failure(self, mock_boto_resource): @patch("boto3.resource") def test_handler_success_update(self, mock_boto_resource): # Arrange + mock_table = self.setup_mock_dynamodb(mock_boto_resource) self.setup_mock_dynamodb(mock_boto_resource) - event = ValuesForTests.get_event(event_name=EventName.UPDATE, operation=Operation.UPDATE) + imms_id = "test-update-imms-id" + event = ValuesForTests.get_event(event_name=EventName.UPDATE, operation=Operation.UPDATE, imms_id=imms_id) # Act result = handler(event, self.context) # Assert self.assertTrue(result) + mock_table.put_item.assert_called() + self.mock_firehose_logger.send_log.assert_called() # check logged + put_item_call_args = mock_table.put_item.call_args # check data written to DynamoDB + put_item_data = put_item_call_args.kwargs["Item"] + self.assertIn("Imms", put_item_data) + self.assertEqual(put_item_data["Imms"]["ACTION_FLAG"], ActionFlag.UPDATE) + self.assertEqual(put_item_data["Operation"], Operation.UPDATE) + self.assertEqual(put_item_data["ImmsID"], imms_id) @patch("boto3.resource") - def test_handler_success_remove(self, mock_boto_resource): + def test_handler_success_delete_physical(self, mock_boto_resource): # Arrange mock_table = self.setup_mock_dynamodb(mock_boto_resource) imms_id = "test-update-imms-id" - event = ValuesForTests.get_event(event_name=EventName.UPDATE, operation=Operation.UPDATE, imms_id=imms_id) + event = ValuesForTests.get_event(event_name=EventName.DELETE_PHYSICAL, operation=Operation.DELETE_PHYSICAL, imms_id=imms_id) # Act result = handler(event, self.context) @@ -151,8 +161,30 @@ def test_handler_success_remove(self, mock_boto_resource): put_item_call_args = mock_table.put_item.call_args # check data written to DynamoDB put_item_data = put_item_call_args.kwargs["Item"] self.assertIn("Imms", put_item_data) - self.assertEqual(put_item_data["Imms"]["ACTION_FLAG"], ActionFlag.UPDATE) - self.assertEqual(put_item_data["Operation"], Operation.UPDATE) + self.assertEqual(put_item_data["Operation"], Operation.DELETE_PHYSICAL) + self.assertEqual(put_item_data["ImmsID"], imms_id) + self.assertEqual(put_item_data["Imms"], "") # check imms has been blanked out + + @patch("boto3.resource") + def test_handler_success_delete_logical(self, mock_boto_resource): + # Arrange + mock_table = self.setup_mock_dynamodb(mock_boto_resource) + imms_id = "test-update-imms-id" + event = ValuesForTests.get_event(event_name=EventName.UPDATE, + operation=Operation.DELETE_LOGICAL, + imms_id=imms_id) + # Act + result = handler(event, self.context) + + # Assert + self.assertTrue(result) + mock_table.put_item.assert_called() + self.mock_firehose_logger.send_log.assert_called() # check logged + put_item_call_args = mock_table.put_item.call_args # check data written to DynamoDB + put_item_data = put_item_call_args.kwargs["Item"] + self.assertIn("Imms", put_item_data) + self.assertEqual(put_item_data["Imms"]["ACTION_FLAG"], ActionFlag.DELETE_LOGICAL) + self.assertEqual(put_item_data["Operation"], Operation.DELETE_LOGICAL) self.assertEqual(put_item_data["ImmsID"], imms_id) @patch("boto3.resource") @@ -195,7 +227,7 @@ def test_handler_exception_intrusion_check_false(self, mocked_intrusion, mock_bo self.assertFalse(response) - @patch("delta.logger.info") # Mock logging + @patch("delta.logger.info") def test_dps_record_skipped(self, mock_logger_info): event = ValuesForTests.get_event(supplier="DPSFULL") context = {} @@ -245,7 +277,6 @@ def test_send_message_multi_records_diverse(self, mock_boto_resource): RecordConfig(EventName.DELETE_LOGICAL, Operation.DELETE_LOGICAL, "id3", ActionFlag.DELETE_LOGICAL), RecordConfig(EventName.DELETE_PHYSICAL, Operation.DELETE_PHYSICAL, "id4"), ] - # Generate the event using ValuesForTests.get_multi_record_event event = ValuesForTests.get_multi_record_event(records_config) # Act @@ -266,7 +297,6 @@ def test_send_message_multi_create(self, mock_boto_resource): RecordConfig(EventName.CREATE, Operation.CREATE, "create-id2", ActionFlag.CREATE), RecordConfig(EventName.CREATE, Operation.CREATE, "create-id3", ActionFlag.CREATE) ] - # Generate the event using ValuesForTests.get_multi_record_event event = ValuesForTests.get_multi_record_event(records_config) # Act @@ -288,7 +318,6 @@ def test_send_message_multi_update(self, mock_boto_resource): RecordConfig(EventName.UPDATE, Operation.UPDATE, "update-id2", ActionFlag.UPDATE), RecordConfig(EventName.UPDATE, Operation.UPDATE, "update-id3", ActionFlag.UPDATE) ] - # Generate the event using ValuesForTests.get_multi_record_event event = ValuesForTests.get_multi_record_event(records_config) # Act @@ -305,11 +334,10 @@ def test_send_message_multi_logical_delete(self, mock_boto_resource): mock_table = self.setup_mock_dynamodb(mock_boto_resource) records_config = [ - RecordConfig(EventName.DELETE_LOGICAL, Operation.DELETE_LOGICAL, "update-id1", ActionFlag.DELETE_LOGICAL), - RecordConfig(EventName.DELETE_LOGICAL, Operation.DELETE_LOGICAL, "update-id2", ActionFlag.DELETE_LOGICAL), - RecordConfig(EventName.DELETE_LOGICAL, Operation.DELETE_LOGICAL, "update-id3", ActionFlag.DELETE_LOGICAL) + RecordConfig(EventName.DELETE_LOGICAL, Operation.DELETE_LOGICAL, "delete-id1", ActionFlag.DELETE_LOGICAL), + RecordConfig(EventName.DELETE_LOGICAL, Operation.DELETE_LOGICAL, "delete-id2", ActionFlag.DELETE_LOGICAL), + RecordConfig(EventName.DELETE_LOGICAL, Operation.DELETE_LOGICAL, "delete-id3", ActionFlag.DELETE_LOGICAL) ] - # Generate the event using ValuesForTests.get_multi_record_event event = ValuesForTests.get_multi_record_event(records_config) # Act @@ -326,11 +354,10 @@ def test_send_message_multi_physical_delete(self, mock_boto_resource): mock_table = self.setup_mock_dynamodb(mock_boto_resource) records_config = [ - RecordConfig(EventName.DELETE_PHYSICAL, Operation.DELETE_PHYSICAL, "update-id1"), - RecordConfig(EventName.DELETE_PHYSICAL, Operation.DELETE_PHYSICAL, "update-id2"), - RecordConfig(EventName.DELETE_PHYSICAL, Operation.DELETE_PHYSICAL, "update-id3") + RecordConfig(EventName.DELETE_PHYSICAL, Operation.DELETE_PHYSICAL, "remove-id1"), + RecordConfig(EventName.DELETE_PHYSICAL, Operation.DELETE_PHYSICAL, "remove-id2"), + RecordConfig(EventName.DELETE_PHYSICAL, Operation.DELETE_PHYSICAL, "remove-id3") ] - # Generate the event using ValuesForTests.get_multi_record_event event = ValuesForTests.get_multi_record_event(records_config) # Act From 5caa193ae2d7f9d55de08a4d0fec23a4e7d9f872 Mon Sep 17 00:00:00 2001 From: nhsdevws Date: Thu, 15 May 2025 12:38:10 +0100 Subject: [PATCH 3/3] Resolve Conflicts --- delta_backend/tests/test_convert_to_flat_json.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/delta_backend/tests/test_convert_to_flat_json.py b/delta_backend/tests/test_convert_to_flat_json.py index 97c3960ef9..4eebd6c701 100644 --- a/delta_backend/tests/test_convert_to_flat_json.py +++ b/delta_backend/tests/test_convert_to_flat_json.py @@ -8,6 +8,7 @@ from SchemaParser import SchemaParser from Converter import Converter from ConversionChecker import ConversionChecker +from common.mappings import ActionFlag, Operation, EventName import ExceptionMessages MOCK_ENV_VARS = { @@ -73,7 +74,7 @@ def tearDown(self): self.mock_firehose_logger.stop() @staticmethod - def get_event(event_name="INSERT", operation="operation", supplier="EMIS"): + def get_event(event_name=EventName.CREATE, operation="operation", supplier="EMIS"): """Returns test event data.""" return ValuesForTests.get_event(event_name, operation, supplier) @@ -83,8 +84,7 @@ def assert_dynamodb_record(self, operation_flag, action_flag, items, expected_va Ignores dynamically generated fields like PK, DateTimeStamp, and ExpiresAt. Ensures that the 'Imms' field matches exactly. """ - self.assertEqual(response["statusCode"], 200) - self.assertEqual(response["body"], "Records processed successfully") + self.assertTrue(response) filtered_items = [ {k: v for k, v in item.items() if k not in ["PK", "DateTimeStamp", "ExpiresAt"]} @@ -140,9 +140,9 @@ def test_fhir_converter_json_error_scenario(self): def test_handler_imms_convert_to_flat_json(self): """Test that the Imms field contains the correct flat JSON data for CREATE, UPDATE, and DELETE operations.""" expected_action_flags = [ - {"Operation": "CREATE", "EXPECTED_ACTION_FLAG": "NEW"}, - {"Operation": "UPDATE", "EXPECTED_ACTION_FLAG": "UPDATE"}, - {"Operation": "DELETE", "EXPECTED_ACTION_FLAG": "DELETE"}, + {"Operation": Operation.CREATE, "EXPECTED_ACTION_FLAG": ActionFlag.CREATE}, + {"Operation": Operation.UPDATE, "EXPECTED_ACTION_FLAG": ActionFlag.UPDATE}, + {"Operation": Operation.DELETE_LOGICAL, "EXPECTED_ACTION_FLAG": ActionFlag.DELETE_LOGICAL}, ] for test_case in expected_action_flags: @@ -546,4 +546,4 @@ def clear_table(self): items = result.get("Items", []) if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main()