From e93785a11f577d07e0764b13e7e8787ecddec9e9 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Wed, 5 Nov 2025 16:15:44 +0000 Subject: [PATCH 01/22] [PRMP-540] - create sqs metadata for expedite files --- .../bulk_upload_metadata_processor_handler.py | 34 +++++++++++++++++++ .../bulk_upload_metadata_processor_service.py | 9 +++++ 2 files changed, 43 insertions(+) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index 8f08d6621..a294c1a7e 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -1,3 +1,7 @@ +import os.path +from pathlib import Path +import urllib.parse + from enums.lloyd_george_pre_process_format import LloydGeorgePreProcessFormat from services.bulk_upload.metadata_general_preprocessor import ( MetadataGeneralPreprocessor, @@ -8,11 +12,13 @@ from services.bulk_upload_metadata_processor_service import ( BulkUploadMetadataProcessorService, ) +from unit.services.test_bulk_upload_metadata_service import metadata_service from utils.audit_logging_setup import LoggingService from utils.decorators.ensure_env_var import ensure_environment_variables from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions from utils.decorators.override_error_check import override_error_check from utils.decorators.set_audit_arg import set_request_context_for_logging +from utils.exceptions import BulkUploadMetadataException logger = LoggingService(__name__) @@ -24,6 +30,34 @@ ) @handle_lambda_exceptions def lambda_handler(event, _context): + if "Records" in event and event["Records"][0].get("eventSource") == "aws:s3": + logger.info("Triggered by S3 listener...") + try: + key_string = event["Records"][0]["s3"]["object"]["key"] + key = urllib.parse.unquote_plus(key_string, encoding="utf-8") + if key.startswith("expedite/"): + logger.info("Processing file from expedite folder") + filepath = os.path.basename(key) + raw_pre_format_type = event.get( + "preFormatType", LloydGeorgePreProcessFormat.GENERAL + ) + formatter_service_class = get_formatter_service(raw_pre_format_type) + metadata_formatter_service = formatter_service_class("expedite") + filename = metadata_formatter_service.validate_record_filename(key) + ods_code = Path(key).parent.name # folder name containing file + + metadata_service = BulkUploadMetadataProcessorService(metadata_formatter_service) + metadata_service.create_sqs_metadata(filepath, filename, ods_code, scan_date) + + else: + failure_msg = "Failed due to unrecognized S3 listener event key" + logger.error(failure_msg) + raise BulkUploadMetadataException(failure_msg) + except KeyError as e: + failure_msg = f"Failed due to missing key: {str(e)}" + logger.error(failure_msg) + raise BulkUploadMetadataException(failure_msg) + practice_directory = event.get("practiceDirectory", "") raw_pre_format_type = event.get( diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index 653d4d01e..6240a05ca 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -139,6 +139,15 @@ def convert_to_sqs_metadata( **file.model_dump(), stored_file_name=stored_file_name ) + def create_sqs_metadata(self, file_path: str, stored_file_name: str, ods_code: str, scan_date) -> StagingSqsMetadata: + # return BulkUploadQueueMetadata(file_path=file_path, stored_file_name=stored_file_name, gp_practice_code=ods_code, scan_date=scan_date) + + return StagingSqsMetadata( + nhs_number=nhs_number, + files=[BulkUploadQueueMetadata(file_path=file_path, stored_file_name=stored_file_name, gp_practice_code=ods_code, scan_date=scan_date)], + ) + + def extract_patient_info(self, file_metadata: MetadataFile) -> tuple[str, str]: nhs_number = file_metadata.nhs_number ods_code = file_metadata.gp_practice_code From 4920d17b497aae5e1caac0e3ca06a876ae139337 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Wed, 19 Nov 2025 09:10:37 +0000 Subject: [PATCH 02/22] [PRMP-540] - Validate expedite event and file --- .../bulk_upload_metadata_processor_handler.py | 45 +++-------- .../bulk_upload_metadata_processor_service.py | 49 ++++++++++-- ..._bulk_upload_metadata_processor_handler.py | 20 +++++ ..._bulk_upload_metadata_processor_service.py | 76 +++++++++++++++++++ 4 files changed, 149 insertions(+), 41 deletions(-) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index a294c1a7e..993ab5e53 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -1,7 +1,3 @@ -import os.path -from pathlib import Path -import urllib.parse - from enums.lloyd_george_pre_process_format import LloydGeorgePreProcessFormat from services.bulk_upload.metadata_general_preprocessor import ( MetadataGeneralPreprocessor, @@ -12,13 +8,11 @@ from services.bulk_upload_metadata_processor_service import ( BulkUploadMetadataProcessorService, ) -from unit.services.test_bulk_upload_metadata_service import metadata_service from utils.audit_logging_setup import LoggingService from utils.decorators.ensure_env_var import ensure_environment_variables from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions from utils.decorators.override_error_check import override_error_check from utils.decorators.set_audit_arg import set_request_context_for_logging -from utils.exceptions import BulkUploadMetadataException logger = LoggingService(__name__) @@ -30,34 +24,6 @@ ) @handle_lambda_exceptions def lambda_handler(event, _context): - if "Records" in event and event["Records"][0].get("eventSource") == "aws:s3": - logger.info("Triggered by S3 listener...") - try: - key_string = event["Records"][0]["s3"]["object"]["key"] - key = urllib.parse.unquote_plus(key_string, encoding="utf-8") - if key.startswith("expedite/"): - logger.info("Processing file from expedite folder") - filepath = os.path.basename(key) - raw_pre_format_type = event.get( - "preFormatType", LloydGeorgePreProcessFormat.GENERAL - ) - formatter_service_class = get_formatter_service(raw_pre_format_type) - metadata_formatter_service = formatter_service_class("expedite") - filename = metadata_formatter_service.validate_record_filename(key) - ods_code = Path(key).parent.name # folder name containing file - - metadata_service = BulkUploadMetadataProcessorService(metadata_formatter_service) - metadata_service.create_sqs_metadata(filepath, filename, ods_code, scan_date) - - else: - failure_msg = "Failed due to unrecognized S3 listener event key" - logger.error(failure_msg) - raise BulkUploadMetadataException(failure_msg) - except KeyError as e: - failure_msg = f"Failed due to missing key: {str(e)}" - logger.error(failure_msg) - raise BulkUploadMetadataException(failure_msg) - practice_directory = event.get("practiceDirectory", "") raw_pre_format_type = event.get( @@ -65,6 +31,17 @@ def lambda_handler(event, _context): ) formatter_service_class = get_formatter_service(raw_pre_format_type) + + if "source" in event and event.get("source") == "aws.s3": + logger.info("Handling EventBridge event from S3") + metadata_formatter_service = formatter_service_class("expedite") + metadata_service = BulkUploadMetadataProcessorService( + metadata_formatter_service + ) + metadata_service.handle_expedite_event(event) + logger.info("Successfully processed expedite event") + return + if not practice_directory: logger.error( "Failed to start metadata processing due to missing practice directory" diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index 6240a05ca..92e08b1d0 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -2,9 +2,11 @@ import os import shutil import tempfile +import urllib.parse import uuid from collections import defaultdict from datetime import datetime +from pathlib import Path from typing import Iterable import pydantic @@ -30,6 +32,7 @@ InvalidFileNameException, LGInvalidFilesException, ) +from utils.filename_utils import extract_nhs_number_from_bulk_upload_file_name from utils.lloyd_george_validator import validate_file_name logger = LoggingService(__name__) @@ -139,14 +142,21 @@ def convert_to_sqs_metadata( **file.model_dump(), stored_file_name=stored_file_name ) - def create_sqs_metadata(self, file_path: str, stored_file_name: str, ods_code: str, scan_date) -> StagingSqsMetadata: - # return BulkUploadQueueMetadata(file_path=file_path, stored_file_name=stored_file_name, gp_practice_code=ods_code, scan_date=scan_date) - + def create_sqs_metadata( + self, file_path: str, stored_file_name: str, ods_code: str, scan_date + ) -> StagingSqsMetadata: + nhs_number = extract_nhs_number_from_bulk_upload_file_name(file_path)[0] return StagingSqsMetadata( - nhs_number=nhs_number, - files=[BulkUploadQueueMetadata(file_path=file_path, stored_file_name=stored_file_name, gp_practice_code=ods_code, scan_date=scan_date)], - ) - + nhs_number=nhs_number, + files=[ + BulkUploadQueueMetadata( + file_path=file_path, + stored_file_name=stored_file_name, + gp_practice_code=ods_code, + scan_date=scan_date, + ) + ], + ) def extract_patient_info(self, file_metadata: MetadataFile) -> tuple[str, str]: nhs_number = file_metadata.nhs_number @@ -167,6 +177,31 @@ def validate_and_correct_filename( return valid_filepath + def handle_expedite_event(self, event): + try: + key_string = event["detail"]["object"]["key"] + key = urllib.parse.unquote_plus(key_string, encoding="utf-8") + if key.startswith("expedite/"): + logger.info("Processing file from expedite folder") + filepath = os.path.basename(key) + filename = self.metadata_formatter_service.validate_record_filename(key) + ods_code = Path(key).parent.name + scan_date = datetime.now().strftime("%d%m%Y") + + sqs_metadata = [ + self.create_sqs_metadata(filepath, filename, ods_code, scan_date) + ] + self.send_metadata_to_fifo_sqs(sqs_metadata) + return + else: + failure_msg = f"Unexpected directory or file location received from EventBridge: {key_string}" + logger.error(failure_msg) + raise BulkUploadMetadataException(failure_msg) + except KeyError as e: + failure_msg = f"Failed due to missing key: {str(e)}" + logger.error(failure_msg) + raise BulkUploadMetadataException(failure_msg) + def handle_invalid_filename( self, file_metadata: MetadataFile, diff --git a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py index b7029f5be..245ede285 100644 --- a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py @@ -28,3 +28,23 @@ def test_metadata_processor_lambda_handler_empty_event( lambda_handler({}, context) mock_metadata_service.process_metadata.assert_not_called() + + +def test_metadata_processor_lambda_handler_s3_event_uses_general_by_default( + set_env, context, mocker, mock_metadata_service +): + # Patch the default formatter (GENERAL) to observe instantiation + mocked_general_cls = mocker.patch( + "handlers.bulk_upload_metadata_processor_handler.MetadataGeneralPreprocessor" + ) + + event = {"source": "aws.s3", "detail": {"object": {"key": "expedite/XYZ"}}} + + lambda_handler(event, context) + + # Should not trigger the normal processing path + mock_metadata_service.process_metadata.assert_not_called() + # Should trigger expedite handling on the service + mock_metadata_service.handle_expedite_event.assert_called_once_with(event) + # Formatter should be instantiated with 'expedite' + mocked_general_cls.assert_called_once_with("expedite") diff --git a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py index 0f5ccd421..e450a010c 100644 --- a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py +++ b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py @@ -589,3 +589,79 @@ def test_validate_and_correct_filename_sad_path( base_metadata_file.file_path ) assert result == "corrected/path/file_corrected.pdf" + + + +def test_create_sqs_metadata_builds_expected_structure(test_service): + file_path = "1234567890_record.pdf" + stored_file_name = "expedite/A12345/1234567890_record.pdf" + ods_code = "A12345" + scan_date = "03022025" + + result = test_service.create_sqs_metadata( + file_path=file_path, + stored_file_name=stored_file_name, + ods_code=ods_code, + scan_date=scan_date, + ) + + assert result.nhs_number == "1234567890" + assert len(result.files) == 1 + item = result.files[0] + assert item.file_path == file_path + assert item.stored_file_name == stored_file_name + assert item.gp_practice_code == ods_code + assert item.scan_date == scan_date + + +@freeze_time("2025-02-03T10:00:00") +def test_handle_expedite_event_happy_path_sends_sqs(test_service, mocker): + ods = "A12345" + key = f"expedite/{ods}/1234567890_record.pdf" + event = {"detail": {"object": {"key": key}}} + + mocked_send = mocker.patch.object( + BulkUploadMetadataProcessorService, "send_metadata_to_fifo_sqs" + ) + + test_service.handle_expedite_event(event) + + mocked_send.assert_called_once() + args, _ = mocked_send.call_args + assert len(args) == 1 + sqs_payload_list = args[0] + sqs_payload = sqs_payload_list[0] + assert sqs_payload.nhs_number == "1234567890" + assert len(sqs_payload.files) == 1 + file_item = sqs_payload.files[0] + assert file_item.file_path == "1234567890_record.pdf" + assert file_item.stored_file_name == key + assert file_item.gp_practice_code == ods + assert file_item.scan_date == "03022025" + + +def test_handle_expedite_event_invalid_directory_raises(test_service, mocker): + mocked_send = mocker.patch.object( + BulkUploadMetadataProcessorService, "send_metadata_to_fifo_sqs" + ) + bad_key = "notexpedite/A12345/1234567890_record.pdf" + event = {"detail": {"object": {"key": bad_key}}} + + with pytest.raises(BulkUploadMetadataException) as exc: + test_service.handle_expedite_event(event) + + assert "Unexpected directory or file location" in str(exc.value) + mocked_send.assert_not_called() + + +def test_handle_expedite_event_missing_key_raises(test_service, mocker): + mocked_send = mocker.patch.object( + BulkUploadMetadataProcessorService, "send_metadata_to_fifo_sqs" + ) + event = {"detail": {}} + + with pytest.raises(BulkUploadMetadataException) as exc: + test_service.handle_expedite_event(event) + + assert "Failed due to missing key" in str(exc.value) + mocked_send.assert_not_called() \ No newline at end of file From 17ca80067a625bdbedfb931f1762f5fd2bb33e6d Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Wed, 19 Nov 2025 11:07:22 +0000 Subject: [PATCH 03/22] [PRMP-540] - Correct scan_date format --- lambdas/services/bulk_upload_metadata_processor_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index 92e08b1d0..2568395d5 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -186,7 +186,7 @@ def handle_expedite_event(self, event): filepath = os.path.basename(key) filename = self.metadata_formatter_service.validate_record_filename(key) ods_code = Path(key).parent.name - scan_date = datetime.now().strftime("%d%m%Y") + scan_date = datetime.now().strftime("%Y%m%d") sqs_metadata = [ self.create_sqs_metadata(filepath, filename, ods_code, scan_date) From 965245d64d53aace15ef10519fe1325a819fb0fe Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Wed, 19 Nov 2025 11:24:07 +0000 Subject: [PATCH 04/22] [PRMP-540] - Refactor handle_expedite_event() --- lambdas/services/bulk_upload_metadata_processor_service.py | 2 +- .../services/test_bulk_upload_metadata_processor_service.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index 2568395d5..a534eaa01 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -186,7 +186,7 @@ def handle_expedite_event(self, event): filepath = os.path.basename(key) filename = self.metadata_formatter_service.validate_record_filename(key) ods_code = Path(key).parent.name - scan_date = datetime.now().strftime("%Y%m%d") + scan_date = datetime.now().strftime("%Y-%m-%d") sqs_metadata = [ self.create_sqs_metadata(filepath, filename, ods_code, scan_date) diff --git a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py index e450a010c..409e7bd53 100644 --- a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py +++ b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py @@ -637,7 +637,7 @@ def test_handle_expedite_event_happy_path_sends_sqs(test_service, mocker): assert file_item.file_path == "1234567890_record.pdf" assert file_item.stored_file_name == key assert file_item.gp_practice_code == ods - assert file_item.scan_date == "03022025" + assert file_item.scan_date == "2025-02-03" def test_handle_expedite_event_invalid_directory_raises(test_service, mocker): From e1e8365a6a8314610678b14d3b760a890c9980a4 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Wed, 19 Nov 2025 11:24:34 +0000 Subject: [PATCH 05/22] [PRMP-540] - Refactor handle_expedite_event() --- .../bulk_upload_metadata_processor_service.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index a534eaa01..33511338c 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -142,10 +142,10 @@ def convert_to_sqs_metadata( **file.model_dump(), stored_file_name=stored_file_name ) + @staticmethod def create_sqs_metadata( - self, file_path: str, stored_file_name: str, ods_code: str, scan_date + nhs_number: str, file_path: str, stored_file_name: str, ods_code: str, scan_date ) -> StagingSqsMetadata: - nhs_number = extract_nhs_number_from_bulk_upload_file_name(file_path)[0] return StagingSqsMetadata( nhs_number=nhs_number, files=[ @@ -183,13 +183,13 @@ def handle_expedite_event(self, event): key = urllib.parse.unquote_plus(key_string, encoding="utf-8") if key.startswith("expedite/"): logger.info("Processing file from expedite folder") - filepath = os.path.basename(key) - filename = self.metadata_formatter_service.validate_record_filename(key) + file_path = os.path.basename(key) + file_name = self.metadata_formatter_service.validate_record_filename(key) ods_code = Path(key).parent.name scan_date = datetime.now().strftime("%Y-%m-%d") - + nhs_number = extract_nhs_number_from_bulk_upload_file_name(file_path)[0] sqs_metadata = [ - self.create_sqs_metadata(filepath, filename, ods_code, scan_date) + self.create_sqs_metadata(nhs_number, file_path, file_name, ods_code, scan_date) ] self.send_metadata_to_fifo_sqs(sqs_metadata) return From a61a3a6e4c73fe5d8917b42de10527bc8d512292 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Wed, 19 Nov 2025 11:39:59 +0000 Subject: [PATCH 06/22] [PRMP-540] - Fix unit tests --- .../services/test_bulk_upload_metadata_processor_service.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py index 409e7bd53..eaa64dc31 100644 --- a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py +++ b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py @@ -593,12 +593,14 @@ def test_validate_and_correct_filename_sad_path( def test_create_sqs_metadata_builds_expected_structure(test_service): + nhs_number = "1234567890" file_path = "1234567890_record.pdf" stored_file_name = "expedite/A12345/1234567890_record.pdf" ods_code = "A12345" scan_date = "03022025" result = test_service.create_sqs_metadata( + nhs_number=nhs_number, file_path=file_path, stored_file_name=stored_file_name, ods_code=ods_code, From 138540aa4dfb264f6fa54834f48ead446718f5bc Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Wed, 19 Nov 2025 12:00:41 +0000 Subject: [PATCH 07/22] [PRMP-540] - Refactor handle_expedite_event() --- .../handlers/bulk_upload_metadata_processor_handler.py | 1 - .../services/bulk_upload_metadata_processor_service.py | 8 ++++---- .../test_bulk_upload_metadata_processor_service.py | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index 993ab5e53..8a6217b8b 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -39,7 +39,6 @@ def lambda_handler(event, _context): metadata_formatter_service ) metadata_service.handle_expedite_event(event) - logger.info("Successfully processed expedite event") return if not practice_directory: diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index 33511338c..b5fbf867c 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -65,7 +65,6 @@ def process_metadata(self): logger.info("Finished parsing metadata") self.send_metadata_to_fifo_sqs(staging_metadata_list) - logger.info("Sent bulk upload metadata to sqs queue") self.copy_metadata_to_dated_folder() @@ -184,15 +183,15 @@ def handle_expedite_event(self, event): if key.startswith("expedite/"): logger.info("Processing file from expedite folder") file_path = os.path.basename(key) + nhs_number = extract_nhs_number_from_bulk_upload_file_name(file_path)[0] file_name = self.metadata_formatter_service.validate_record_filename(key) ods_code = Path(key).parent.name scan_date = datetime.now().strftime("%Y-%m-%d") - nhs_number = extract_nhs_number_from_bulk_upload_file_name(file_path)[0] sqs_metadata = [ - self.create_sqs_metadata(nhs_number, file_path, file_name, ods_code, scan_date) + self.create_sqs_metadata(nhs_number, file_name, file_name, ods_code, scan_date) ] self.send_metadata_to_fifo_sqs(sqs_metadata) - return + logger.info("Successfully processed expedite event") else: failure_msg = f"Unexpected directory or file location received from EventBridge: {key_string}" logger.error(failure_msg) @@ -237,6 +236,7 @@ def send_metadata_to_fifo_sqs( nhs_number=nhs_number, group_id=sqs_group_id, ) + logger.info("Sent bulk upload metadata to sqs queue") def copy_metadata_to_dated_folder(self): logger.info("Copying metadata CSV to dated folder") diff --git a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py index eaa64dc31..33c314d1a 100644 --- a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py +++ b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py @@ -636,7 +636,7 @@ def test_handle_expedite_event_happy_path_sends_sqs(test_service, mocker): assert sqs_payload.nhs_number == "1234567890" assert len(sqs_payload.files) == 1 file_item = sqs_payload.files[0] - assert file_item.file_path == "1234567890_record.pdf" + assert file_item.file_path == key assert file_item.stored_file_name == key assert file_item.gp_practice_code == ods assert file_item.scan_date == "2025-02-03" From 7df91ebe3a2130f9379622e291e9dc9d1fefe5b4 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Thu, 20 Nov 2025 09:04:26 +0000 Subject: [PATCH 08/22] [PRMP-540] - Raise exception if file not 1of1 --- .../bulk_upload_metadata_processor_service.py | 24 ++++++++++++++++--- ..._bulk_upload_metadata_processor_service.py | 17 +++++++++++-- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index b5fbf867c..c7a949378 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -32,7 +32,10 @@ InvalidFileNameException, LGInvalidFilesException, ) -from utils.filename_utils import extract_nhs_number_from_bulk_upload_file_name +from utils.filename_utils import ( + extract_document_number_bulk_upload_file_name, + extract_nhs_number_from_bulk_upload_file_name, +) from utils.lloyd_george_validator import validate_file_name logger = LoggingService(__name__) @@ -183,12 +186,27 @@ def handle_expedite_event(self, event): if key.startswith("expedite/"): logger.info("Processing file from expedite folder") file_path = os.path.basename(key) + first_document_number, second_document_number, current_file_name = ( + extract_document_number_bulk_upload_file_name(file_path) + ) + + if (first_document_number, second_document_number) != (1, 1): + failure_msg = ( + "Failed processing expedite event due to file not being a 1of1" + ) + logger.error(failure_msg) + raise BulkUploadMetadataException(failure_msg) + nhs_number = extract_nhs_number_from_bulk_upload_file_name(file_path)[0] - file_name = self.metadata_formatter_service.validate_record_filename(key) + file_name = self.metadata_formatter_service.validate_record_filename( + key + ) ods_code = Path(key).parent.name scan_date = datetime.now().strftime("%Y-%m-%d") sqs_metadata = [ - self.create_sqs_metadata(nhs_number, file_name, file_name, ods_code, scan_date) + self.create_sqs_metadata( + nhs_number, file_name, file_name, ods_code, scan_date + ) ] self.send_metadata_to_fifo_sqs(sqs_metadata) logger.info("Successfully processed expedite event") diff --git a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py index 33c314d1a..bd3c8706f 100644 --- a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py +++ b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py @@ -619,7 +619,7 @@ def test_create_sqs_metadata_builds_expected_structure(test_service): @freeze_time("2025-02-03T10:00:00") def test_handle_expedite_event_happy_path_sends_sqs(test_service, mocker): ods = "A12345" - key = f"expedite/{ods}/1234567890_record.pdf" + key = f"expedite/{ods}/1of1_1234567890_record.pdf" event = {"detail": {"object": {"key": key}}} mocked_send = mocker.patch.object( @@ -666,4 +666,17 @@ def test_handle_expedite_event_missing_key_raises(test_service, mocker): test_service.handle_expedite_event(event) assert "Failed due to missing key" in str(exc.value) - mocked_send.assert_not_called() \ No newline at end of file + mocked_send.assert_not_called() + +def test_handle_expedite_event_rejects_non_1of1(test_service, mocker): + mocked_send = mocker.patch.object( + BulkUploadMetadataProcessorService, "send_metadata_to_fifo_sqs" + ) + key = "expedite/A12345/2of3_1234567890_record.pdf" + event = {"detail": {"object": {"key": key}}} + + with pytest.raises(BulkUploadMetadataException) as exc: + test_service.handle_expedite_event(event) + + assert "not being a 1of1" in str(exc.value) + mocked_send.assert_not_called() From 1f535a03af499f50e2765cc1be138180973078de Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Thu, 20 Nov 2025 09:40:26 +0000 Subject: [PATCH 09/22] [PRMP-540] - Refactor --- .../bulk_upload_metadata_processor_handler.py | 22 ------------------- .../bulk_upload_metadata_processor_service.py | 2 +- ..._bulk_upload_metadata_processor_handler.py | 4 ---- 3 files changed, 1 insertion(+), 27 deletions(-) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index aa8d3b034..7d2fc58a5 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -27,11 +27,6 @@ ) @handle_lambda_exceptions def lambda_handler(event, _context): - if "source" in event and event.get("source") == "aws.s3": - logger.info("Handling EventBridge event from S3") - handle_expedite_event(event) - return - practice_directory = event.get("practiceDirectory", "") raw_pre_format_type = event.get( "preFormatType", LloydGeorgePreProcessFormat.GENERAL @@ -81,20 +76,3 @@ def get_formatter_service(raw_pre_format_type): f"Invalid preFormatType: '{raw_pre_format_type}', defaulting to {LloydGeorgePreProcessFormat.GENERAL}." ) return MetadataGeneralPreprocessor - - -def handle_expedite_event(event): - try: - key_string = event["detail"]["object"]["key"] - key = urllib.parse.unquote_plus(key_string, encoding="utf-8") - if key.startswith("expedite/"): - logger.info("Processing file from expedite folder") - return # To be added upon by ticket PRMP-540 - else: - failure_msg = f"Unexpected directory or file location received from EventBridge: {key_string}" - logger.error(failure_msg) - raise BulkUploadMetadataException(failure_msg) - except KeyError as e: - failure_msg = f"Failed due to missing key: {str(e)}" - logger.error(failure_msg) - raise BulkUploadMetadataException(failure_msg) diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index e759050f7..509fed4e7 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -292,7 +292,7 @@ def send_metadata_to_fifo_sqs( nhs_number=nhs_number, group_id=sqs_group_id, ) - logger.info("Sent bulk upload metadata to sqs queue") + logger.info("Sent bulk upload metadata to sqs queue") def copy_metadata_to_dated_folder(self): """Copy processed metadata CSV into a dated archive folder in S3.""" diff --git a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py index 847db49d3..0ffec2fec 100644 --- a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py @@ -44,7 +44,6 @@ def test_metadata_processor_lambda_handler_empty_event( def test_metadata_processor_lambda_handler_s3_event_uses_general_by_default( set_env, context, mocker, mock_metadata_service ): - # Patch the default formatter (GENERAL) to observe instantiation mocked_general_cls = mocker.patch( "handlers.bulk_upload_metadata_processor_handler.MetadataGeneralPreprocessor" ) @@ -53,11 +52,8 @@ def test_metadata_processor_lambda_handler_s3_event_uses_general_by_default( lambda_handler(event, context) - # Should not trigger the normal processing path mock_metadata_service.process_metadata.assert_not_called() - # Should trigger expedite handling on the service mock_metadata_service.handle_expedite_event.assert_called_once_with(event) - # Formatter should be instantiated with 'expedite' mocked_general_cls.assert_called_once_with("expedite") From 0b7bde323ddee098b78ac95fcd042d50ffef20a5 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Thu, 20 Nov 2025 09:50:26 +0000 Subject: [PATCH 10/22] [PRMP-540] - Fix unit tests and formatting --- .../bulk_upload_metadata_processor_handler.py | 2 - .../bulk_upload_metadata_processor_service.py | 3 +- ..._bulk_upload_metadata_processor_handler.py | 37 ++++--------------- ..._bulk_upload_metadata_processor_service.py | 2 + 4 files changed, 10 insertions(+), 34 deletions(-) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index 7d2fc58a5..689c89a3e 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -1,4 +1,3 @@ -import urllib.parse from enums.lloyd_george_pre_process_format import LloydGeorgePreProcessFormat from services.bulk_upload.metadata_general_preprocessor import ( @@ -15,7 +14,6 @@ from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions from utils.decorators.override_error_check import override_error_check from utils.decorators.set_audit_arg import set_request_context_for_logging -from utils.exceptions import BulkUploadMetadataException logger = LoggingService(__name__) diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index 509fed4e7..c7508cf46 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -7,7 +7,6 @@ from collections import defaultdict from datetime import datetime from pathlib import Path -from typing import Iterable import pydantic from botocore.exceptions import ClientError @@ -201,7 +200,7 @@ def create_sqs_metadata( ) ], ) - + @staticmethod def extract_patient_info(file_metadata: MetadataFile) -> tuple[str, str]: """Extract key patient identifiers.""" diff --git a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py index 0ffec2fec..26cf3c3c2 100644 --- a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py @@ -18,10 +18,10 @@ def eventbridge_event_with_s3_key(key: str): return { "source": "aws.s3", "detail": { - "object":{ - "key": key, - }, - } + "object": { + "key": key, + }, + }, } @@ -56,36 +56,13 @@ def test_metadata_processor_lambda_handler_s3_event_uses_general_by_default( mock_metadata_service.handle_expedite_event.assert_called_once_with(event) mocked_general_cls.assert_called_once_with("expedite") - -def test_s3_event_with_expedite_key_processes( - set_env, context, mock_metadata_service, caplog -): + +def test_s3_event_with_expedite_key_processes(set_env, context, mock_metadata_service): event = eventbridge_event_with_s3_key( "expedite%2F1of1_Lloyd_George_Record_[John Michael SMITH]_[1234567890]_[15-05-1990].pdf" ) - lambda_handler(event, context) - - assert any( - f"Handling EventBridge event from S3" - in r.message - for r in caplog.records - ) - assert any( - "Processing file from expedite folder" in r.message for r in caplog.records - ) - - -def test_s3_event_with_non_expedite_key_is_rejected( - set_env, context, mock_metadata_service, caplog -): - key_string = "uploads/1of1_Lloyd_George_Record_[John Michael SMITH]_[1234567890]_[15-05-1990].pdf" - event = eventbridge_event_with_s3_key(key_string) lambda_handler(event, context) - assert any( - f"Unexpected directory or file location received from EventBridge: {key_string}" - in r.message - for r in caplog.records - ) mock_metadata_service.process_metadata.assert_not_called() + mock_metadata_service.handle_expedite_event.assert_called_once_with(event) diff --git a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py index 087601592..b68d9f521 100644 --- a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py +++ b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py @@ -672,6 +672,7 @@ def test_handle_expedite_event_missing_key_raises(test_service, mocker): assert "Failed due to missing key" in str(exc.value) mocked_send.assert_not_called() + def test_handle_expedite_event_rejects_non_1of1(test_service, mocker): mocked_send = mocker.patch.object( BulkUploadMetadataProcessorService, "send_metadata_to_fifo_sqs" @@ -685,6 +686,7 @@ def test_handle_expedite_event_rejects_non_1of1(test_service, mocker): assert "not being a 1of1" in str(exc.value) mocked_send.assert_not_called() + @pytest.fixture def mock_csv_content(): header = "FILEPATH,PAGE COUNT,GP-PRACTICE-CODE,NHS-NO,SECTION,SUB-SECTION,SCAN-DATE,SCAN-ID,USER-ID,UPLOAD" From 222b4b1c264579984706b8187d6ef17278b9bf27 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Thu, 20 Nov 2025 09:56:00 +0000 Subject: [PATCH 11/22] [PRMP-540] - Remove unused return value --- lambdas/services/bulk_upload_metadata_processor_service.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index c7508cf46..85ca6ff64 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -216,8 +216,6 @@ def validate_and_correct_filename(self, file_metadata: MetadataFile) -> str: file_metadata.file_path ) - return valid_filepath - def handle_expedite_event(self, event): try: key_string = event["detail"]["object"]["key"] From cbcdd30cb36fdad42c575807b115f69dc0295902 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Thu, 20 Nov 2025 09:58:42 +0000 Subject: [PATCH 12/22] [PRMP-540] - Address SonarCloud suggestion --- lambdas/services/bulk_upload_metadata_processor_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index 85ca6ff64..472e2942c 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -223,7 +223,7 @@ def handle_expedite_event(self, event): if key.startswith("expedite/"): logger.info("Processing file from expedite folder") file_path = os.path.basename(key) - first_document_number, second_document_number, current_file_name = ( + first_document_number, second_document_number, _ = ( extract_document_number_bulk_upload_file_name(file_path) ) From 4aa85e6b29aeb40d00384462d03e81427bd165ec Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Thu, 20 Nov 2025 10:50:01 +0000 Subject: [PATCH 13/22] [PRMP-540] - format and describe handle_expedite_event() --- lambdas/services/bulk_upload_metadata_processor_service.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index 472e2942c..07716fba0 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -217,11 +217,14 @@ def validate_and_correct_filename(self, file_metadata: MetadataFile) -> str: ) def handle_expedite_event(self, event): + """Process S3 EventBridge expedite uploads: ensure 1of1, extract identifiers, and send metadata to SQS.""" try: key_string = event["detail"]["object"]["key"] key = urllib.parse.unquote_plus(key_string, encoding="utf-8") + if key.startswith("expedite/"): logger.info("Processing file from expedite folder") + file_path = os.path.basename(key) first_document_number, second_document_number, _ = ( extract_document_number_bulk_upload_file_name(file_path) @@ -240,11 +243,13 @@ def handle_expedite_event(self, event): ) ods_code = Path(key).parent.name scan_date = datetime.now().strftime("%Y-%m-%d") + sqs_metadata = [ self.create_sqs_metadata( nhs_number, file_name, file_name, ods_code, scan_date ) ] + self.send_metadata_to_fifo_sqs(sqs_metadata) logger.info("Successfully processed expedite event") else: From 9ae243fff2db6130193b164401978c7356723307 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Thu, 20 Nov 2025 14:16:47 +0000 Subject: [PATCH 14/22] [PRMP-540] - Address PR comments --- .../bulk_upload_metadata_processor_service.py | 49 ++++++++++--------- ..._bulk_upload_metadata_processor_service.py | 40 +++++++++------ 2 files changed, 51 insertions(+), 38 deletions(-) diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index 07716fba0..175943e3c 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -185,16 +185,15 @@ def convert_to_sqs_metadata( **file.model_dump(), stored_file_name=stored_file_name ) - @staticmethod - def create_sqs_metadata( - nhs_number: str, file_path: str, stored_file_name: str, ods_code: str, scan_date + def create_sqs_metadata(self, key ) -> StagingSqsMetadata: + nhs_number, file_path, ods_code, scan_date = self.validate_expedite_file(key) return StagingSqsMetadata( nhs_number=nhs_number, files=[ BulkUploadQueueMetadata( file_path=file_path, - stored_file_name=stored_file_name, + stored_file_name=file_path, gp_practice_code=ods_code, scan_date=scan_date, ) @@ -216,6 +215,27 @@ def validate_and_correct_filename(self, file_metadata: MetadataFile) -> str: file_metadata.file_path ) + def validate_expedite_file(self, key: str): + file_path = os.path.basename(key) + first_document_number, second_document_number, _ = ( + extract_document_number_bulk_upload_file_name(file_path) + ) + + if (first_document_number, second_document_number) != (1, 1): + failure_msg = ( + "Failed processing expedite event due to file not being a 1of1" + ) + logger.error(failure_msg) + raise BulkUploadMetadataException(failure_msg) + + nhs_number = extract_nhs_number_from_bulk_upload_file_name(file_path)[0] + file_name = self.metadata_formatter_service.validate_record_filename( + key + ) + ods_code = Path(key).parent.name + scan_date = datetime.now().strftime("%Y-%m-%d") + return nhs_number, file_name, ods_code, scan_date + def handle_expedite_event(self, event): """Process S3 EventBridge expedite uploads: ensure 1of1, extract identifiers, and send metadata to SQS.""" try: @@ -225,28 +245,9 @@ def handle_expedite_event(self, event): if key.startswith("expedite/"): logger.info("Processing file from expedite folder") - file_path = os.path.basename(key) - first_document_number, second_document_number, _ = ( - extract_document_number_bulk_upload_file_name(file_path) - ) - - if (first_document_number, second_document_number) != (1, 1): - failure_msg = ( - "Failed processing expedite event due to file not being a 1of1" - ) - logger.error(failure_msg) - raise BulkUploadMetadataException(failure_msg) - - nhs_number = extract_nhs_number_from_bulk_upload_file_name(file_path)[0] - file_name = self.metadata_formatter_service.validate_record_filename( - key - ) - ods_code = Path(key).parent.name - scan_date = datetime.now().strftime("%Y-%m-%d") - sqs_metadata = [ self.create_sqs_metadata( - nhs_number, file_name, file_name, ods_code, scan_date + key ) ] diff --git a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py index b68d9f521..5a6123099 100644 --- a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py +++ b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py @@ -596,28 +596,20 @@ def test_validate_and_correct_filename_sad_path( assert result == "corrected/path/file_corrected.pdf" +@freeze_time("2025-02-03T10:00:00") def test_create_sqs_metadata_builds_expected_structure(test_service): - nhs_number = "1234567890" - file_path = "1234567890_record.pdf" - stored_file_name = "expedite/A12345/1234567890_record.pdf" ods_code = "A12345" - scan_date = "03022025" + key = f"expedite/{ods_code}/1of1_1234567890_record.pdf" - result = test_service.create_sqs_metadata( - nhs_number=nhs_number, - file_path=file_path, - stored_file_name=stored_file_name, - ods_code=ods_code, - scan_date=scan_date, - ) + result = test_service.create_sqs_metadata(key) assert result.nhs_number == "1234567890" assert len(result.files) == 1 item = result.files[0] - assert item.file_path == file_path - assert item.stored_file_name == stored_file_name + assert item.file_path == key + assert item.stored_file_name == key assert item.gp_practice_code == ods_code - assert item.scan_date == scan_date + assert item.scan_date == "2025-02-03" @freeze_time("2025-02-03T10:00:00") @@ -920,3 +912,23 @@ def test_no_remapping_logic( retries=0, ) ] + + + +@freeze_time("2025-02-03T10:00:00") +def test_validate_expedite_file_happy_path_returns_expected_tuple(test_service): + ods_code = "A12345" + key = f"expedite/{ods_code}/1of1_1234567890_record.pdf" + + nhs_number, file_name, extracted_ods, scan_date = test_service.validate_expedite_file(key) + + assert nhs_number == "1234567890" + assert file_name == key + assert extracted_ods == ods_code + assert scan_date == "2025-02-03" + + +def test_validate_expedite_file_rejects_non_1of1(test_service): + key = "expedite/A12345/2of3_1234567890_record.pdf" + with pytest.raises(BulkUploadMetadataException): + test_service.validate_expedite_file(key) From 76debbf5df9cac483dc3342f25807935d1342dbe Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Thu, 20 Nov 2025 14:31:26 +0000 Subject: [PATCH 15/22] [PRMP-540] - Format and rename variables --- .../bulk_upload_metadata_processor_service.py | 33 ++++++++++++------- ..._bulk_upload_metadata_processor_service.py | 4 +-- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index 175943e3c..d779d4464 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -185,8 +185,9 @@ def convert_to_sqs_metadata( **file.model_dump(), stored_file_name=stored_file_name ) - def create_sqs_metadata(self, key - ) -> StagingSqsMetadata: + def create_expedite_sqs_metadata(self, key + ) -> StagingSqsMetadata: + """Build a single-patient SQS metadata payload for an expedite upload.""" nhs_number, file_path, ods_code, scan_date = self.validate_expedite_file(key) return StagingSqsMetadata( nhs_number=nhs_number, @@ -215,8 +216,16 @@ def validate_and_correct_filename(self, file_metadata: MetadataFile) -> str: file_metadata.file_path ) - def validate_expedite_file(self, key: str): - file_path = os.path.basename(key) + def validate_expedite_file(self, s3_object_key: str): + """Validate and extract fields from an expedite S3 key. + + This ensures the file represents a single document (1of1) and derives + the key fields required to build SQS metadata. + + Parameters: + s3_object_key: Full S3 object key for the expedite file. Expected format: + """ + file_path = os.path.basename(s3_object_key) first_document_number, second_document_number, _ = ( extract_document_number_bulk_upload_file_name(file_path) ) @@ -230,31 +239,31 @@ def validate_expedite_file(self, key: str): nhs_number = extract_nhs_number_from_bulk_upload_file_name(file_path)[0] file_name = self.metadata_formatter_service.validate_record_filename( - key + s3_object_key ) - ods_code = Path(key).parent.name + ods_code = Path(s3_object_key).parent.name scan_date = datetime.now().strftime("%Y-%m-%d") return nhs_number, file_name, ods_code, scan_date def handle_expedite_event(self, event): """Process S3 EventBridge expedite uploads: ensure 1of1, extract identifiers, and send metadata to SQS.""" try: - key_string = event["detail"]["object"]["key"] - key = urllib.parse.unquote_plus(key_string, encoding="utf-8") + unparsed_s3_object_key = event["detail"]["object"]["key"] + s3_object_key = urllib.parse.unquote_plus(unparsed_s3_object_key, encoding="utf-8") - if key.startswith("expedite/"): + if s3_object_key.startswith("expedite/"): logger.info("Processing file from expedite folder") sqs_metadata = [ - self.create_sqs_metadata( - key + self.create_expedite_sqs_metadata( + s3_object_key ) ] self.send_metadata_to_fifo_sqs(sqs_metadata) logger.info("Successfully processed expedite event") else: - failure_msg = f"Unexpected directory or file location received from EventBridge: {key_string}" + failure_msg = f"Unexpected directory or file location received from EventBridge: {s3_object_key}" logger.error(failure_msg) raise BulkUploadMetadataException(failure_msg) except KeyError as e: diff --git a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py index 5a6123099..2c539463f 100644 --- a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py +++ b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py @@ -597,11 +597,11 @@ def test_validate_and_correct_filename_sad_path( @freeze_time("2025-02-03T10:00:00") -def test_create_sqs_metadata_builds_expected_structure(test_service): +def test_create_expedite_sqs_metadata_builds_expected_structure(test_service): ods_code = "A12345" key = f"expedite/{ods_code}/1of1_1234567890_record.pdf" - result = test_service.create_sqs_metadata(key) + result = test_service.create_expedite_sqs_metadata(key) assert result.nhs_number == "1234567890" assert len(result.files) == 1 From 5bf16d6f6b56e93206bb55e43367a68a871632de Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Fri, 21 Nov 2025 10:13:07 +0000 Subject: [PATCH 16/22] [PRMP-540] - Address PR comment --- .../bulk_upload_metadata_processor_handler.py | 1 - .../bulk_upload_metadata_processor_service.py | 30 +++++-------------- ..._bulk_upload_metadata_processor_service.py | 5 ++-- 3 files changed, 11 insertions(+), 25 deletions(-) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index 689c89a3e..3cca04fe0 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -1,4 +1,3 @@ - from enums.lloyd_george_pre_process_format import LloydGeorgePreProcessFormat from services.bulk_upload.metadata_general_preprocessor import ( MetadataGeneralPreprocessor, diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index d779d4464..4ebd0531d 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -32,10 +32,7 @@ InvalidFileNameException, LGInvalidFilesException, ) -from utils.filename_utils import ( - extract_document_number_bulk_upload_file_name, - extract_nhs_number_from_bulk_upload_file_name, -) +from utils.filename_utils import extract_nhs_number_from_bulk_upload_file_name from utils.lloyd_george_validator import validate_file_name logger = LoggingService(__name__) @@ -185,8 +182,7 @@ def convert_to_sqs_metadata( **file.model_dump(), stored_file_name=stored_file_name ) - def create_expedite_sqs_metadata(self, key - ) -> StagingSqsMetadata: + def create_expedite_sqs_metadata(self, key) -> StagingSqsMetadata: """Build a single-patient SQS metadata payload for an expedite upload.""" nhs_number, file_path, ods_code, scan_date = self.validate_expedite_file(key) return StagingSqsMetadata( @@ -218,19 +214,11 @@ def validate_and_correct_filename(self, file_metadata: MetadataFile) -> str: def validate_expedite_file(self, s3_object_key: str): """Validate and extract fields from an expedite S3 key. - This ensures the file represents a single document (1of1) and derives - the key fields required to build SQS metadata. - - Parameters: - s3_object_key: Full S3 object key for the expedite file. Expected format: - """ + the key fields required to build SQS metadata.""" file_path = os.path.basename(s3_object_key) - first_document_number, second_document_number, _ = ( - extract_document_number_bulk_upload_file_name(file_path) - ) - if (first_document_number, second_document_number) != (1, 1): + if not file_path.startswith("1of1"): failure_msg = ( "Failed processing expedite event due to file not being a 1of1" ) @@ -249,16 +237,14 @@ def handle_expedite_event(self, event): """Process S3 EventBridge expedite uploads: ensure 1of1, extract identifiers, and send metadata to SQS.""" try: unparsed_s3_object_key = event["detail"]["object"]["key"] - s3_object_key = urllib.parse.unquote_plus(unparsed_s3_object_key, encoding="utf-8") + s3_object_key = urllib.parse.unquote_plus( + unparsed_s3_object_key, encoding="utf-8" + ) if s3_object_key.startswith("expedite/"): logger.info("Processing file from expedite folder") - sqs_metadata = [ - self.create_expedite_sqs_metadata( - s3_object_key - ) - ] + sqs_metadata = [self.create_expedite_sqs_metadata(s3_object_key)] self.send_metadata_to_fifo_sqs(sqs_metadata) logger.info("Successfully processed expedite event") diff --git a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py index 2c539463f..3b1a9361a 100644 --- a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py +++ b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py @@ -914,13 +914,14 @@ def test_no_remapping_logic( ] - @freeze_time("2025-02-03T10:00:00") def test_validate_expedite_file_happy_path_returns_expected_tuple(test_service): ods_code = "A12345" key = f"expedite/{ods_code}/1of1_1234567890_record.pdf" - nhs_number, file_name, extracted_ods, scan_date = test_service.validate_expedite_file(key) + nhs_number, file_name, extracted_ods, scan_date = ( + test_service.validate_expedite_file(key) + ) assert nhs_number == "1234567890" assert file_name == key From f61e9e3f1829564373de19df83342792f655ac5d Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Fri, 21 Nov 2025 13:34:26 +0000 Subject: [PATCH 17/22] [PRMP-540] - Fix bug --- lambdas/handlers/bulk_upload_metadata_processor_handler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index 3cca04fe0..e2621402f 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -29,12 +29,14 @@ def lambda_handler(event, _context): "preFormatType", LloydGeorgePreProcessFormat.GENERAL ) formatter_service_class = get_formatter_service(raw_pre_format_type) + remappings = event.get("metadataFieldRemappings", {}) if "source" in event and event.get("source") == "aws.s3": logger.info("Handling EventBridge event from S3") metadata_formatter_service = formatter_service_class("expedite") metadata_service = BulkUploadMetadataProcessorService( - metadata_formatter_service + metadata_formatter_service=metadata_formatter_service, + metadata_heading_remap=remappings, ) metadata_service.handle_expedite_event(event) return @@ -49,8 +51,6 @@ def lambda_handler(event, _context): f"Starting metadata processing for practice directory: {practice_directory}" ) - remappings = event.get("metadataFieldRemappings", {}) - metadata_formatter_service = formatter_service_class(practice_directory) metadata_service = BulkUploadMetadataProcessorService( metadata_formatter_service=metadata_formatter_service, From c71ce42ee9b50a73080171151d74e2d8be59c24a Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Thu, 27 Nov 2025 11:41:23 +0000 Subject: [PATCH 18/22] [PRMP-540] - Fix unit test post-merge conflicts --- .../bulk_upload_metadata_processor_service.py | 28 ++++--------------- ..._bulk_upload_metadata_processor_handler.py | 2 +- ..._bulk_upload_metadata_processor_service.py | 15 ++++++++++ 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index 3156b48c3..d75e0d3d8 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -248,7 +248,7 @@ def validate_expedite_file(self, s3_object_key: str): return nhs_number, file_name, ods_code, scan_date def handle_expedite_event(self, event): - """Process S3 EventBridge expedite uploads: ensure 1of1, extract identifiers, and send metadata to SQS.""" + """Process S3 EventBridge expedite uploads: enforce virus scan, ensure 1of1, extract identifiers, and send metadata to SQS.""" try: unparsed_s3_object_key = event["detail"]["object"]["key"] s3_object_key = urllib.parse.unquote_plus( @@ -258,6 +258,9 @@ def handle_expedite_event(self, event): if s3_object_key.startswith("expedite/"): logger.info("Processing file from expedite folder") + self.enforce_virus_scanner(s3_object_key) + self.check_file_status(s3_object_key) + sqs_metadata = [self.create_expedite_sqs_metadata(s3_object_key)] self.send_metadata_to_fifo_sqs(sqs_metadata) @@ -339,7 +342,7 @@ def enforce_virus_scanner(self, file_key: str): try: result = self.s3_repo.check_file_tag_status_on_staging_bucket(file_key) - if(result != ""): + if result != "": logger.info("The file has been scanned before") return logger.info(f"Virus scan tag missing for {file_key}.") @@ -355,27 +358,6 @@ def enforce_virus_scanner(self, file_key: str): else: raise - def handle_expedite_event(self, event): - try: - key_string = event["detail"]["object"]["key"] - key = urllib.parse.unquote_plus(key_string, encoding="utf-8") - - if key.startswith("expedite/"): - logger.info("Processing file from expedite folder") - - self.enforce_virus_scanner(key) - self.check_file_status(key) - - return # To be added upon by ticket PRMP-540 - else: - failure_msg = f"Unexpected directory or file location received from EventBridge: {key_string}" - logger.error(failure_msg) - raise BulkUploadMetadataException(failure_msg) - except KeyError as e: - failure_msg = f"Failed due to missing key: {str(e)}" - logger.error(failure_msg) - raise BulkUploadMetadataException(failure_msg) - def get_formatter_service(raw_pre_format_type): try: diff --git a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py index 5029c2cd4..2fb380729 100644 --- a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py @@ -73,7 +73,7 @@ def test_metadata_processor_lambda_handler_s3_event_uses_general_by_default( mock_metadata_service.process_metadata.assert_not_called() mock_metadata_service.handle_expedite_event.assert_called_once_with(event) - mocked_general_cls.assert_called_once_with("expedite") + mocked_general_cls.assert_called_once_with("") def test_s3_event_with_expedite_key_processes( diff --git a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py index ad964994e..2190c69d3 100644 --- a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py +++ b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py @@ -638,6 +638,12 @@ def test_handle_expedite_event_happy_path_sends_sqs(test_service, mocker): key = f"expedite/{ods}/1of1_1234567890_record.pdf" event = {"detail": {"object": {"key": key}}} + mocker.patch.object( + BulkUploadMetadataProcessorService, "enforce_virus_scanner" + ) + mocker.patch.object( + BulkUploadMetadataProcessorService, "check_file_status" + ) mocked_send = mocker.patch.object( BulkUploadMetadataProcessorService, "send_metadata_to_fifo_sqs" ) @@ -686,6 +692,12 @@ def test_handle_expedite_event_missing_key_raises(test_service, mocker): def test_handle_expedite_event_rejects_non_1of1(test_service, mocker): + mocker.patch.object( + BulkUploadMetadataProcessorService, "enforce_virus_scanner" + ) + mocker.patch.object( + BulkUploadMetadataProcessorService, "check_file_status" + ) mocked_send = mocker.patch.object( BulkUploadMetadataProcessorService, "send_metadata_to_fifo_sqs" ) @@ -972,6 +984,9 @@ def test_handle_expedite_event_calls_enforce_for_expedite_key(mocker, test_servi encoded_key = urllib.parse.quote_plus("expedite/folder/some file.pdf") event = {"detail": {"object": {"key": encoded_key}}} + mocker.patch.object( + BulkUploadMetadataProcessorService, "create_expedite_sqs_metadata" + ) mocked_enforce = mocker.patch.object(test_service, "enforce_virus_scanner") mocked_check_status = mocker.patch.object(test_service, "check_file_status") From 3d729e7da883bba24fb6708b4b7e29a3ce68ab2c Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Thu, 27 Nov 2025 12:12:48 +0000 Subject: [PATCH 19/22] [PRMP-540] - Fix unit tests --- ..._bulk_upload_metadata_processor_handler.py | 37 +------------------ 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py index 2fb380729..cc0c12b28 100644 --- a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py @@ -60,40 +60,6 @@ def test_metadata_processor_lambda_handler_s3_event_triggers_expedite( mock_metadata_service.process_metadata.assert_not_called() -def test_metadata_processor_lambda_handler_s3_event_uses_general_by_default( - set_env, context, mocker, mock_metadata_service -): - mocked_general_cls = mocker.patch( - "handlers.bulk_upload_metadata_processor_handler.MetadataGeneralPreprocessor" - ) - - event = {"source": "aws.s3", "detail": {"object": {"key": "expedite/XYZ"}}} - - lambda_handler(event, context) - - mock_metadata_service.process_metadata.assert_not_called() - mock_metadata_service.handle_expedite_event.assert_called_once_with(event) - mocked_general_cls.assert_called_once_with("") - - -def test_s3_event_with_expedite_key_processes( - set_env, context, mock_metadata_service, caplog -): - event = eventbridge_event_with_s3_key( - "expedite%2F1of1_Lloyd_George_Record_[John Michael SMITH]_[1234567890]_[15-05-1990].pdf" - ) - - with caplog.at_level("INFO"): - lambda_handler(event, context) - - assert any( - "Handling EventBridge event from S3" in r.message for r in caplog.records - ) - - mock_metadata_service.handle_expedite_event.assert_called_once_with(event) - mock_metadata_service.process_metadata.assert_not_called() - - def test_s3_event_with_non_expedite_key_is_rejected_2( set_env, context, mock_metadata_service, caplog ): @@ -106,7 +72,8 @@ def test_s3_event_with_non_expedite_key_is_rejected_2( mock_metadata_service.handle_expedite_event.assert_called_once_with(event) mock_metadata_service.process_metadata.assert_not_called() -def test_s3_event_with_expedite_key_processes_2(set_env, context, mock_metadata_service): + +def test_s3_event_with_expedite_key_processes(set_env, context, mock_metadata_service): event = eventbridge_event_with_s3_key( "expedite%2F1of1_Lloyd_George_Record_[John Michael SMITH]_[1234567890]_[15-05-1990].pdf" ) From 1e681393bee434ab6a05bfee0107e759d83d9d2c Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Thu, 27 Nov 2025 12:15:47 +0000 Subject: [PATCH 20/22] [PRMP-540] - Fix test name --- .../handlers/test_bulk_upload_metadata_processor_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py index cc0c12b28..8bfe02d8f 100644 --- a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py @@ -60,7 +60,7 @@ def test_metadata_processor_lambda_handler_s3_event_triggers_expedite( mock_metadata_service.process_metadata.assert_not_called() -def test_s3_event_with_non_expedite_key_is_rejected_2( +def test_s3_event_with_non_expedite_key_is_rejected( set_env, context, mock_metadata_service, caplog ): key_string = "uploads/1of1_Lloyd_George_Record_[John Michael SMITH]_[1234567890]_[15-05-1990].pdf" From b69495bb264d16fdc06bb07fe434476784f2eb53 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Thu, 27 Nov 2025 12:16:50 +0000 Subject: [PATCH 21/22] [PRMP-540] - make format --- .../bulk_upload_metadata_processor_service.py | 1 - ..._bulk_upload_metadata_processor_handler.py | 1 - ..._bulk_upload_metadata_processor_service.py | 24 +++++++------------ 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index d75e0d3d8..ef3fa8c1b 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -9,7 +9,6 @@ import pydantic from botocore.exceptions import ClientError - from enums.lloyd_george_pre_process_format import LloydGeorgePreProcessFormat from enums.upload_status import UploadStatus from enums.virus_scan_result import VirusScanResult diff --git a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py index 8bfe02d8f..913c51166 100644 --- a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py @@ -1,5 +1,4 @@ import pytest - from handlers.bulk_upload_metadata_processor_handler import lambda_handler from services.bulk_upload_metadata_processor_service import ( BulkUploadMetadataProcessorService, diff --git a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py index 2190c69d3..7ea7f50d1 100644 --- a/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py +++ b/lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py @@ -8,9 +8,8 @@ import pytest from botocore.exceptions import ClientError from enums.upload_status import UploadStatus -from freezegun import freeze_time - from enums.virus_scan_result import VirusScanResult +from freezegun import freeze_time from models.staging_metadata import ( METADATA_FILENAME, BulkUploadQueueMetadata, @@ -35,7 +34,7 @@ BulkUploadMetadataException, InvalidFileNameException, LGInvalidFilesException, - VirusScanNoResultException, VirusScanFailedException, + VirusScanFailedException, ) METADATA_FILE_DIR = "tests/unit/helpers/data/bulk_upload" @@ -638,12 +637,8 @@ def test_handle_expedite_event_happy_path_sends_sqs(test_service, mocker): key = f"expedite/{ods}/1of1_1234567890_record.pdf" event = {"detail": {"object": {"key": key}}} - mocker.patch.object( - BulkUploadMetadataProcessorService, "enforce_virus_scanner" - ) - mocker.patch.object( - BulkUploadMetadataProcessorService, "check_file_status" - ) + mocker.patch.object(BulkUploadMetadataProcessorService, "enforce_virus_scanner") + mocker.patch.object(BulkUploadMetadataProcessorService, "check_file_status") mocked_send = mocker.patch.object( BulkUploadMetadataProcessorService, "send_metadata_to_fifo_sqs" ) @@ -692,12 +687,8 @@ def test_handle_expedite_event_missing_key_raises(test_service, mocker): def test_handle_expedite_event_rejects_non_1of1(test_service, mocker): - mocker.patch.object( - BulkUploadMetadataProcessorService, "enforce_virus_scanner" - ) - mocker.patch.object( - BulkUploadMetadataProcessorService, "check_file_status" - ) + mocker.patch.object(BulkUploadMetadataProcessorService, "enforce_virus_scanner") + mocker.patch.object(BulkUploadMetadataProcessorService, "check_file_status") mocked_send = mocker.patch.object( BulkUploadMetadataProcessorService, "send_metadata_to_fifo_sqs" ) @@ -1130,6 +1121,7 @@ def test_enforce_virus_scanner_re_raises_unexpected_client_error(mocker, test_se mock_scan.assert_not_called() + def test_check_file_status_clean_does_nothing(mocker, test_service, caplog): file_key = "expedite/folder/file.pdf" mock_check = mocker.patch.object( @@ -1162,4 +1154,4 @@ def test_check_file_status_logs_issue_when_not_clean(mocker, test_service, caplo assert any( f"Found an issue with the file {file_key}." in record.msg for record in caplog.records - ) \ No newline at end of file + ) From 87986a3180624b6fb4ca84d3b9b51146a6233949 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Thu, 27 Nov 2025 13:39:52 +0000 Subject: [PATCH 22/22] [PRMP-540] - Fix line too long error --- lambdas/services/bulk_upload_metadata_processor_service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lambdas/services/bulk_upload_metadata_processor_service.py b/lambdas/services/bulk_upload_metadata_processor_service.py index ef3fa8c1b..8aae45b8a 100644 --- a/lambdas/services/bulk_upload_metadata_processor_service.py +++ b/lambdas/services/bulk_upload_metadata_processor_service.py @@ -247,7 +247,8 @@ def validate_expedite_file(self, s3_object_key: str): return nhs_number, file_name, ods_code, scan_date def handle_expedite_event(self, event): - """Process S3 EventBridge expedite uploads: enforce virus scan, ensure 1of1, extract identifiers, and send metadata to SQS.""" + """Process S3 EventBridge expedite uploads: enforce virus scan, ensure 1of1, extract identifiers + and send metadata to SQS.""" try: unparsed_s3_object_key = event["detail"]["object"]["key"] s3_object_key = urllib.parse.unquote_plus(