From 482ee42573fa3837be2c62a756d2c5e40017457f Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Tue, 4 Nov 2025 10:02:59 +0000 Subject: [PATCH 01/14] [PRMP-539] - Handle S3 event and unit tests --- .../bulk_upload_metadata_processor_handler.py | 14 ++++++++ ..._bulk_upload_metadata_processor_handler.py | 32 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index 8f08d6621..9f28ca258 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -1,3 +1,5 @@ +import urllib.parse + from enums.lloyd_george_pre_process_format import LloydGeorgePreProcessFormat from services.bulk_upload.metadata_general_preprocessor import ( MetadataGeneralPreprocessor, @@ -24,6 +26,18 @@ ) @handle_lambda_exceptions def lambda_handler(event, _context): + if 'Records' in event and \ + event['Records'][0].get('eventSource') == 's3.amazonaws.com': + logger.info("Triggered by S3 listener...") + key_string = event['Records'][0]['s3']['object']['key'] + key = urllib.parse.unquote_plus(key_string, encoding='utf-8') + if key.startswith("expedite/"): + logger.info("Processing file from expedite folder") + return # To be added upon by ticket PRMP-540 + else: + logger.error("Unrecognized S3 listener event, cancelling.") + return + practice_directory = event.get("practiceDirectory", "") raw_pre_format_type = event.get( diff --git a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py index b7029f5be..bbdb5889e 100644 --- a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py @@ -14,6 +14,17 @@ def mock_metadata_service(mocker): return mocked_instance +def s3_event_with_key(key: str): + return { + "Records": [ + { + "eventSource": "s3.amazonaws.com", + "s3": {"object": {"key": key}}, + } + ] + } + + def test_metadata_processor_lambda_handler_valid_event( set_env, context, mock_metadata_service ): @@ -28,3 +39,24 @@ def test_metadata_processor_lambda_handler_empty_event( lambda_handler({}, context) mock_metadata_service.process_metadata.assert_not_called() + + +def test_s3_event_with_expedite_key_processes( + set_env, context, mock_metadata_service, caplog +): + event = s3_event_with_key("expedite%2F1of1_Lloyd_George_Record_[John Michael SMITH]_[1234567890]_[15-05-1990].pdf") + lambda_handler(event, context) + + assert any("Triggered by S3 listener" in r.message for r in caplog.records) + assert any("Processing file from expedite folder" in r.message for r in caplog.records) + + +def test_s3_event_with_non_expedite_key_is_rejected( + set_env, context, mock_metadata_service, caplog +): + event = s3_event_with_key("uploads/1of1_Lloyd_George_Record_[John Michael SMITH]_[1234567890]_[15-05-1990].pdf") + lambda_handler(event, context) + + assert any("Unrecognized S3 listener event, cancelling." in r.message for r in caplog.records) + mock_metadata_service.process_metadata.assert_not_called() + From 259ddc8038a522e7090c0b8c557e53739f9cfa28 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Tue, 4 Nov 2025 10:16:55 +0000 Subject: [PATCH 02/14] [PRMP-539] - make format --- .../bulk_upload_metadata_processor_handler.py | 12 +++++++----- ...t_bulk_upload_metadata_processor_handler.py | 18 +++++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index 9f28ca258..eba89b00f 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -26,14 +26,16 @@ ) @handle_lambda_exceptions def lambda_handler(event, _context): - if 'Records' in event and \ - event['Records'][0].get('eventSource') == 's3.amazonaws.com': + if ( + "Records" in event + and event["Records"][0].get("eventSource") == "s3.amazonaws.com" + ): logger.info("Triggered by S3 listener...") - key_string = event['Records'][0]['s3']['object']['key'] - key = urllib.parse.unquote_plus(key_string, encoding='utf-8') + key_string = event["Records"][0]["s3"]["object"]["key"] + key = urllib.parse.unquote_plus(key_string, encoding="utf-8") if key.startswith("expedite/"): logger.info("Processing file from expedite folder") - return # To be added upon by ticket PRMP-540 + return # To be added upon by ticket PRMP-540 else: logger.error("Unrecognized S3 listener event, cancelling.") return diff --git a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py index bbdb5889e..328ce19eb 100644 --- a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py @@ -44,19 +44,27 @@ def test_metadata_processor_lambda_handler_empty_event( def test_s3_event_with_expedite_key_processes( set_env, context, mock_metadata_service, caplog ): - event = s3_event_with_key("expedite%2F1of1_Lloyd_George_Record_[John Michael SMITH]_[1234567890]_[15-05-1990].pdf") + event = s3_event_with_key( + "expedite%2F1of1_Lloyd_George_Record_[John Michael SMITH]_[1234567890]_[15-05-1990].pdf" + ) lambda_handler(event, context) assert any("Triggered by S3 listener" in r.message for r in caplog.records) - assert any("Processing file from expedite folder" in r.message for r in caplog.records) + assert any( + "Processing file from expedite folder" in r.message for r in caplog.records + ) def test_s3_event_with_non_expedite_key_is_rejected( set_env, context, mock_metadata_service, caplog ): - event = s3_event_with_key("uploads/1of1_Lloyd_George_Record_[John Michael SMITH]_[1234567890]_[15-05-1990].pdf") + event = s3_event_with_key( + "uploads/1of1_Lloyd_George_Record_[John Michael SMITH]_[1234567890]_[15-05-1990].pdf" + ) lambda_handler(event, context) - assert any("Unrecognized S3 listener event, cancelling." in r.message for r in caplog.records) + assert any( + "Unrecognized S3 listener event, cancelling." in r.message + for r in caplog.records + ) mock_metadata_service.process_metadata.assert_not_called() - From 343bf7f5a22ae16859c916330ab9a321021db154 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Tue, 4 Nov 2025 10:50:26 +0000 Subject: [PATCH 03/14] [PRMP-539] - Fix eventSource conditional check --- lambdas/handlers/bulk_upload_metadata_processor_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index eba89b00f..1a5a31c02 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -28,7 +28,7 @@ def lambda_handler(event, _context): if ( "Records" in event - and event["Records"][0].get("eventSource") == "s3.amazonaws.com" + and event["Records"][0].get("eventSource") == "aws:s3" ): logger.info("Triggered by S3 listener...") key_string = event["Records"][0]["s3"]["object"]["key"] From d5ddc7da8a1f353317c82b1bf37d645e8f0b01ac Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Tue, 4 Nov 2025 10:54:04 +0000 Subject: [PATCH 04/14] [PRMP-539] - Fix unit test --- .../handlers/test_bulk_upload_metadata_processor_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py index 328ce19eb..e4ae8236d 100644 --- a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py @@ -18,7 +18,7 @@ def s3_event_with_key(key: str): return { "Records": [ { - "eventSource": "s3.amazonaws.com", + "eventSource": "aws:s3", "s3": {"object": {"key": key}}, } ] From a1d67bfe66b85646e3804e5f021d53c765467970 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Tue, 4 Nov 2025 13:16:40 +0000 Subject: [PATCH 05/14] [PRMP-539] - Add try-catch for failures --- .../bulk_upload_metadata_processor_handler.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index 1a5a31c02..f7544b3a7 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -15,6 +15,7 @@ from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions from utils.decorators.override_error_check import override_error_check from utils.decorators.set_audit_arg import set_request_context_for_logging +from utils.exceptions import BulkUploadMetadataException logger = LoggingService(__name__) @@ -31,14 +32,20 @@ def lambda_handler(event, _context): and event["Records"][0].get("eventSource") == "aws:s3" ): logger.info("Triggered by S3 listener...") - key_string = event["Records"][0]["s3"]["object"]["key"] - key = urllib.parse.unquote_plus(key_string, encoding="utf-8") - if key.startswith("expedite/"): - logger.info("Processing file from expedite folder") - return # To be added upon by ticket PRMP-540 - else: - logger.error("Unrecognized S3 listener event, cancelling.") - return + try: + key_string = event["Records"][0]["s3"]["object"]["key"] + key = urllib.parse.unquote_plus(key_string, encoding="utf-8") + if key.startswith("expedite/"): + logger.info("Processing file from expedite folder") + return # To be added upon by ticket PRMP-540 + else: + failure_msg = "Unrecognized S3 listener event, cancelling." + logger.error(failure_msg) + raise BulkUploadMetadataException(failure_msg) + except KeyError as e: + failure_msg = f"Failed due to missing key: {str(e)}" + logger.error(failure_msg) + raise BulkUploadMetadataException(failure_msg) practice_directory = event.get("practiceDirectory", "") From 3461e48198b4b8f7e8075b39bed7e7baf79d7ccb Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Tue, 4 Nov 2025 13:25:05 +0000 Subject: [PATCH 06/14] [PRMP-539] - Modify failure message --- lambdas/handlers/bulk_upload_metadata_processor_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index f7544b3a7..740081eb3 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -39,7 +39,7 @@ def lambda_handler(event, _context): logger.info("Processing file from expedite folder") return # To be added upon by ticket PRMP-540 else: - failure_msg = "Unrecognized S3 listener event, cancelling." + failure_msg = "Failed due to unrecognized S3 listener event key" logger.error(failure_msg) raise BulkUploadMetadataException(failure_msg) except KeyError as e: From a414562586d0c4f0cf28141ea3ab19469382c292 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Tue, 4 Nov 2025 13:28:05 +0000 Subject: [PATCH 07/14] [PRMP-539] - Fix unit tests --- .../handlers/test_bulk_upload_metadata_processor_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py index e4ae8236d..eb6e80c82 100644 --- a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py @@ -64,7 +64,7 @@ def test_s3_event_with_non_expedite_key_is_rejected( lambda_handler(event, context) assert any( - "Unrecognized S3 listener event, cancelling." in r.message + "Failed due to unrecognized S3 listener event key" in r.message for r in caplog.records ) mock_metadata_service.process_metadata.assert_not_called() From 5e61d4ddcc56f3fbf4f9c2504feb453068783ae3 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Tue, 4 Nov 2025 13:29:39 +0000 Subject: [PATCH 08/14] [PRMP-539] - format --- lambdas/handlers/bulk_upload_metadata_processor_handler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index 740081eb3..b41eb2048 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -27,10 +27,7 @@ ) @handle_lambda_exceptions def lambda_handler(event, _context): - if ( - "Records" in event - and event["Records"][0].get("eventSource") == "aws:s3" - ): + if "Records" in event and event["Records"][0].get("eventSource") == "aws:s3": logger.info("Triggered by S3 listener...") try: key_string = event["Records"][0]["s3"]["object"]["key"] From 74d6c98be3bca762e6b8261afd9b77161b489867 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Wed, 5 Nov 2025 16:22:30 +0000 Subject: [PATCH 09/14] [PRMP-539] - Address PR comments --- .../handlers/bulk_upload_metadata_processor_handler.py | 4 ++-- .../test_bulk_upload_metadata_processor_handler.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index b41eb2048..fedf59af5 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -28,7 +28,7 @@ @handle_lambda_exceptions def lambda_handler(event, _context): if "Records" in event and event["Records"][0].get("eventSource") == "aws:s3": - logger.info("Triggered by S3 listener...") + logger.info(f"Triggering event sent by S3 listener {event['Records'][0].get('eventSource')}") try: key_string = event["Records"][0]["s3"]["object"]["key"] key = urllib.parse.unquote_plus(key_string, encoding="utf-8") @@ -36,7 +36,7 @@ def lambda_handler(event, _context): logger.info("Processing file from expedite folder") return # To be added upon by ticket PRMP-540 else: - failure_msg = "Failed due to unrecognized S3 listener event key" + failure_msg = f"Unexpected directory or file location received from S3 Listener: {key_string}" logger.error(failure_msg) raise BulkUploadMetadataException(failure_msg) except KeyError as e: diff --git a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py index eb6e80c82..5799fe028 100644 --- a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py @@ -49,7 +49,7 @@ def test_s3_event_with_expedite_key_processes( ) lambda_handler(event, context) - assert any("Triggered by S3 listener" in r.message for r in caplog.records) + assert any(f"Triggering event sent by S3 listener {event['Records'][0].get('eventSource')}" in r.message for r in caplog.records) assert any( "Processing file from expedite folder" in r.message for r in caplog.records ) @@ -58,13 +58,15 @@ def test_s3_event_with_expedite_key_processes( def test_s3_event_with_non_expedite_key_is_rejected( set_env, context, mock_metadata_service, caplog ): + key_string = "uploads/1of1_Lloyd_George_Record_[John Michael SMITH]_[1234567890]_[15-05-1990].pdf" event = s3_event_with_key( - "uploads/1of1_Lloyd_George_Record_[John Michael SMITH]_[1234567890]_[15-05-1990].pdf" + key_string ) + lambda_handler(event, context) assert any( - "Failed due to unrecognized S3 listener event key" in r.message + f"Unexpected directory or file location received from S3 Listener: {key_string}" in r.message for r in caplog.records ) mock_metadata_service.process_metadata.assert_not_called() From d77c8a425a050496251dd8699c1e424d51c25116 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Wed, 5 Nov 2025 16:25:13 +0000 Subject: [PATCH 10/14] [PRMP-539] - make format --- .../bulk_upload_metadata_processor_handler.py | 4 +++- .../test_bulk_upload_metadata_processor_handler.py | 13 ++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index fedf59af5..f6dc8f221 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -28,7 +28,9 @@ @handle_lambda_exceptions def lambda_handler(event, _context): if "Records" in event and event["Records"][0].get("eventSource") == "aws:s3": - logger.info(f"Triggering event sent by S3 listener {event['Records'][0].get('eventSource')}") + logger.info( + f"Triggering event sent by S3 listener {event['Records'][0].get('eventSource')}" + ) try: key_string = event["Records"][0]["s3"]["object"]["key"] key = urllib.parse.unquote_plus(key_string, encoding="utf-8") diff --git a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py index 5799fe028..f9c22cf11 100644 --- a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py @@ -49,7 +49,11 @@ def test_s3_event_with_expedite_key_processes( ) lambda_handler(event, context) - assert any(f"Triggering event sent by S3 listener {event['Records'][0].get('eventSource')}" in r.message for r in caplog.records) + assert any( + f"Triggering event sent by S3 listener {event['Records'][0].get('eventSource')}" + in r.message + for r in caplog.records + ) assert any( "Processing file from expedite folder" in r.message for r in caplog.records ) @@ -59,14 +63,13 @@ def test_s3_event_with_non_expedite_key_is_rejected( set_env, context, mock_metadata_service, caplog ): key_string = "uploads/1of1_Lloyd_George_Record_[John Michael SMITH]_[1234567890]_[15-05-1990].pdf" - event = s3_event_with_key( - key_string - ) + event = s3_event_with_key(key_string) lambda_handler(event, context) assert any( - f"Unexpected directory or file location received from S3 Listener: {key_string}" in r.message + f"Unexpected directory or file location received from S3 Listener: {key_string}" + in r.message for r in caplog.records ) mock_metadata_service.process_metadata.assert_not_called() From 4565769ffa5baecc6f4f8300aea60d1944bd965a Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Tue, 11 Nov 2025 15:01:49 +0000 Subject: [PATCH 11/14] [PRMP-538] - Modify handler to process EventBridge events --- .../handlers/bulk_upload_metadata_processor_handler.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index f6dc8f221..ed34fffb0 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -27,18 +27,16 @@ ) @handle_lambda_exceptions def lambda_handler(event, _context): - if "Records" in event and event["Records"][0].get("eventSource") == "aws:s3": - logger.info( - f"Triggering event sent by S3 listener {event['Records'][0].get('eventSource')}" - ) + if "source" in event and event.get("source") == "aws.s3": + logger.info("Triggering EventBridge event from S3") try: - key_string = event["Records"][0]["s3"]["object"]["key"] + key_string = event["detail"]["object"]["key"] key = urllib.parse.unquote_plus(key_string, encoding="utf-8") if key.startswith("expedite/"): logger.info("Processing file from expedite folder") return # To be added upon by ticket PRMP-540 else: - failure_msg = f"Unexpected directory or file location received from S3 Listener: {key_string}" + failure_msg = f"Unexpected directory or file location received from EventBridge: {key_string}" logger.error(failure_msg) raise BulkUploadMetadataException(failure_msg) except KeyError as e: From 4eb18f43bec97a19f4e241464c667ce33dca2916 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Tue, 11 Nov 2025 15:11:37 +0000 Subject: [PATCH 12/14] [PRMP-539] - Refactor handling expedite event --- .../bulk_upload_metadata_processor_handler.py | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index ed34fffb0..e681ce713 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -28,21 +28,9 @@ @handle_lambda_exceptions def lambda_handler(event, _context): if "source" in event and event.get("source") == "aws.s3": - logger.info("Triggering EventBridge event from S3") - try: - key_string = event["detail"]["object"]["key"] - key = urllib.parse.unquote_plus(key_string, encoding="utf-8") - if key.startswith("expedite/"): - logger.info("Processing file from expedite folder") - return # To be added upon by ticket PRMP-540 - else: - failure_msg = f"Unexpected directory or file location received from EventBridge: {key_string}" - logger.error(failure_msg) - raise BulkUploadMetadataException(failure_msg) - except KeyError as e: - failure_msg = f"Failed due to missing key: {str(e)}" - logger.error(failure_msg) - raise BulkUploadMetadataException(failure_msg) + logger.info("Handling EventBridge event from S3") + handle_expedite_event(event) + return practice_directory = event.get("practiceDirectory", "") @@ -80,3 +68,19 @@ def get_formatter_service(raw_pre_format_type): f"Invalid preFormatType: '{raw_pre_format_type}', defaulting to {LloydGeorgePreProcessFormat.GENERAL}." ) return MetadataGeneralPreprocessor + +def handle_expedite_event(event): + try: + key_string = event["detail"]["object"]["key"] + key = urllib.parse.unquote_plus(key_string, encoding="utf-8") + if key.startswith("expedite/"): + logger.info("Processing file from expedite folder") + return # To be added upon by ticket PRMP-540 + else: + failure_msg = f"Unexpected directory or file location received from EventBridge: {key_string}" + logger.error(failure_msg) + raise BulkUploadMetadataException(failure_msg) + except KeyError as e: + failure_msg = f"Failed due to missing key: {str(e)}" + logger.error(failure_msg) + raise BulkUploadMetadataException(failure_msg) \ No newline at end of file From 17bd4d2dff8a3f5a163c39979075f13a598053b2 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Tue, 11 Nov 2025 15:12:16 +0000 Subject: [PATCH 13/14] [PRMP-539] - make format --- lambdas/handlers/bulk_upload_metadata_processor_handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lambdas/handlers/bulk_upload_metadata_processor_handler.py b/lambdas/handlers/bulk_upload_metadata_processor_handler.py index e681ce713..50a0838c5 100644 --- a/lambdas/handlers/bulk_upload_metadata_processor_handler.py +++ b/lambdas/handlers/bulk_upload_metadata_processor_handler.py @@ -69,6 +69,7 @@ def get_formatter_service(raw_pre_format_type): ) return MetadataGeneralPreprocessor + def handle_expedite_event(event): try: key_string = event["detail"]["object"]["key"] @@ -83,4 +84,4 @@ def handle_expedite_event(event): except KeyError as e: failure_msg = f"Failed due to missing key: {str(e)}" logger.error(failure_msg) - raise BulkUploadMetadataException(failure_msg) \ No newline at end of file + raise BulkUploadMetadataException(failure_msg) From 619c731c2004b041c47faa52a98ff6146a796bf1 Mon Sep 17 00:00:00 2001 From: MohammadIqbalAD-NHS <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> Date: Tue, 11 Nov 2025 15:24:33 +0000 Subject: [PATCH 14/14] [PRMP-539] - Fix unit tests --- ..._bulk_upload_metadata_processor_handler.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py index f9c22cf11..d65717e5b 100644 --- a/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py +++ b/lambdas/tests/unit/handlers/test_bulk_upload_metadata_processor_handler.py @@ -14,14 +14,14 @@ def mock_metadata_service(mocker): return mocked_instance -def s3_event_with_key(key: str): +def eventbridge_event_with_s3_key(key: str): return { - "Records": [ - { - "eventSource": "aws:s3", - "s3": {"object": {"key": key}}, - } - ] + "source": "aws.s3", + "detail": { + "object":{ + "key": key, + }, + } } @@ -44,13 +44,13 @@ def test_metadata_processor_lambda_handler_empty_event( def test_s3_event_with_expedite_key_processes( set_env, context, mock_metadata_service, caplog ): - event = s3_event_with_key( + event = eventbridge_event_with_s3_key( "expedite%2F1of1_Lloyd_George_Record_[John Michael SMITH]_[1234567890]_[15-05-1990].pdf" ) lambda_handler(event, context) assert any( - f"Triggering event sent by S3 listener {event['Records'][0].get('eventSource')}" + f"Handling EventBridge event from S3" in r.message for r in caplog.records ) @@ -63,12 +63,12 @@ def test_s3_event_with_non_expedite_key_is_rejected( set_env, context, mock_metadata_service, caplog ): key_string = "uploads/1of1_Lloyd_George_Record_[John Michael SMITH]_[1234567890]_[15-05-1990].pdf" - event = s3_event_with_key(key_string) + event = eventbridge_event_with_s3_key(key_string) lambda_handler(event, context) assert any( - f"Unexpected directory or file location received from S3 Listener: {key_string}" + f"Unexpected directory or file location received from EventBridge: {key_string}" in r.message for r in caplog.records )