[PRMP-1621] Enhance invalid filename handling to check file existence before sending to review

Author: NogaNHS

lambdas/services/bulk_upload_metadata_processor_service.py

@@ -366,7 +366,10 @@ def handle_invalid_filename(
         ods_code: str,
         failed_files: dict[tuple[str, str], list[BulkUploadQueueMetadata]],
     ) -> None:
-        """Handle invalid filenames by logging, storing failure in Dynamo, and tracking for review."""
+        """Handle invalid filenames by logging, storing failure in Dynamo, and tracking for review.
+        Files that do not exist on the staging bucket are marked as failed only —
+        they are never added to the review queue since they cannot be reviewed.
+        """
         logger.error(
             f"Failed to process {file_metadata.file_path} due to error: {error}",
         )
@@ -375,14 +378,21 @@ def handle_invalid_filename(
             file_metadata.file_path,
         )
         failed_file.file_path = self.add_directory_path_to_file_path(file_metadata)
-        failed_files[(nhs_number, ods_code)].append(failed_file)
+
+        file_exists = self.s3_repo.file_exists_on_staging_bucket(failed_file.file_path)
+        if not file_exists:
+            logger.info(
+                f"File {failed_file.file_path} not found on staging bucket. Will not send to review.",
+            )
+        else:
+            failed_files[(nhs_number, ods_code)].append(failed_file)
 
         failed_entry = StagingSqsMetadata(nhs_number=nhs_number, files=[failed_file])
         self.dynamo_repository.write_report_upload_to_dynamo(
             failed_entry,
             UploadStatus.FAILED,
             str(error),
-            sent_to_review=self.send_to_review_enabled,
+            sent_to_review=self.send_to_review_enabled and file_exists,
         )
 
     def send_failed_files_to_review_queue(

lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py

@@ -585,6 +585,8 @@ def test_handle_invalid_filename_writes_failed_entry_to_dynamo(
     failed_files = defaultdict(list)
     error = InvalidFileNameException("Invalid filename format")
 
+    test_service.s3_repo.file_exists_on_staging_bucket.return_value = True
+
     mock_staging_metadata = mocker.patch(
         "services.bulk_upload_metadata_processor_service.StagingSqsMetadata",
     )
@@ -636,6 +638,9 @@ def test_handle_invalid_filename_sets_sent_to_review_true_when_review_enabled(
     failed_files = defaultdict(list)
     error = InvalidFileNameException("Invalid filename format")
 
+    test_service_with_review_enabled.s3_repo.file_exists_on_staging_bucket.return_value = (
+        True
+    )
     mock_write = mocker.patch.object(
         test_service_with_review_enabled.dynamo_repository,
         "write_report_upload_to_dynamo",
@@ -656,6 +661,43 @@ def test_handle_invalid_filename_sets_sent_to_review_true_when_review_enabled(
         str(error),
         sent_to_review=True,
     )
+    assert (nhs_number, ods_code) in failed_files
+
+
+def test_handle_invalid_filename_does_not_send_to_review_when_file_missing(
+    mocker,
+    test_service_with_review_enabled,
+    base_metadata_file,
+):
+    nhs_number = "1234567890"
+    ods_code = "Y12345"
+    failed_files = defaultdict(list)
+    error = InvalidFileNameException("Invalid filename format")
+
+    test_service_with_review_enabled.s3_repo.file_exists_on_staging_bucket.return_value = (
+        False
+    )
+    mock_write = mocker.patch.object(
+        test_service_with_review_enabled.dynamo_repository,
+        "write_report_upload_to_dynamo",
+    )
+    mocker.patch("services.bulk_upload_metadata_processor_service.StagingSqsMetadata")
+
+    test_service_with_review_enabled.handle_invalid_filename(
+        base_metadata_file,
+        error,
+        nhs_number,
+        ods_code,
+        failed_files,
+    )
+
+    mock_write.assert_called_once_with(
+        mocker.ANY,
+        UploadStatus.FAILED,
+        str(error),
+        sent_to_review=False,
+    )
+    assert (nhs_number, ods_code) not in failed_files
 
 
 def test_csv_to_sqs_metadata_sends_failed_files_to_review_queue_when_enabled(
@@ -676,6 +718,9 @@ def test_csv_to_sqs_metadata_sends_failed_files_to_review_queue_when_enabled(
         "validate_and_correct_filename",
         side_effect=InvalidFileNameException("invalid"),
     )
+    test_service_with_review_enabled.s3_repo.file_exists_on_staging_bucket.return_value = (
+        True
+    )
 
     result = test_service_with_review_enabled.csv_to_sqs_metadata(MOCK_METADATA_CSV)