[PRMP-401] Parse scan date in bulk upload (#795)

Author: NogaNHS

lambdas/services/bulk_upload_service.py

@@ -1,7 +1,6 @@
 import json
 import os
 import uuid
-from datetime import datetime
 
 import pydantic
 from botocore.exceptions import ClientError
@@ -39,6 +38,7 @@
     validate_filename_with_patient_details_lenient,
     validate_filename_with_patient_details_strict,
     validate_lg_file_names,
+    validate_scan_date,
 )
 from utils.request_context import request_context
 from utils.unicode_utils import (
@@ -126,10 +126,10 @@ def handle_sqs_message(self, message: dict):
         logger.info("SQS event is valid. Validating NHS number and file names")
 
         try:
-            file_names = [
-                os.path.basename(metadata.file_path)
-                for metadata in staging_metadata.files
-            ]
+            file_names = []
+            for file_metadata in staging_metadata.files:
+                file_names.append(os.path.basename(file_metadata.file_path))
+                file_metadata.scan_date = validate_scan_date(file_metadata.scan_date)
             request_context.patient_nhs_no = staging_metadata.nhs_number
             validate_nhs_number(staging_metadata.nhs_number)
             validate_lg_file_names(file_names, staging_metadata.nhs_number)
@@ -139,6 +139,7 @@ def handle_sqs_message(self, message: dict):
             patient_ods_code = (
                 pds_patient_details.get_ods_code_or_inactive_status_for_gp()
             )
+
             if not self.bypass_pds:
                 if not self.strict_mode:
                     (
@@ -395,12 +396,6 @@ def convert_to_document_reference(
     ) -> DocumentReference:
         s3_bucket_name = self.bulk_upload_s3_repository.lg_bucket_name
         file_name = os.path.basename(file_metadata.file_path)
-        if file_metadata.scan_date:
-            scan_date_formatted = datetime.strptime(
-                file_metadata.scan_date, "%d/%m/%Y"
-            ).strftime("%Y-%m-%d")
-        else:
-            scan_date_formatted = None
         document_reference = DocumentReference(
             id=str(uuid.uuid4()),
             nhs_number=nhs_number,
@@ -409,7 +404,7 @@ def convert_to_document_reference(
             current_gp_ods=current_gp_ods,
             custodian=current_gp_ods,
             author=file_metadata.gp_practice_code,
-            document_scan_creation=scan_date_formatted,
+            document_scan_creation=file_metadata.scan_date,
             doc_status="preliminary",
         )
         document_reference.set_virus_scanner_result(VirusScanResult.CLEAN)

lambdas/tests/unit/helpers/data/bulk_upload/test_data.py

@@ -243,7 +243,9 @@ def build_test_staging_metadata(file_names: list[str], nhs_number: str = "900000
     for file_name in file_names:
         source_file_path = f"/{nhs_number}/{file_name}"
         files.append(
-            sample_sqs_metadata_model.model_copy(update={"file_path": source_file_path})
+            sample_sqs_metadata_model.model_copy(
+                update={"file_path": source_file_path, "scan_date": "2022-09-03"}
+            )
         )
     return StagingSqsMetadata(files=files, nhs_number=nhs_number)
 

lambdas/tests/unit/services/test_bulk_upload_service.py

@@ -57,10 +57,10 @@
 
 @pytest.fixture
 def repo_under_test(set_env, mocker):
+    mocker.patch("services.bulk_upload_service.BulkUploadDynamoRepository")
+    mocker.patch("services.bulk_upload_service.BulkUploadSqsRepository")
+    mocker.patch("services.bulk_upload_service.BulkUploadS3Repository")
     service = BulkUploadService(strict_mode=True)
-    mocker.patch.object(service, "dynamo_repository")
-    mocker.patch.object(service, "sqs_repository")
-    mocker.patch.object(service, "bulk_upload_s3_repository")
     yield service
 
 
@@ -889,25 +889,29 @@ def test_convert_to_document_reference(set_env, mock_uuid, repo_under_test):
 
 
 @freeze_time("2024-01-01 12:00:00")
-def test_convert_to_document_reference_missing_scan_date(
-    set_env, mock_uuid, repo_under_test
+def test_reject_document_reference_if_missing_scan_date(
+    set_env, mock_uuid, repo_under_test, mocker
 ):
     TEST_STAGING_METADATA.retries = 0
-    repo_under_test.bulk_upload_s3_repository.lg_bucket_name = "test_lg_s3_bucket"
-    expected = TEST_DOCUMENT_REFERENCE
-    expected.document_scan_creation = None
-    TEST_FILE_METADATA.scan_date = ""
+    modify_test_sqs_message = json.loads(TEST_SQS_MESSAGE.get("body"))
+    modify_test_sqs_message["files"][0]["scan_date"] = ""
+    TEST_STAGING_METADATA.files[0].scan_date = ""
+    mock_report_upload_failure = mocker.patch.object(
+        repo_under_test.dynamo_repository, "write_report_upload_to_dynamo"
+    )
 
-    actual = repo_under_test.convert_to_document_reference(
-        file_metadata=TEST_FILE_METADATA,
-        nhs_number=TEST_STAGING_METADATA.nhs_number,
-        current_gp_ods=TEST_CURRENT_GP_ODS,
+    repo_under_test.handle_sqs_message(
+        message={"body": json.dumps(modify_test_sqs_message)}
     )
 
-    assert actual == expected
+    repo_under_test.dynamo_repository.write_report_upload_to_dynamo.assert_called()
+
+    mock_report_upload_failure.assert_called_with(
+        TEST_STAGING_METADATA, UploadStatus.FAILED, "Invalid scan date format", ""
+    )
+    repo_under_test.sqs_repository.send_message_to_pdf_stitching_queue.assert_not_called()
 
-    TEST_FILE_METADATA.scan_date = "03/09/2022"
-    TEST_DOCUMENT_REFERENCE.document_scan_creation = "2022-09-03"
+    TEST_STAGING_METADATA.files[0].scan_date = "2022-09-03"
 
 
 def test_raise_client_error_from_ssm_with_pds_service(

lambdas/tests/unit/utils/test_lloyd_george_validator.py

@@ -1,9 +1,10 @@
 import pytest
 from botocore.exceptions import ClientError
+from requests import Response
+
 from enums.supported_document_types import SupportedDocumentTypes
 from enums.validation_score import ValidationResult, ValidationScore
 from models.pds_models import Patient
-from requests import Response
 from services.base.ssm_service import SSMService
 from services.document_service import DocumentService
 from tests.unit.conftest import TEST_NHS_NUMBER, expect_not_to_raise
@@ -57,7 +58,7 @@
     validate_patient_date_of_birth,
     validate_patient_name_lenient,
     validate_patient_name_strict,
-    validate_patient_name_using_full_name_history,
+    validate_patient_name_using_full_name_history, validate_scan_date,
 )
 
 
@@ -1346,3 +1347,14 @@ def test_calculate_validation_score_for_lenient_check(
     )
     assert historical == expected_historical_match
     assert actual_result == expected_score
+
+
+def test_validates_correct_scan_date_format():
+    scan_date = "12-03-2023"
+    result = validate_scan_date(scan_date)
+    assert result == "2023-03-12"
+
+@pytest.mark.parametrize("scan_date", ["", "not-a-date", "1678246", "12/12/2024 12:12", "2023/03/12"])
+def test_raises_exception_for_invalid_scan_date_format(scan_date):
+    with pytest.raises(LGInvalidFilesException, match="Invalid scan date format"):
+        validate_scan_date(scan_date)

lambdas/tests/unit/utils/test_utilities.py

@@ -1,3 +1,5 @@
+from datetime import datetime
+
 import pytest
 from services.mock_pds_service import MockPdsApiService
 from services.pds_api_service import PdsApiService
@@ -8,6 +10,7 @@
     format_cloudfront_url,
     get_file_key_from_s3_url,
     get_pds_service,
+    parse_date,
     redact_id_to_last_4_chars,
     validate_nhs_number,
 )
@@ -111,3 +114,24 @@ def test_format_cloudfront_url_valid():
     cloudfront_domain = "d12345.cloudfront.net"
     expected_url = "https://d12345.cloudfront.net/path/to/resource"
     assert format_cloudfront_url(presign_url, cloudfront_domain) == expected_url
+
+
+@pytest.mark.parametrize(
+    "input_date, expected_date",
+    [
+        ("25/12/2023", datetime(2023, 12, 25)),
+        ("2023-12-25", datetime(2023, 12, 25)),
+        ("25-12-2023", datetime(2023, 12, 25)),
+        ("Dec 25, 2023", datetime(2023, 12, 25)),
+        ("25-Dec-2023", datetime(2023, 12, 25)),
+        ("24-NOV-2023", datetime(2023, 11, 24)),
+        ("12/12/2024 12:12", None),
+        ("25.12.2023", None),
+        ("", None),
+        ("test_text", None),
+        (None, None),
+    ],
+)
+def test_parse_date_returns_correct_date_for_valid_formats(input_date, expected_date):
+    result = parse_date(input_date)
+    assert result == expected_date

lambdas/utils/lloyd_george_validator.py

@@ -27,7 +27,7 @@
     name_ends_with,
     name_starts_with,
 )
-from utils.utilities import get_pds_service
+from utils.utilities import get_pds_service, parse_date
 
 logger = LoggingService(__name__)
 
@@ -386,3 +386,10 @@ def allowed_to_ingest_ods_code(patient_ods_code: str) -> bool:
         return True
 
     return patient_ods_code.upper() in allowed_ods_codes
+
+
+def validate_scan_date(scan_date: str):
+    if formatted_date := parse_date(scan_date):
+        return formatted_date.strftime("%Y-%m-%d")
+    else:
+        raise LGInvalidFilesException("Invalid scan date format")

lambdas/utils/utilities.py

@@ -105,3 +105,25 @@ def generate_date_folder_name(date: str) -> str:
 def format_cloudfront_url(presign_url: str, cloudfront_domain: str) -> str:
     formatted_url = f"https://{cloudfront_domain}/{presign_url}"
     return formatted_url
+
+
+def parse_date(date_string: str) -> datetime | None:
+    if not date_string:
+        return None
+
+    SUPPORTED_FORMATS = [
+        "%d/%m/%Y",
+        "%Y-%m-%d",
+        "%d-%m-%Y",
+        "%b %d, %Y",
+        "%d-%b-%Y",
+        "%d-%B-%Y",
+    ]
+
+    for fmt in SUPPORTED_FORMATS:
+        try:
+            date_object = datetime.strptime(date_string, fmt)
+            return date_object
+        except ValueError:
+            continue
+    return None