Skip file encoding check for internal systems

Author: dlzhry2nhs

recordprocessor/src/constants.py

@@ -48,6 +48,9 @@
 ]
 
 
+INTERNAL_SUPPLIER_NAMES = {"DPSFULL", "DPSREDUCED"}
+
+
 class FileStatus:
     """File status constants"""
 

recordprocessor/src/file_level_validation.py

@@ -78,7 +78,7 @@ def file_level_validation(incoming_message_body: dict) -> dict:
         created_at_formatted_string = incoming_message_body.get("created_at_formatted_string")
 
         # Fetch the data
-        csv_reader = get_csv_content_dict_reader(file_key)
+        csv_reader = get_csv_content_dict_reader(file_key, supplier)
 
         validate_content_headers(csv_reader)
 

recordprocessor/src/utils_for_recordprocessor.py

@@ -5,7 +5,7 @@
 from csv import DictReader
 from io import BytesIO, TextIOWrapper
 from clients import s3_client, lambda_client, logger
-from constants import SOURCE_BUCKET_NAME, FILE_NAME_PROC_LAMBDA_NAME
+from constants import SOURCE_BUCKET_NAME, FILE_NAME_PROC_LAMBDA_NAME, INTERNAL_SUPPLIER_NAMES
 
 
 def get_environment() -> str:
@@ -15,18 +15,24 @@ def get_environment() -> str:
     return _env if _env in ["internal-dev", "int", "ref", "sandbox", "prod"] else "internal-dev"
 
 
-def get_csv_content_dict_reader(file_key: str) -> DictReader:
+def get_csv_content_dict_reader(file_key: str, supplier: str) -> DictReader:
     """Returns the requested file contents from the source bucket in the form of a DictReader"""
     response = s3_client.get_object(Bucket=SOURCE_BUCKET_NAME, Key=file_key)
     s3_object_bytes_io = BytesIO(response["Body"].read())
-    encoding = "utf-8" if is_utf8(s3_object_bytes_io, file_key) else "windows-1252"
+    encoding = "utf-8"
+
+    # Skip encoding check if supplier is internal e.g. DPS
+    if supplier not in INTERNAL_SUPPLIER_NAMES:
+        if not is_utf8(s3_object_bytes_io, file_key):
+            encoding = "windows-1252"
+
     text_io = TextIOWrapper(s3_object_bytes_io, encoding=encoding, newline="")
     return DictReader(text_io, delimiter="|")
 
 
 def is_utf8(file_bytes: BytesIO, file_key: str) -> bool:
-    """Best effort attempt to check if the given file is UTF-8. VED-754 some suppliers may provide non UTF-8
-    encoded CSV files e.g. Windows-1252, so we need to know whether or not to fallback"""
+    """Checks if the given file is UTF-8. VED-754 some suppliers may provide non UTF-8encoded CSV files
+    e.g. Windows-1252, so we need to know whether or not to fallback"""
     for line in file_bytes:
         try:
             line.decode("utf-8")

recordprocessor/tests/test_recordprocessor_main.py

@@ -60,9 +60,13 @@ def tearDown(self) -> None:
         GenericTearDown(s3_client, firehose_client, kinesis_client)
 
     @staticmethod
-    def upload_source_files(source_file_content):  # pylint: disable=dangerous-default-value
-        """Uploads a test file with the TEST_FILE_KEY (RSV EMIS file) the given file content to the source bucket"""
-        s3_client.put_object(Bucket=BucketNames.SOURCE, Key=mock_rsv_emis_file.file_key, Body=source_file_content)
+    def upload_source_files(
+        source_file_content: str,
+        file_key: str = mock_rsv_emis_file.file_key
+    ):
+        """Uploads a test file with the TEST_FILE_KEY (RSV EMIS file as default) and the given file content to the
+        source bucket"""
+        s3_client.put_object(Bucket=BucketNames.SOURCE, Key=file_key, Body=source_file_content)
 
     @staticmethod
     def get_shard_iterator():
@@ -96,7 +100,7 @@ def make_inf_ack_assertions(self, file_details: FileDetails, passed_validation:
 
         self.assertEqual(actual_rows, [InfAckFileRows.HEADERS, expected_row])
 
-    def make_kinesis_assertions(self, test_cases):
+    def make_kinesis_assertions(self, test_cases, file_details: FileDetails = mock_rsv_emis_file):
         """
         The input is a list of test_case tuples where each tuple is structured as
         (test_name, index, expected_kinesis_data_ignoring_fhir_json, expect_success).
@@ -122,7 +126,7 @@ def make_kinesis_assertions(self, test_cases):
             with self.subTest(test_name):
 
                 kinesis_record = kinesis_records[index]
-                self.assertEqual(kinesis_record["PartitionKey"], mock_rsv_emis_file.queue_name)
+                self.assertEqual(kinesis_record["PartitionKey"], file_details.queue_name)
                 self.assertEqual(kinesis_record["SequenceNumber"], f"{index+1}")
 
                 # Ensure that arrival times are sequential
@@ -132,11 +136,11 @@ def make_kinesis_assertions(self, test_cases):
 
                 kinesis_data = json.loads(kinesis_record["Data"].decode("utf-8"), parse_float=Decimal)
                 expected_kinesis_data = {
-                    "row_id": f"{mock_rsv_emis_file.message_id}^{index+1}",
-                    "file_key": mock_rsv_emis_file.file_key,
-                    "supplier": mock_rsv_emis_file.supplier,
-                    "vax_type": mock_rsv_emis_file.vaccine_type,
-                    "created_at_formatted_string": mock_rsv_emis_file.created_at_formatted_string,
+                    "row_id": f"{file_details.message_id}^{index+1}",
+                    "file_key": file_details.file_key,
+                    "supplier": file_details.supplier,
+                    "vax_type": file_details.vaccine_type,
+                    "created_at_formatted_string": file_details.created_at_formatted_string,
                     **expected_kinesis_data,
                 }
                 if expect_success and "fhir_json" not in expected_kinesis_data:
@@ -370,7 +374,7 @@ def test_e2e_successfully_processes_windows_1252_encoded_file_contents(self):
 
         main(mock_rsv_emis_file.event_full_permissions)
 
-        # Assertion case tuples are stuctured as
+        # Assertion case tuples are structured as
         # (test_name, index, expected_kinesis_data_ignoring_fhir_json,expect_success)
         assertion_cases = [
             (
@@ -387,6 +391,34 @@ def test_e2e_successfully_processes_windows_1252_encoded_file_contents(self):
             mock_rsv_emis_file.file_key
         )
 
+    @patch("utils_for_recordprocessor.is_utf8")
+    def test_e2e_skips_encoding_check_for_internal_suppliers(self, mock_utf_checker):
+        """
+        VED-754 tests the handler successfully processes the file and skips the encoding check when the supplier is
+        internal e.g. DPS. This is because we know they will be using the desired utf-8 encoding and avoids the extra
+        processing time to validate the encoding. They will also be providing very large batch files so skipping this is
+        very important.
+        """
+        self.upload_source_files(ValidMockFileContent.with_new_dps_record, MockFileDetails.dps_flu.file_key)
+
+        main(MockFileDetails.dps_flu.event_full_permissions)
+
+        # Assertion case tuples are structured as
+        # (test_name, index, expected_kinesis_data_ignoring_fhir_json,expect_success)
+        assertion_cases = [
+            (
+                "CREATE success",
+                0,
+                {"operation_requested": "CREATE", "local_id": "DPS_ID_1234^DPS_SYSTEM"},
+                True,
+            )
+        ]
+        self.make_inf_ack_assertions(file_details=MockFileDetails.dps_flu, passed_validation=True)
+        self.mock_util_logger.info.assert_not_called()
+        mock_utf_checker.assert_not_called()
+        self.make_kinesis_assertions(assertion_cases, file_details=MockFileDetails.dps_flu)
+        self.mock_util_logger.info.assert_not_called()
+
 
 if __name__ == "__main__":
     unittest.main()

recordprocessor/tests/test_utils_for_recordprocessor.py

@@ -48,7 +48,7 @@ def test_get_csv_content_dict_reader(self):
         """Tests that get_csv_content_dict_reader returns the correct csv data"""
         self.upload_source_file(test_file.file_key, ValidMockFileContent.with_new_and_update)
         expected_output = csv.DictReader(StringIO(ValidMockFileContent.with_new_and_update), delimiter="|")
-        result = get_csv_content_dict_reader(test_file.file_key)
+        result = get_csv_content_dict_reader(test_file.file_key, "RAVS")
         self.assertEqual(list(result), list(expected_output))
 
     def test_get_environment(self):

recordprocessor/tests/utils_for_recordprocessor_tests/values_for_recordprocessor_tests.py

@@ -112,6 +112,17 @@ class MockFileRows:
         '"J82068"|"https://fhir.nhs.uk/Id/ods-organization-code"'
     )
 
+    NEW_DPS_RECORD = (
+        '9674963871|"SABINA"|"GREIR"|"20190131"|"2"|"GU14 6TU"|"20240610T183325"|"J82067"|'
+        '"https://fhir.nhs.uk/Id/ods-organization-code"|"DPS_ID_1234"|"DPS_SYSTEM"|'
+        '"new"|"Ellena"|"O\'Reilly"|"20240101"|"TRUE"|'
+        '"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|'
+        '1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) '
+        '(product)"|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|'
+        '"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"|"Test"|'
+        '"J82067"|"https://fhir.nhs.uk/Id/ods-organization-code"'
+    )
+
     # For test case VED-754 - windows-1252 encoding issues only surfaces with characters outside of 0-127 ASCII
     NEW_WITH_SPECIAL_CHARACTERS = (
         '9674963871|"SABINA"|"GRÉIR"|"20190131"|"2"|"GU14 6TU"|"20240610T183325"|"J82067"|'
@@ -131,6 +142,7 @@ class ValidMockFileContent:
     headers = MockFileRows.HEADERS
     with_new = headers + "\n" + MockFileRows.NEW
     with_new_special_char = headers + "\n" + MockFileRows.NEW_WITH_SPECIAL_CHARACTERS
+    with_new_dps_record = headers + "\n" + MockFileRows.NEW_DPS_RECORD
     with_update = headers + "\n" + MockFileRows.UPDATE
     with_delete = headers + "\n" + MockFileRows.DELETE
     with_update_and_delete = headers + "\n" + MockFileRows.UPDATE + "\n" + MockFileRows.DELETE
@@ -210,6 +222,7 @@ class MockFileDetails:
     rsv_emis = FileDetails("RSV", "EMIS", "8HK48")
     flu_emis = FileDetails("FLU", "EMIS", "YGM41")
     ravs_flu = FileDetails("FLU", "RSV", "X26")
+    dps_flu = FileDetails("FLU", "DPSFULL", "DPSFULL")
 
 
 class UnorderedFieldDictionaries: