validation error

Author: nhsdevws

recordprocessor/src/batch_processor.py

@@ -25,17 +25,8 @@ def process_csv_to_fhir(incoming_message_body: dict) -> int:
         # and encoder to the incoming message body
         incoming_message_body["encoder"] = encoder
         interim_message_body = file_level_validation(incoming_message_body=incoming_message_body)
-    except InvalidEncoding as error:
-        logger.warning("Invalid Encoding detected in process_csv_to_fhir: %s", error)
-        # retry with cp1252 encoding
-        encoder = "cp1252"
-        try:
-            incoming_message_body["encoder"] = encoder
-            interim_message_body = file_level_validation(incoming_message_body=incoming_message_body)
-        except Exception as error:
-            logger.error(f"Error in file_level_validation with {encoder} encoding: %s", error)
-            return 0
-    except (InvalidHeaders, NoOperationPermissions, Exception):  # pylint: disable=broad-exception-caught
+    except (InvalidHeaders, NoOperationPermissions, Exception) as e:  # pylint: disable=broad-exception-caught
+        logger.error(f"File level validation failed: {e}")
         # If the file is invalid, processing should cease immediately
         return 0
 

recordprocessor/src/file_level_validation.py

@@ -2,6 +2,7 @@
 Functions for completing file-level validation
 (validating headers and ensuring that the supplier has permission to perform at least one of the requested operations)
 """
+from curses import error
 from clients import logger, s3_client
 from make_and_upload_ack_file import make_and_upload_ack_file
 from utils_for_recordprocessor import get_csv_content_dict_reader, invoke_filename_lambda
@@ -79,9 +80,17 @@ def file_level_validation(incoming_message_body: dict) -> dict:
         encoder = incoming_message_body.get("encoder", "utf-8")
 
         # Fetch the data
-        csv_reader = get_csv_content_dict_reader(file_key, encoder=encoder)
-
-        validate_content_headers(csv_reader)
+        try:
+            csv_reader = get_csv_content_dict_reader(file_key, encoder=encoder)
+            validate_content_headers(csv_reader)
+        except Exception as e:
+            if hasattr(e, 'reason') and e.reason == "invalid continuation byte" and encoder == "utf-8":
+                logger.warning("Invalid Encoding detected: %s", e)
+                # retry with cp1252 encoding
+                csv_reader = get_csv_content_dict_reader(file_key, encoder="cp1252")
+                validate_content_headers(csv_reader)
+            else:
+                raise
 
         # Validate has permission to perform at least one of the requested actions
         allowed_operations_set = get_permitted_operations(supplier, vaccine, permission)

recordprocessor/tests/test_recordprocessor_edge_cases.py

@@ -104,6 +104,7 @@ def test_process_large_file_with_utf8(self):
     def test_process_cp1252_small_file(self):
         """ Test processing a small file with cp1252 encoding """
         data = self.create_test_data_from_file("test-batch-data-cp1252.csv")
+        data = self.insert_cp1252_at_end(data, b'D\xe9cembre', 2)
         data = [line if line.endswith(b"\n") else line + b"\n" for line in data]
         n_rows = len(data) - 1  # Exclude header
 
@@ -124,7 +125,7 @@ def test_process_cp1252_small_file(self):
         self.assertEqual(self.mock_send_to_kinesis.call_count, n_rows)
         self.mock_logger_warning.assert_called()
         warning_call_args = self.mock_logger_warning.call_args[0][0]
-        self.assertTrue(warning_call_args.startswith("Invalid Encoding detected in process_csv_to_fhir"))
+        self.assertTrue(warning_call_args.startswith("Invalid Encoding detected"))
 
     def test_process_utf8_small_file(self):
         """ Test processing a small file with utf-8 encoding """