unit tests - failures=1, errors=4

Author: nhsdevws

recordprocessor/src/batch_processor.py

@@ -20,13 +20,16 @@ def process_csv_to_fhir(incoming_message_body: dict) -> None:
     """
     encoder = "utf-8"  # default encoding
     try:
-        interim_message_body = file_level_validation(incoming_message_body=incoming_message_body, encoder=encoder)
+        # and encoder to the incoming message body
+        incoming_message_body["encoder"] = encoder
+        interim_message_body = file_level_validation(incoming_message_body=incoming_message_body)
     except InvalidEncoding as error:
         logger.warning("Invalid Encoding detected in process_csv_to_fhir: %s", error)
         # retry with cp1252 encoding
         encoder = "cp1252"
         try:
-            interim_message_body = file_level_validation(incoming_message_body=incoming_message_body, encoder=encoder)
+            incoming_message_body["encoder"] = encoder
+            interim_message_body = file_level_validation(incoming_message_body=incoming_message_body)
         except Exception as error:
             logger.error(f"Error in file_level_validation with {encoder} encoding: %s", error)
             return 0
@@ -102,8 +105,12 @@ def process_rows(file_id, vaccine, supplier, file_key, allowed_operations, creat
                 total_rows_processed_count += 1
                 logger.info("Total rows processed: %s", total_rows_processed_count)
     except Exception as error:  # pylint: disable=broad-exception-caught
+        # if error reason is 'invalid continuation byte', then it's a decode error
         logger.error("Error processing row %s: %s", row_count, error)
-        return total_rows_processed_count, error
+        if hasattr(error, 'reason') and error.reason == "invalid continuation byte":
+            return total_rows_processed_count, error
+        else:
+            raise error
     return total_rows_processed_count, None
 
 

recordprocessor/src/file_level_validation.py

@@ -61,7 +61,7 @@ def move_file(bucket_name: str, source_file_key: str, destination_file_key: str)
 
 
 @file_level_validation_logging_decorator
-def file_level_validation(incoming_message_body: dict, encoder: str) -> dict:
+def file_level_validation(incoming_message_body: dict) -> dict:
     """
     Validates that the csv headers are correct and that the supplier has permission to perform at least one of
     the requested operations. Uploades the inf ack file and moves the source file to the processing folder.
@@ -76,6 +76,7 @@ def file_level_validation(incoming_message_body: dict, encoder: str) -> dict:
         file_key = incoming_message_body.get("filename")
         permission = incoming_message_body.get("permission")
         created_at_formatted_string = incoming_message_body.get("created_at_formatted_string")
+        encoder = incoming_message_body.get("encoder", "utf-8")
 
         # Fetch the data
         csv_reader = get_csv_content_dict_reader(file_key, encoder=encoder)
@@ -99,9 +100,10 @@ def file_level_validation(incoming_message_body: dict, encoder: str) -> dict:
             "csv_dict_reader": csv_reader,
         }
     except (InvalidHeaders, NoOperationPermissions, Exception) as error:
-        if error.reason == "invalid continuation byte" and encoder == "utf-8":
-            # propagate the error to trigger a retry with cp1252 encoding
-            raise InvalidEncoding(f"Error File encoding {encoder} is invalid.")
+        reason = getattr(error, 'reason', None)
+        if reason is not None:
+            if reason == "invalid continuation byte" and encoder == "utf-8":
+                raise InvalidEncoding(f"Error File encoding {encoder} is invalid.")
         logger.error("Error in file_level_validation: %s", error)
 
         # NOTE: The Exception may occur before the file_id, file_key and created_at_formatted_string are assigned