comment - replace InvalidEncoding with UnicodeDecodeError

Author: nhsdevws

recordprocessor/src/batch_processor.py

@@ -8,7 +8,7 @@
 from send_to_kinesis import send_to_kinesis
 from clients import logger
 from file_level_validation import file_level_validation
-from errors import NoOperationPermissions, InvalidHeaders, InvalidEncoding
+from errors import NoOperationPermissions, InvalidHeaders
 from utils_for_recordprocessor import get_csv_content_dict_reader
 from typing import Optional
 
@@ -43,7 +43,7 @@ def process_csv_to_fhir(incoming_message_body: dict) -> int:
                                   created_at_formatted_string, csv_reader, target_disease)
 
     if err:
-        if isinstance(err, InvalidEncoding):
+        if isinstance(err, UnicodeDecodeError):
             """ resolves encoding issue VED-754 """
             logger.warning(f"Encoding Error: {err}.")
             new_encoder = "cp1252"
@@ -95,13 +95,10 @@ def process_rows(file_id, vaccine, supplier, file_key, allowed_operations, creat
                 send_to_kinesis(supplier, outgoing_message_body, vaccine)
                 total_rows_processed_count += 1
 
-    except Exception as error:  # pylint: disable=broad-exception-caught
-        # if error reason is 'invalid continuation byte', then it's a decode error
+    except UnicodeDecodeError as error:  # pylint: disable=broad-exception-caught
         logger.error("Error processing row %s: %s", row_count, error)
-        if hasattr(error, 'reason') and error.reason == "invalid continuation byte":
-            return total_rows_processed_count, InvalidEncoding("Invalid continuation byte")
-        else:
-            raise error
+        return total_rows_processed_count, error
+
     return total_rows_processed_count, None
 
 

recordprocessor/src/errors.py

@@ -9,9 +9,5 @@ class InvalidHeaders(Exception):
     """A custom exception for when the file headers are invalid."""
 
 
-class InvalidEncoding(Exception):
-    """A custom exception for when the file encoding is invalid."""
-
-
 class UnhandledAuditTableError(Exception):
     """A custom exception for when an unexpected error occurs whilst adding the file to the audit table."""

recordprocessor/src/file_level_validation.py

@@ -5,7 +5,7 @@
 from clients import logger, s3_client
 from make_and_upload_ack_file import make_and_upload_ack_file
 from utils_for_recordprocessor import get_csv_content_dict_reader, invoke_filename_lambda
-from errors import InvalidHeaders, NoOperationPermissions, InvalidEncoding
+from errors import InvalidHeaders, NoOperationPermissions
 from logging_decorator import file_level_validation_logging_decorator
 from audit_table import change_audit_table_status_to_processed, get_next_queued_file_details
 from constants import SOURCE_BUCKET_NAME, EXPECTED_CSV_HEADERS, permission_to_operation_map, Permission
@@ -83,11 +83,10 @@ def file_level_validation(incoming_message_body: dict) -> dict:
             csv_reader = get_csv_content_dict_reader(file_key, encoder=encoder)
             validate_content_headers(csv_reader)
         except UnicodeDecodeError as e:
-            if hasattr(e, 'reason') and e.reason == "invalid continuation byte" and encoder == "utf-8":
-                logger.warning("Invalid Encoding detected: %s", e)
-                # retry with cp1252 encoding
-                csv_reader = get_csv_content_dict_reader(file_key, encoder="cp1252")
-                validate_content_headers(csv_reader)
+            logger.warning("Invalid Encoding detected: %s", e)
+            # retry with cp1252 encoding
+            csv_reader = get_csv_content_dict_reader(file_key, encoder="cp1252")
+            validate_content_headers(csv_reader)
 
         # Validate has permission to perform at least one of the requested actions
         allowed_operations_set = get_permitted_operations(supplier, vaccine, permission)