tidy

Author: nhsdevws

recordprocessor/src/batch_processor.py

@@ -50,10 +50,10 @@ def process_csv_to_fhir(incoming_message_body: dict) -> None:
     logger.info(f"process with encoder {encoder} from row {row_count+1}")
     row_count, err = process_rows(file_id, vaccine, supplier, file_key, allowed_operations,
                                   created_at_formatted_string, csv_reader, target_disease)
+
     if err:
-        logger.warning(f"Error processing: {err}.")
-        # check if it's a decode error
-        if err.reason == "invalid continuation byte":
+        logger.warning(f"Processing Error: {err}.")
+        if isinstance(err, InvalidEncoding):
             new_encoder = "cp1252"
             logger.info(f"Encode error at row {row_count} with {encoder}. Switch to {new_encoder}")
             encoder = new_encoder
@@ -108,7 +108,7 @@ def process_rows(file_id, vaccine, supplier, file_key, allowed_operations, creat
         # if error reason is 'invalid continuation byte', then it's a decode error
         logger.error("Error processing row %s: %s", row_count, error)
         if hasattr(error, 'reason') and error.reason == "invalid continuation byte":
-            return total_rows_processed_count, error
+            return total_rows_processed_count, InvalidEncoding("Invalid continuation byte")
         else:
             raise error
     return total_rows_processed_count, None

recordprocessor/tests/test_recordprocessor_edge_cases.py

@@ -77,7 +77,7 @@ def test_process_large_file_with_cp1252(self):
         # check logger.warning called for decode error
         self.mock_logger_warning.assert_called()
         warning_call_args = self.mock_logger_warning.call_args[0][0]
-        self.assertTrue(warning_call_args.startswith("Error processing: 'utf-8' codec can't decode byte"))
+        self.assertEqual(warning_call_args, "Processing Error: Invalid continuation byte.")
 
     def test_process_large_file_with_utf8(self):
         """ Test processing a large file with utf-8 encoding """