Comments

Author: nhsdevws

recordprocessor/src/batch_processor.py

@@ -82,9 +82,6 @@ def process_rows(file_id, vaccine, supplier, file_key, allowed_operations, creat
                 if (start_row > 0 and row_count <= start_row+10):
                     logger.info(f"Restarted Process (log up to first 10): {total_rows_processed_count+1}")
 
-                total_rows_processed_count += 1
-                logger.info(f"Process row: {total_rows_processed_count}")
-
                 # Process the row to obtain the details needed for the message_body and ack file
                 details_from_processing = process_row(target_disease, allowed_operations, row)
                 # Create the message body for sending
@@ -97,6 +94,7 @@ def process_rows(file_id, vaccine, supplier, file_key, allowed_operations, creat
                     **details_from_processing,
                 }
                 send_to_kinesis(supplier, outgoing_message_body, vaccine)
+                total_rows_processed_count += 1
 
     except Exception as error:  # pylint: disable=broad-exception-caught
         # if error reason is 'invalid continuation byte', then it's a decode error

recordprocessor/tests/test_recordprocessor_edge_cases.py

@@ -150,26 +150,3 @@ def test_process_small_file_utf8(self):
         self.mock_logger_warning.assert_not_called()
         self.mock_logger_error.assert_not_called()
 
-    def test_fix_cp1252(self):
-        # create a cp1252 string that contains an accented E
-        # this is not a unit test as such but checks/confirms encoding assumptions
-        source_text = b'D\xe9cembre'
-        test_dict = {
-            "date": source_text,
-            "name": "Test Name"}
-        utf8_dict = dict_decode(test_dict, "cp1252")
-        self.assertEqual(utf8_dict["date"], "Décembre")
-        self.assertEqual(utf8_dict["name"], "Test Name")
-
-
-def dict_decode(input_dict: dict, encoding: str) -> dict:
-    """
-    Decode all byte strings in a dictionary to UTF-8 strings using the specified encoding.
-    """
-    decoded_dict = {}
-    for key, value in input_dict.items():
-        if isinstance(value, bytes):
-            decoded_dict[key] = value.decode(encoding)
-        else:
-            decoded_dict[key] = value
-    return decoded_dict