comments

Author: nhsdevws

recordprocessor/src/batch_processor.py

@@ -22,7 +22,6 @@ def process_csv_to_fhir(incoming_message_body: dict) -> int:
     """
     encoder = "utf-8"  # default encoding
     try:
-        # and encoder to the incoming message body
         incoming_message_body["encoder"] = encoder
         interim_message_body = file_level_validation(incoming_message_body=incoming_message_body)
     except (InvalidHeaders, NoOperationPermissions, Exception) as e:  # pylint: disable=broad-exception-caught
@@ -77,14 +76,11 @@ def process_rows(file_id, vaccine, supplier, file_key, allowed_operations, creat
                 row_id = f"{file_id}^{row_count}"
                 logger.info("MESSAGE ID : %s", row_id)
                 # Log progress every 1000 rows and the first 10 rows after a restart
-                if (total_rows_processed_count % 1000 == 0):
+                if total_rows_processed_count % 1000 == 0:
                     logger.info(f"Process: {total_rows_processed_count+1}")
-                if (start_row > 0 and row_count <= start_row+10):
+                if start_row > 0 and row_count <= start_row+10:
                     logger.info(f"Restarted Process (log up to first 10): {total_rows_processed_count+1}")
 
-                total_rows_processed_count += 1
-                logger.info(f"Process row: {total_rows_processed_count}")
-
                 # Process the row to obtain the details needed for the message_body and ack file
                 details_from_processing = process_row(target_disease, allowed_operations, row)
                 # Create the message body for sending
@@ -97,6 +93,7 @@ def process_rows(file_id, vaccine, supplier, file_key, allowed_operations, creat
                     **details_from_processing,
                 }
                 send_to_kinesis(supplier, outgoing_message_body, vaccine)
+                total_rows_processed_count += 1
 
     except Exception as error:  # pylint: disable=broad-exception-caught
         # if error reason is 'invalid continuation byte', then it's a decode error

recordprocessor/tests/test_recordprocessor_edge_cases.py

@@ -150,26 +150,3 @@ def test_process_small_file_utf8(self):
         self.mock_logger_warning.assert_not_called()
         self.mock_logger_error.assert_not_called()
 
-    def test_fix_cp1252(self):
-        # create a cp1252 string that contains an accented E
-        # this is not a unit test as such but checks/confirms encoding assumptions
-        source_text = b'D\xe9cembre'
-        test_dict = {
-            "date": source_text,
-            "name": "Test Name"}
-        utf8_dict = dict_decode(test_dict, "cp1252")
-        self.assertEqual(utf8_dict["date"], "Décembre")
-        self.assertEqual(utf8_dict["name"], "Test Name")
-
-
-def dict_decode(input_dict: dict, encoding: str) -> dict:
-    """
-    Decode all byte strings in a dictionary to UTF-8 strings using the specified encoding.
-    """
-    decoded_dict = {}
-    for key, value in input_dict.items():
-        if isinstance(value, bytes):
-            decoded_dict[key] = value.decode(encoding)
-        else:
-            decoded_dict[key] = value
-    return decoded_dict