log rows returned

Author: nhsdevws

recordprocessor/src/batch_processor.py

@@ -13,10 +13,12 @@
 from typing import Optional
 
 
-def process_csv_to_fhir(incoming_message_body: dict) -> None:
+def process_csv_to_fhir(incoming_message_body: dict) -> int:
     """
     For each row of the csv, attempts to transform into FHIR format, sends a message to kinesis,
     and documents the outcome for each row in the ack file.
+    Returns the number of rows processed. While this is not used by the handler, the number of rows
+    processed must be correct and therefore is returned for logging and test purposes.
     """
     encoder = "utf-8"  # default encoding
     try:
@@ -66,7 +68,6 @@ def process_csv_to_fhir(incoming_message_body: dict) -> None:
             logger.error(f"Non-decode error: {err}. Cannot retry. Call someone.")
             raise err
 
-    logger.info("Total rows processed: %s", row_count)
     return row_count
 
 
@@ -118,11 +119,13 @@ def main(event: str) -> None:
     """Process each row of the file"""
     logger.info("task started")
     start = time.time()
+    n_rows_processed = 0
     try:
-        process_csv_to_fhir(incoming_message_body=json.loads(event))
+        n_rows_processed = process_csv_to_fhir(incoming_message_body=json.loads(event))
     except Exception as error:  # pylint: disable=broad-exception-caught
         logger.error("Error processing message: %s", error)
     end = time.time()
+    logger.info("Total rows processed: %s", n_rows_processed)
     logger.info("Total time for completion: %ss", round(end - start, 5))