validation error

Author: nhsdevws

.github/workflows/sonarcloud.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
 
@@ -60,7 +60,7 @@ jobs:
         continue-on-error: true
         run: |
           poetry install
-          poetry run coverage run -m unittest discover -s "./tests" -p "*batch*.py" || echo "recordforwarder tests failed" >> ../failed_tests.txt
+          poetry run coverage run -m unittest discover -p "*batch*.py" || echo "recordforwarder tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../recordforwarder-coverage.xml
 
       - name: Run unittest with coverage-ack-lambda

recordprocessor/pyproject.toml

@@ -1,10 +1,10 @@
 [tool.poetry]
-name = "filenameprocessor"
-version = "0.1.0"
+name        = "processor"
+version     = "0.1.0"
 description = ""
-readme = "README.md"
-authors = ["Your Name <[email protected]>"]
-packages = [{ include = "src" }]
+authors     = ["Your Name <[email protected]>"]
+readme      = "README.md"
+packages    = [{include = "src"}]
 
 [tool.poetry.dependencies]
 python = "~3.11"

recordprocessor/src/batch_processor.py

@@ -25,17 +25,8 @@ def process_csv_to_fhir(incoming_message_body: dict) -> int:
         # and encoder to the incoming message body
         incoming_message_body["encoder"] = encoder
         interim_message_body = file_level_validation(incoming_message_body=incoming_message_body)
-    except InvalidEncoding as error:
-        logger.warning("Invalid Encoding detected in process_csv_to_fhir: %s", error)
-        # retry with cp1252 encoding
-        encoder = "cp1252"
-        try:
-            incoming_message_body["encoder"] = encoder
-            interim_message_body = file_level_validation(incoming_message_body=incoming_message_body)
-        except Exception as error:
-            logger.error(f"Error in file_level_validation with {encoder} encoding: %s", error)
-            return 0
-    except (InvalidHeaders, NoOperationPermissions, Exception):  # pylint: disable=broad-exception-caught
+    except (InvalidHeaders, NoOperationPermissions, Exception) as e:  # pylint: disable=broad-exception-caught
+        logger.error(f"File level validation failed: {e}")
         # If the file is invalid, processing should cease immediately
         return 0
 

recordprocessor/src/file_level_validation.py

@@ -79,9 +79,17 @@ def file_level_validation(incoming_message_body: dict) -> dict:
         encoder = incoming_message_body.get("encoder", "utf-8")
 
         # Fetch the data
-        csv_reader = get_csv_content_dict_reader(file_key, encoder=encoder)
-
-        validate_content_headers(csv_reader)
+        try:
+            csv_reader = get_csv_content_dict_reader(file_key, encoder=encoder)
+            validate_content_headers(csv_reader)
+        except Exception as e:
+            if hasattr(e, 'reason') and e.reason == "invalid continuation byte" and encoder == "utf-8":
+                logger.warning("Invalid Encoding detected: %s", e)
+                # retry with cp1252 encoding
+                csv_reader = get_csv_content_dict_reader(file_key, encoder="cp1252")
+                validate_content_headers(csv_reader)
+            else:
+                raise
 
         # Validate has permission to perform at least one of the requested actions
         allowed_operations_set = get_permitted_operations(supplier, vaccine, permission)

recordprocessor/tests/test_recordprocessor_edge_cases.py

@@ -54,7 +54,7 @@ def insert_cp1252_at_end(self, data: list[bytes], new_text: bytes, field: int) -
             break
         return data
 
-    def test_process_large_file_with_cp1252(self):
+    def test_process_large_file_cp1252(self):
         """ Test processing a large file with cp1252 encoding """
         n_rows = 500
         data = self.create_test_data_from_file("test-batch-data.csv")
@@ -79,7 +79,7 @@ def test_process_large_file_with_cp1252(self):
         warning_call_args = self.mock_logger_warning.call_args[0][0]
         self.assertEqual(warning_call_args, "Processing Error: Invalid continuation byte.")
 
-    def test_process_large_file_with_utf8(self):
+    def test_process_large_file_utf8(self):
         """ Test processing a large file with utf-8 encoding """
         n_rows = 500
         data = self.create_test_data_from_file("test-batch-data.csv")
@@ -101,9 +101,10 @@ def test_process_large_file_with_utf8(self):
         self.mock_logger_warning.assert_not_called()
         self.mock_logger_error.assert_not_called()
 
-    def test_process_cp1252_small_file(self):
+    def test_process_small_file_cp1252(self):
         """ Test processing a small file with cp1252 encoding """
         data = self.create_test_data_from_file("test-batch-data-cp1252.csv")
+        data = self.insert_cp1252_at_end(data, b'D\xe9cembre', 2)
         data = [line if line.endswith(b"\n") else line + b"\n" for line in data]
         n_rows = len(data) - 1  # Exclude header
 
@@ -124,9 +125,9 @@ def test_process_cp1252_small_file(self):
         self.assertEqual(self.mock_send_to_kinesis.call_count, n_rows)
         self.mock_logger_warning.assert_called()
         warning_call_args = self.mock_logger_warning.call_args[0][0]
-        self.assertTrue(warning_call_args.startswith("Invalid Encoding detected in process_csv_to_fhir"))
+        self.assertTrue(warning_call_args.startswith("Invalid Encoding detected"))
 
-    def test_process_utf8_small_file(self):
+    def test_process_small_file_utf8(self):
         """ Test processing a small file with utf-8 encoding """
         data = self.create_test_data_from_file("test-batch-data.csv")
         data = [line if line.endswith(b"\n") else line + b"\n" for line in data]