VED-767 Non-ASCII Bug (#819)

* typo

* processing/{file_key}

* run standalone

* unit test

Author: JamesW1-NHS

recordprocessor/src/batch_processor.py

@@ -55,7 +55,7 @@ def process_csv_to_fhir(incoming_message_body: dict) -> int:
             encoder = new_encoder
 
             # load alternative encoder
-            csv_reader = get_csv_content_dict_reader(file_key, encoder=encoder)
+            csv_reader = get_csv_content_dict_reader(f"processing/{file_key}", encoder=encoder)
             # re-read the file and skip processed rows
             row_count, err = process_rows(file_id, vaccine, supplier, file_key, allowed_operations,
                                           created_at_formatted_string, csv_reader, target_disease, row_count)

recordprocessor/src/file_level_validation.py

@@ -89,8 +89,6 @@ def file_level_validation(incoming_message_body: dict) -> dict:
             csv_reader = get_csv_content_dict_reader(file_key, encoder="cp1252")
             validate_content_headers(csv_reader)
 
-        validate_content_headers(csv_reader)
-
         # Validate has permission to perform at least one of the requested actions
         allowed_operations_set = get_permitted_operations(supplier, vaccine, permission)
 

recordprocessor/tests/test_process_csv_to_fhir.py

@@ -6,38 +6,37 @@
 import boto3
 from moto import mock_s3, mock_firehose, mock_dynamodb
 
-from constants import FileStatus, AUDIT_TABLE_NAME
-from utils_for_recordprocessor_tests.generic_setup_and_teardown import (
+from tests.utils_for_recordprocessor_tests.generic_setup_and_teardown import (
     GenericSetUp,
     GenericTearDown,
 )
-from utils_for_recordprocessor_tests.utils_for_recordprocessor_tests import add_entry_to_table
-from utils_for_recordprocessor_tests.values_for_recordprocessor_tests import (
+from tests.utils_for_recordprocessor_tests.utils_for_recordprocessor_tests import add_entry_to_table
+from tests.utils_for_recordprocessor_tests.values_for_recordprocessor_tests import (
     MockFileDetails,
     ValidMockFileContent,
     REGION_NAME,
 )
-from utils_for_recordprocessor_tests.mock_environment_variables import MOCK_ENVIRONMENT_DICT, BucketNames
+from tests.utils_for_recordprocessor_tests.mock_environment_variables import MOCK_ENVIRONMENT_DICT, BucketNames
 
 with patch("os.environ", MOCK_ENVIRONMENT_DICT):
+    from constants import FileStatus, AUDIT_TABLE_NAME
     from batch_processor import process_csv_to_fhir
 
-
 dynamodb_client = boto3.client("dynamodb", region_name=REGION_NAME)
 s3_client = boto3.client("s3", region_name=REGION_NAME)
 firehose_client = boto3.client("firehose", region_name=REGION_NAME)
 test_file = MockFileDetails.rsv_emis
 
 
-@patch.dict("os.environ", MOCK_ENVIRONMENT_DICT)
 @mock_s3
 @mock_firehose
 @mock_dynamodb
+@patch.dict("os.environ", MOCK_ENVIRONMENT_DICT)
 class TestProcessCsvToFhir(unittest.TestCase):
     """Tests for process_csv_to_fhir function"""
 
     def setUp(self) -> None:
-        GenericSetUp(s3_client, firehose_client, dynamodb_client=dynamodb_client)
+        GenericSetUp(s3_client=s3_client, firehose_client=firehose_client, dynamodb_client=dynamodb_client)
 
         redis_patcher = patch("mappings.redis_client")
         self.addCleanup(redis_patcher.stop)
@@ -48,7 +47,7 @@ def setUp(self) -> None:
         }])
 
     def tearDown(self) -> None:
-        GenericTearDown(s3_client, firehose_client, dynamodb_client=dynamodb_client)
+        GenericTearDown(s3_client=s3_client, firehose_client=firehose_client, dynamodb_client=dynamodb_client)
 
     @staticmethod
     def upload_source_file(file_key, file_content):

recordprocessor/tests/test_recordprocessor_edge_cases.py

@@ -1,7 +1,7 @@
 import unittest
 import os
 from io import BytesIO
-from unittest.mock import patch
+from unittest.mock import call, patch
 from batch_processor import process_csv_to_fhir
 from tests.utils_for_recordprocessor_tests.utils_for_recordprocessor_tests import create_patch
 
@@ -69,6 +69,7 @@ def test_process_large_file_cp1252(self):
         message_body = {
                     "vaccine_type": "vax-type-1",
                     "supplier": "test-supplier",
+                    "filename": "test-filename"
                 }
         self.mock_map_target_disease.return_value = "some disease"
 
@@ -79,6 +80,10 @@ def test_process_large_file_cp1252(self):
         self.mock_logger_warning.assert_called()
         warning_call_args = self.mock_logger_warning.call_args[0][0]
         self.assertTrue(warning_call_args.startswith("Encoding Error: 'utf-8' codec can't decode byte 0xe9"))
+        self.mock_s3_get_object.assert_has_calls([
+            call(Bucket=None, Key="test-filename"),
+            call(Bucket=None, Key="processing/test-filename"),
+        ])
 
     def test_process_large_file_utf8(self):
         """ Test processing a large file with utf-8 encoding """