mocking with filenameprocessor tests

Author: Akol125

filenameprocessor/src/file_key_validation.py

@@ -3,7 +3,7 @@
 from re import match
 from datetime import datetime
 from constants import Constants
-from vaccine_types import get_valid_vaccine_types_from_cache
+from supplier_permissions import get_valid_vaccine_types_from_cache
 from utils_for_filenameprocessor import identify_supplier
 from errors import InvalidFileKeyError
 

filenameprocessor/src/supplier_permissions.py

@@ -4,7 +4,6 @@
 import json
 from constants import PERMISSIONS_CONFIG_FILE_KEY, VACCINE_TYPE_TO_DISEASES_HASH_KEY
 from errors import VaccineTypePermissionsError
-from elasticache import get_permissions_config_json_from_cache
 
 
 def get_supplier_permissions(supplier: str) -> list[str]:
@@ -22,6 +21,9 @@ def get_permissions_config_json_from_cache() -> dict:
     """Gets and returns the permissions config file content from ElastiCache (Redis)."""
     return json.loads(redis_client.get(PERMISSIONS_CONFIG_FILE_KEY))
 
+def get_valid_vaccine_types_from_cache() -> list[str]:
+    return redis_client.hkeys(VACCINE_TYPE_TO_DISEASES_HASH_KEY)
+
 
 def validate_vaccine_type_permissions(vaccine_type: str, supplier: str) -> list:
     """
@@ -31,7 +33,7 @@ def validate_vaccine_type_permissions(vaccine_type: str, supplier: str) -> list:
     supplier_permissions = get_supplier_permissions(supplier)
 
     # Validate that supplier has at least one permissions for the vaccine type
-    if not any(permission.split(".")[0] == vaccine_type for permission in supplier_permissions):
+    if not any(vaccine_type in permission for permission in supplier_permissions):
         error_message = f"Initial file validation failed: {supplier} does not have permissions for {vaccine_type}"
         logger.error(error_message)
         raise VaccineTypePermissionsError(error_message)

filenameprocessor/src/utils_for_filenameprocessor.py

@@ -19,10 +19,14 @@ def identify_supplier(ods_code: str) -> str:
     return Constants.ODS_TO_SUPPLIER_MAPPINGS.get(ods_code, "")
 
 
-def move_file(bucket_name: str, source_file_key: str, destination_file_key: str) -> None:
-    """Moves a file from one location to another within a single S3 bucket by copying and then deleting the file."""
+def move_file(s3_client, bucket_name: str, source_file_key: str, destination_file_key: str) -> None:
+    """
+    Moves a file from one location to another within a single S3 bucket by copying and then deleting the file.
+    """
     s3_client.copy_object(
-        Bucket=bucket_name, CopySource={"Bucket": bucket_name, "Key": source_file_key}, Key=destination_file_key
+        Bucket=bucket_name,
+        CopySource={"Bucket": bucket_name, "Key": source_file_key},
+        Key=destination_file_key,
     )
     s3_client.delete_object(Bucket=bucket_name, Key=source_file_key)
     logger.info("File moved from %s to %s", source_file_key, destination_file_key)

filenameprocessor/tests/test_audit_table.py

@@ -2,8 +2,8 @@
 
 from unittest import TestCase
 from unittest.mock import patch
-from boto3 import client as boto3_client
-from moto import mock_dynamodb
+import boto3
+from moto import mock_dynamodb, mock_sqs
 
 from tests.utils_for_tests.mock_environment_variables import MOCK_ENVIRONMENT_DICT
 from tests.utils_for_tests.generic_setup_and_teardown import GenericSetUp, GenericTearDown
@@ -23,15 +23,34 @@
 
 FILE_DETAILS = MockFileDetails.ravs_rsv_1
 
-
+@mock_sqs
 @mock_dynamodb
 @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT)
 class TestAuditTable(TestCase):
     """Tests for audit table functions"""
 
     def setUp(self):
         """Set up test values to be used for the tests"""
-        self.dynamodb_client = boto3_client("dynamodb", region_name=REGION_NAME)
+        self.dynamodb_client = boto3.client("dynamodb", region_name=REGION_NAME)
+        self.dynamodb_resource = boto3.resource("dynamodb", region_name=REGION_NAME)
+
+        # Patch both the client and resource inside the audit_table module
+        patcher_client = patch("audit_table.dynamodb_client", self.dynamodb_client)
+        patcher_resource = patch("audit_table.dynamodb_resource", self.dynamodb_resource)
+
+        patcher_client.start()
+        patcher_resource.start()
+
+        self.addCleanup(patcher_client.stop)
+        self.addCleanup(patcher_resource.stop)
+
+        # Patch both the client and resource used inside the audit_table module
+        patcher_client = patch("audit_table.dynamodb_client", self.dynamodb_client)
+
+        patcher_client.start()
+
+        self.addCleanup(patcher_client.stop)
+
         GenericSetUp(dynamodb_client=self.dynamodb_client)
 
     def tearDown(self):
@@ -68,6 +87,7 @@ def test_get_next_queued_file_details(self):
         add_entry_to_table(MockFileDetails.ravs_rsv_3, file_status=FileStatus.QUEUED)
         add_entry_to_table(MockFileDetails.ravs_rsv_4, file_status=FileStatus.QUEUED)
         self.assertEqual(get_next_queued_file_details(queue_to_check), deserialize_dynamodb_types(expected_table_entry))
+    
 
     def test_ensure_file_is_not_a_duplicate(self):
         """
@@ -222,7 +242,7 @@ def test_upsert_audit_table(self):
             )
 
         # Final reconciliation: ensure that all of the correct items are in the audit table
-        table_items = self.get_table_items()
+        table_items = self.get_table_items(self)
         assert len(table_items) == 7
         assert_audit_table_entry(MockFileDetails.emis_flu, FileStatus.QUEUED)
         assert_audit_table_entry(MockFileDetails.emis_rsv, FileStatus.QUEUED)

filenameprocessor/tests/test_file_key_validation.py

@@ -38,7 +38,7 @@ def test_is_valid_datetime(self):
             with self.subTest():
                 self.assertEqual(is_valid_datetime(date_time_string), expected_result)
 
-    @patch("elasticache.redis_client.hkeys", return_value=["FLU", "RSV"])
+    @patch("supplier_permissions.redis_client.hkeys", return_value=["FLU", "RSV"])
     def test_validate_file_key(self, _mock_hkeys):
         """Tests that file_key_validation returns True if all elements pass validation, and False otherwise"""
         # Test case tuples are structured as (file_key, expected_result)

filenameprocessor/tests/test_make_and_upload_ack_file.py

@@ -3,8 +3,8 @@
 from unittest import TestCase
 from unittest.mock import patch
 from copy import deepcopy
-from boto3 import client as boto3_client
-from moto import mock_s3
+import boto3
+from moto import mock_s3, mock_sqs
 
 from tests.utils_for_tests.utils_for_filenameprocessor_tests import get_csv_file_dict_reader
 from tests.utils_for_tests.mock_environment_variables import MOCK_ENVIRONMENT_DICT, BucketNames
@@ -15,9 +15,6 @@
     from make_and_upload_ack_file import make_the_ack_data, upload_ack_file, make_and_upload_the_ack_file
     from clients import REGION_NAME
 
-
-s3_client = boto3_client("s3", region_name=REGION_NAME)
-
 FILE_DETAILS = MockFileDetails.emis_flu
 
 # NOTE: The expected ack data is the same for all scenarios as the ack file is only created if an error occurs
@@ -38,14 +35,16 @@
 }
 
 
+@mock_sqs
 @mock_s3
 @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT)
 class TestMakeAndUploadAckFile(TestCase):
     """Tests for make_and_upload_ack_file functions"""
 
     def setUp(self):
         """Set up the bucket for the ack files"""
-        s3_client.create_bucket(
+        self.s3_client = boto3.client("s3", region_name=REGION_NAME)
+        self.s3_client.create_bucket(
             Bucket=BucketNames.DESTINATION, CreateBucketConfiguration={"LocationConstraint": REGION_NAME}
         )
 
@@ -63,29 +62,31 @@ def test_make_ack_data(self):
 
     def test_upload_ack_file(self):
         """Test that upload_ack_file successfully uploads the ack file"""
-        upload_ack_file(
-            file_key=FILE_DETAILS.file_key,
-            ack_data=deepcopy(EXPECTED_ACK_DATA),
-            created_at_formatted_string=FILE_DETAILS.created_at_formatted_string,
-        )
+        with patch("make_and_upload_ack_file.s3_client", self.s3_client):
+            upload_ack_file(
+                file_key=FILE_DETAILS.file_key,
+                ack_data=deepcopy(EXPECTED_ACK_DATA),
+                created_at_formatted_string=FILE_DETAILS.created_at_formatted_string,
+            )
 
         expected_result = [deepcopy(EXPECTED_ACK_DATA)]
         # Note that the data downloaded from the CSV will contain the bool as a string
         expected_result[0]["MESSAGE_DELIVERY"] = "False"
-        csv_dict_reader = get_csv_file_dict_reader(s3_client, BucketNames.DESTINATION, FILE_DETAILS.ack_file_key)
+        csv_dict_reader = get_csv_file_dict_reader(self.s3_client, BucketNames.DESTINATION, FILE_DETAILS.ack_file_key)
         self.assertEqual(list(csv_dict_reader), expected_result)
 
     def test_make_and_upload_ack_file(self):
         """Test that make_and_upload_ack_file uploads an ack file containing the correct values"""
-        make_and_upload_the_ack_file(
-            message_id=FILE_DETAILS.message_id,
-            file_key=FILE_DETAILS.file_key,
-            message_delivered=False,
-            created_at_formatted_string=FILE_DETAILS.created_at_formatted_string,
-        )
+        with patch("make_and_upload_ack_file.s3_client", self.s3_client):
+            make_and_upload_the_ack_file(
+                message_id=FILE_DETAILS.message_id,
+                file_key=FILE_DETAILS.file_key,
+                message_delivered=False,
+                created_at_formatted_string=FILE_DETAILS.created_at_formatted_string,
+            )
 
         expected_result = [deepcopy(EXPECTED_ACK_DATA)]
         # Note that the data downloaded from the CSV will contain the bool as a string
         expected_result[0]["MESSAGE_DELIVERY"] = "False"
-        csv_dict_reader = get_csv_file_dict_reader(s3_client, BucketNames.DESTINATION, FILE_DETAILS.ack_file_key)
+        csv_dict_reader = get_csv_file_dict_reader(self.s3_client, BucketNames.DESTINATION, FILE_DETAILS.ack_file_key)
         self.assertEqual(list(csv_dict_reader), expected_result)

filenameprocessor/tests/test_send_sqs_message.py

@@ -5,7 +5,7 @@
 from json import loads as json_loads
 from copy import deepcopy
 from moto import mock_sqs
-from boto3 import client as boto3_client
+import boto3
 
 from tests.utils_for_tests.mock_environment_variables import MOCK_ENVIRONMENT_DICT, Sqs
 from tests.utils_for_tests.values_for_tests import MockFileDetails
@@ -16,10 +16,9 @@
     from errors import UnhandledSqsError, InvalidSupplierError
     from clients import REGION_NAME
 
-sqs_client = boto3_client("sqs", region_name=REGION_NAME)
-
 FLU_EMIS_FILE_DETAILS = MockFileDetails.emis_flu
 RSV_RAVS_FILE_DETAILS = MockFileDetails.ravs_rsv_1
+fake_queue_url = "https://sqs.eu-west-2.amazonaws.com/123456789012/non_existent_queue"
 
 NON_EXISTENT_QUEUE_ERROR_MESSAGE = (
     "An unexpected error occurred whilst sending to SQS: An error occurred (AWS.SimpleQueueService.NonExistent"
@@ -32,54 +31,63 @@
 class TestSendSQSMessage(TestCase):
     """Tests for send_sqs_message functions"""
 
+    def setUp(self):
+        self.sqs_client = boto3.client("sqs", region_name=REGION_NAME)
+        self.queue_url = self.sqs_client.create_queue(
+            QueueName=Sqs.QUEUE_NAME, Attributes=Sqs.ATTRIBUTES
+        )["QueueUrl"]
+
     def test_send_to_supplier_queue_success(self):
         """Test send_to_supplier_queue function for a successful message send"""
-        # Set up the sqs_queue
-        queue_url = sqs_client.create_queue(QueueName=Sqs.QUEUE_NAME, Attributes=Sqs.ATTRIBUTES)["QueueUrl"]
-
-        # Send three separate messages to the queue to test that they are all received and appropriately
-        # partitioned by supplier and vaccine_type
         flu_emis_1 = deepcopy(FLU_EMIS_FILE_DETAILS)
         flu_emis_2 = deepcopy(FLU_EMIS_FILE_DETAILS)
         flu_emis_2.sqs_message_body["message_id"] = "flu_emis_test_id_2"
         rsv_ravs_1 = deepcopy(RSV_RAVS_FILE_DETAILS)
 
-        for file_details in [flu_emis_1, rsv_ravs_1, flu_emis_2]:
-            self.assertIsNone(
-                send_to_supplier_queue(
-                    message_body=deepcopy(file_details.sqs_message_body),
-                    vaccine_type=file_details.vaccine_type,
-                    supplier=file_details.supplier,
+        with patch.dict("os.environ", {"QUEUE_URL": self.queue_url}):
+            with patch("send_sqs_message.sqs_client", self.sqs_client):
+                for file_details in [flu_emis_1, rsv_ravs_1, flu_emis_2]:
+                    self.assertIsNone(
+                        send_to_supplier_queue(
+                        message_body=deepcopy(file_details.sqs_message_body),
+                        vaccine_type=file_details.vaccine_type,
+                        supplier=file_details.supplier,
+                    )
                 )
-            )
 
-        # Check that the FIFO queue contains the expected messages, in correct order, and with correct MessageGroupId
-        received_messages = sqs_client.receive_message(
-            QueueUrl=queue_url, MaxNumberOfMessages=10, AttributeNames=["All"]
-        )["Messages"]
+            # Verify messages in queue
+            messages = self.sqs_client.receive_message(
+                QueueUrl=self.queue_url, MaxNumberOfMessages=10, AttributeNames=["All"]
+            )["Messages"]
 
-        self.assertEqual(len(received_messages), 3)
-        self.assertEqual(json_loads(received_messages[0]["Body"]), flu_emis_1.sqs_message_body)
-        self.assertEqual(received_messages[0]["Attributes"]["MessageGroupId"], flu_emis_1.queue_name)
-        self.assertEqual(json_loads(received_messages[1]["Body"]), rsv_ravs_1.sqs_message_body)
-        self.assertEqual(received_messages[1]["Attributes"]["MessageGroupId"], rsv_ravs_1.queue_name)
-        self.assertEqual(json_loads(received_messages[2]["Body"]), flu_emis_2.sqs_message_body)
-        self.assertEqual(received_messages[2]["Attributes"]["MessageGroupId"], flu_emis_2.queue_name)
+            self.assertEqual(len(messages), 3)
+            self.assertEqual(json_loads(messages[0]["Body"]), flu_emis_1.sqs_message_body)
+            self.assertEqual(messages[0]["Attributes"]["MessageGroupId"], flu_emis_1.queue_name)
+            self.assertEqual(json_loads(messages[1]["Body"]), rsv_ravs_1.sqs_message_body)
+            self.assertEqual(messages[1]["Attributes"]["MessageGroupId"], rsv_ravs_1.queue_name)
+            self.assertEqual(json_loads(messages[2]["Body"]), flu_emis_2.sqs_message_body)
+            self.assertEqual(messages[2]["Attributes"]["MessageGroupId"], flu_emis_2.queue_name)
 
     def test_send_to_supplier_queue_failure_due_to_queue_does_not_exist(self):
         """Test send_to_supplier_queue function for a failed message send due to queue not existing"""
-        with self.assertRaises(UnhandledSqsError) as context:
-            send_to_supplier_queue(
+
+        with patch.dict("os.environ", {"QUEUE_URL": fake_queue_url}):
+            with patch("send_sqs_message.sqs_client", self.sqs_client):
+                with self.assertRaises(UnhandledSqsError) as context:
+                 send_to_supplier_queue(
                 message_body=deepcopy(FLU_EMIS_FILE_DETAILS.sqs_message_body),
                 vaccine_type=FLU_EMIS_FILE_DETAILS.vaccine_type,
                 supplier=FLU_EMIS_FILE_DETAILS.supplier,
             )
-        self.assertEqual(NON_EXISTENT_QUEUE_ERROR_MESSAGE, str(context.exception))
+        self.assertIn("An unexpected error occurred whilst sending to SQS", str(context.exception))
+        self.assertTrue(
+            "Queue does not exist" in str(context.exception) or "NonExistentQueue" in str(context.exception)
+        )
 
     def test_send_to_supplier_queue_failure_due_to_absent_supplier_or_vaccine_type(self):
         """Test send_to_supplier_queue function for a failed message send"""
         # Set up the sqs_queue
-        sqs_client.create_queue(QueueName=Sqs.QUEUE_NAME, Attributes=Sqs.ATTRIBUTES)
+        self.sqs_client.create_queue(QueueName=Sqs.QUEUE_NAME, Attributes=Sqs.ATTRIBUTES)
         expected_error_message = (
             "Message not sent to supplier queue as unable to identify supplier and/ or vaccine type"
         )
@@ -100,33 +108,36 @@ def test_send_to_supplier_queue_failure_due_to_absent_supplier_or_vaccine_type(s
     def test_make_and_send_sqs_message_success(self):
         """Test make_and_send_sqs_message function for a successful message send"""
         # Create a mock SQS queue
-        queue_url = sqs_client.create_queue(QueueName=Sqs.QUEUE_NAME, Attributes=Sqs.ATTRIBUTES)["QueueUrl"]
+        queue_url = self.sqs_client.create_queue(QueueName=Sqs.QUEUE_NAME, Attributes=Sqs.ATTRIBUTES)["QueueUrl"]
 
         # Call the send_to_supplier_queue function
-        self.assertIsNone(
-            make_and_send_sqs_message(
-                file_key=FLU_EMIS_FILE_DETAILS.file_key,
-                message_id=FLU_EMIS_FILE_DETAILS.message_id,
-                permission=deepcopy(FLU_EMIS_FILE_DETAILS.permissions_list),
-                vaccine_type=FLU_EMIS_FILE_DETAILS.vaccine_type,
-                supplier=FLU_EMIS_FILE_DETAILS.supplier,
-                created_at_formatted_string=FLU_EMIS_FILE_DETAILS.created_at_formatted_string,
-            )
+        with patch.dict("os.environ", {"QUEUE_URL": queue_url}):
+            with patch("send_sqs_message.sqs_client", self.sqs_client):
+                self.assertIsNone(
+                    make_and_send_sqs_message(
+                        file_key=FLU_EMIS_FILE_DETAILS.file_key,
+                        message_id=FLU_EMIS_FILE_DETAILS.message_id,
+                        permission=deepcopy(FLU_EMIS_FILE_DETAILS.permissions_list),
+                        vaccine_type=FLU_EMIS_FILE_DETAILS.vaccine_type,
+                        supplier=FLU_EMIS_FILE_DETAILS.supplier,
+                        created_at_formatted_string=FLU_EMIS_FILE_DETAILS.created_at_formatted_string,
+                )
         )
-
-        # Assert that correct message has reached the queue
-        messages = sqs_client.receive_message(QueueUrl=queue_url, MaxNumberOfMessages=1)
-        self.assertEqual(json_loads(messages["Messages"][0]["Body"]), deepcopy(FLU_EMIS_FILE_DETAILS.sqs_message_body))
+            # Assert that correct message has reached the queue
+            messages = self.sqs_client.receive_message(QueueUrl=queue_url, MaxNumberOfMessages=1)
+            self.assertEqual(json_loads(messages["Messages"][0]["Body"]), deepcopy(FLU_EMIS_FILE_DETAILS.sqs_message_body))
 
     def test_make_and_send_sqs_message_failure(self):
-        """Test make_and_send_sqs_message function for a failure due to queue not existing"""
-        with self.assertRaises(UnhandledSqsError) as context:
-            make_and_send_sqs_message(
-                file_key=FLU_EMIS_FILE_DETAILS.file_key,
-                message_id=FLU_EMIS_FILE_DETAILS.message_id,
-                permission=deepcopy(FLU_EMIS_FILE_DETAILS.permissions_list),
-                vaccine_type=FLU_EMIS_FILE_DETAILS.vaccine_type,
-                supplier=FLU_EMIS_FILE_DETAILS.supplier,
-                created_at_formatted_string=FLU_EMIS_FILE_DETAILS.created_at_formatted_string,
-            )
+        
+        with patch.dict("os.environ", {"QUEUE_URL": fake_queue_url}):
+            with patch("send_sqs_message.sqs_client", boto3.client("sqs", region_name=REGION_NAME)):
+                with self.assertRaises(UnhandledSqsError) as context:
+                    make_and_send_sqs_message(
+                        file_key=FLU_EMIS_FILE_DETAILS.file_key,
+                        message_id=FLU_EMIS_FILE_DETAILS.message_id,
+                        permission=deepcopy(FLU_EMIS_FILE_DETAILS.permissions_list),
+                        vaccine_type=FLU_EMIS_FILE_DETAILS.vaccine_type,
+                        supplier=FLU_EMIS_FILE_DETAILS.supplier,
+                        created_at_formatted_string=FLU_EMIS_FILE_DETAILS.created_at_formatted_string,
+                    )
         self.assertIn(NON_EXISTENT_QUEUE_ERROR_MESSAGE, str(context.exception))