VED-386: Handle the new format for supplier permissions.

Author: mfjarvis

filenameprocessor/src/constants.py

@@ -16,7 +16,7 @@
 AUDIT_TABLE_QUEUE_NAME_GSI = "queue_name_index"
 AUDIT_TABLE_FILENAME_GSI = "filename_index"
 
-PERMISSIONS_CONFIG_FILE_KEY = "permissions_config.json"
+SUPPLIER_PERMISSIONS_HASH_KEY = "supplier_permissions"
 VACCINE_TYPE_TO_DISEASES_HASH_KEY = "vacc_to_diseases"
 
 ERROR_TYPE_TO_STATUS_CODE_MAP = {

filenameprocessor/src/elasticache.py

@@ -1,21 +1,14 @@
 "Upload the content from a config file in S3 to ElastiCache (Redis)"
 
 import json
-from clients import s3_client, redis_client
-from constants import PERMISSIONS_CONFIG_FILE_KEY, VACCINE_TYPE_TO_DISEASES_HASH_KEY
+from clients import redis_client
+from constants import VACCINE_TYPE_TO_DISEASES_HASH_KEY, SUPPLIER_PERMISSIONS_HASH_KEY
 
 
-def upload_to_elasticache(file_key: str, bucket_name: str) -> None:
-    """Uploads the config file content from S3 to ElastiCache (Redis)."""
-    config_file = s3_client.get_object(Bucket=bucket_name, Key=file_key)
-    config_file_content = config_file["Body"].read().decode("utf-8")
-    # Use the file_key as the Redis key and file content as the value
-    redis_client.set(file_key, config_file_content)
-
-
-def get_permissions_config_json_from_cache() -> dict:
+def get_supplier_permissions_from_cache(supplier_system: str) -> list[str]:
     """Gets and returns the permissions config file content from ElastiCache (Redis)."""
-    return json.loads(redis_client.get(PERMISSIONS_CONFIG_FILE_KEY))
+    permissions_str = redis_client.hget(SUPPLIER_PERMISSIONS_HASH_KEY, supplier_system)
+    return json.loads(permissions_str) if permissions_str else []
 
 
 def get_valid_vaccine_types_from_cache() -> list[str]:

filenameprocessor/src/file_name_processor.py

@@ -14,7 +14,6 @@
 from make_and_upload_ack_file import make_and_upload_the_ack_file
 from audit_table import upsert_audit_table, get_next_queued_file_details, ensure_file_is_not_a_duplicate
 from clients import logger
-from elasticache import upload_to_elasticache
 from logging_decorator import logging_decorator
 from supplier_permissions import validate_vaccine_type_permissions
 from errors import (
@@ -140,17 +139,6 @@ def handle_record(record) -> dict:
                 "supplier": supplier
             }
 
-    elif "config" in bucket_name:
-        try:
-            upload_to_elasticache(file_key, bucket_name)
-            logger.info("%s content successfully uploaded to cache", file_key)
-            message = "File content successfully uploaded to cache"
-            return {"statusCode": 200, "message": message, "file_key": file_key}
-        except Exception as error:  # pylint: disable=broad-except
-            logger.error("Error uploading to cache for file '%s': %s", file_key, error)
-            message = "Failed to upload file content to cache"
-            return {"statusCode": 500, "message": message, "file_key": file_key, "error": str(error)}
-
     else:
         try:
             vaccine_type, supplier = validate_file_key(file_key)

filenameprocessor/src/send_sqs_message.py

@@ -27,7 +27,7 @@ def send_to_supplier_queue(message_body: dict, vaccine_type: str, supplier: str)
 
 
 def make_and_send_sqs_message(
-    file_key: str, message_id: str, permission: str, vaccine_type: str, supplier: str, created_at_formatted_string: str
+    file_key: str, message_id: str, permission: list[str], vaccine_type: str, supplier: str, created_at_formatted_string: str
 ) -> None:
     """Attempts to send a message to the SQS queue. Raises an exception if the message is not successfully sent."""
     message_body = {

filenameprocessor/src/supplier_permissions.py

@@ -2,27 +2,18 @@
 
 from clients import logger
 from errors import VaccineTypePermissionsError
-from elasticache import get_permissions_config_json_from_cache
-
-
-def get_supplier_permissions(supplier: str) -> list:
-    """
-    Returns the permissions for the given supplier.
-    Defaults return value is an empty list, including when the supplier has no permissions.
-    """
-    permissions_config = get_permissions_config_json_from_cache()
-    return permissions_config.get("all_permissions", {}).get(supplier, [])
+from elasticache import get_supplier_permissions_from_cache
 
 
 def validate_vaccine_type_permissions(vaccine_type: str, supplier: str) -> list:
     """
     Returns the list of permissions for the given supplier.
     Raises an exception if the supplier does not have at least one permission for the vaccine type.
     """
-    supplier_permissions = get_supplier_permissions(supplier)
+    supplier_permissions = get_supplier_permissions_from_cache(supplier)
 
     # Validate that supplier has at least one permissions for the vaccine type
-    if not any(vaccine_type in permission for permission in supplier_permissions):
+    if not any(permission.split(".")[0] == vaccine_type for permission in supplier_permissions):
         error_message = f"Initial file validation failed: {supplier} does not have permissions for {vaccine_type}"
         logger.error(error_message)
         raise VaccineTypePermissionsError(error_message)

filenameprocessor/tests/test_elasticache.py

@@ -1,20 +1,17 @@
 """Tests for elasticache functions"""
-
+import json
 from unittest import TestCase
 from unittest.mock import patch
-import fakeredis
 from boto3 import client as boto3_client
 from moto import mock_s3
 
-from tests.utils_for_tests.mock_environment_variables import MOCK_ENVIRONMENT_DICT, BucketNames
+from tests.utils_for_tests.mock_environment_variables import MOCK_ENVIRONMENT_DICT
 from tests.utils_for_tests.generic_setup_and_teardown import GenericSetUp, GenericTearDown
-from tests.utils_for_tests.utils_for_filenameprocessor_tests import generate_permissions_config_content
 
 # Ensure environment variables are mocked before importing from src files
 with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT):
-    from elasticache import upload_to_elasticache, get_permissions_config_json_from_cache
+    from elasticache import get_supplier_permissions_from_cache
     from clients import REGION_NAME
-    from constants import PERMISSIONS_CONFIG_FILE_KEY
 
 s3_client = boto3_client("s3", region_name=REGION_NAME)
 
@@ -32,32 +29,16 @@ def tearDown(self):
         """Tear down the S3 buckets"""
         GenericTearDown(s3_client)
 
-    def test_permissions_caching(self):
-        """
-        Test that upload_to_elasticache successfully uploads the file to elasticache, which is then successfully read
-        by get_permissions_config_json_from_cache
-        """
-        mock_permissions_1 = {"test_supplier_1": ["RSV_FULL"], "test_supplier_2": ["FLU_CREATE", "FLU_UPDATE"]}
-        mock_permissions_config_1 = generate_permissions_config_content(mock_permissions_1)
-
-        mock_permissions_2 = {
-            "test_supplier_1": ["FLU_FULL"],
-            "test_supplier_2": ["RSV_CREATE"],
-            "test_supplier_3": ["RSV_UPDATE"],
-        }
-        mock_permissions_config_2 = generate_permissions_config_content(mock_permissions_2)
-
-        with patch("elasticache.redis_client", fakeredis.FakeStrictRedis()):
-            # Test that the permissions config is successfully uploaded to elasticache
-            s3_client.put_object(
-                Bucket=BucketNames.CONFIG, Key=PERMISSIONS_CONFIG_FILE_KEY, Body=mock_permissions_config_1
-            )
-            upload_to_elasticache(PERMISSIONS_CONFIG_FILE_KEY, BucketNames.CONFIG)
-            self.assertEqual(get_permissions_config_json_from_cache(), {"all_permissions": mock_permissions_1})
-
-            # Test that the cache is updated with the new permissions config
-            s3_client.put_object(
-                Bucket=BucketNames.CONFIG, Key=PERMISSIONS_CONFIG_FILE_KEY, Body=mock_permissions_config_2
-            )
-            upload_to_elasticache(PERMISSIONS_CONFIG_FILE_KEY, BucketNames.CONFIG)
-            self.assertEqual(get_permissions_config_json_from_cache(), {"all_permissions": mock_permissions_2})
+    @patch("elasticache.redis_client")
+    def test_get_supplier_permissions_from_cache(self, mock_redis_client):
+        mock_redis_client.hget.return_value = json.dumps(["COVID19.CRUDS", "RSV.CRUDS"])
+        result = get_supplier_permissions_from_cache("TEST_SUPPLIER")
+        self.assertEqual(result, ["COVID19.CRUDS", "RSV.CRUDS"])
+        mock_redis_client.hget.assert_called_once_with("supplier_permissions", "TEST_SUPPLIER")
+
+    @patch("elasticache.redis_client")
+    def test_get_supplier_permissions_from_cache_not_found(self, mock_redis_client):
+        mock_redis_client.hget.return_value = None
+        result = get_supplier_permissions_from_cache("TEST_SUPPLIER")
+        self.assertEqual(result, [])
+        mock_redis_client.hget.assert_called_once_with("supplier_permissions", "TEST_SUPPLIER")

filenameprocessor/tests/test_lambda_handler.py

@@ -1,20 +1,17 @@
 """Tests for lambda_handler"""
+import json
 import sys
 from unittest.mock import patch
 from unittest import TestCase
 from json import loads as json_loads
 from contextlib import ExitStack
 from copy import deepcopy
-from constants import VACCINE_TYPE_TO_DISEASES_HASH_KEY
-from elasticache import get_valid_vaccine_types_from_cache
 import fakeredis
 from boto3 import client as boto3_client
 from moto import mock_s3, mock_sqs, mock_firehose, mock_dynamodb
 
 from tests.utils_for_tests.generic_setup_and_teardown import GenericSetUp, GenericTearDown
 from tests.utils_for_tests.utils_for_filenameprocessor_tests import (
-    generate_permissions_config_content,
-    generate_dict_full_permissions_all_suppliers_and_vaccine_types,
     add_entry_to_table,
     assert_audit_table_entry,
 )
@@ -25,7 +22,7 @@
 with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT):
     from file_name_processor import lambda_handler, handle_record
     from clients import REGION_NAME
-    from constants import PERMISSIONS_CONFIG_FILE_KEY, AUDIT_TABLE_NAME, FileStatus, AuditTableKeys
+    from constants import AUDIT_TABLE_NAME, FileStatus, AuditTableKeys
 
 
 s3_client = boto3_client("s3", region_name=REGION_NAME)
@@ -37,11 +34,9 @@
 # for all vaccine types. This default is overridden for some specific tests.
 all_vaccine_types_in_this_test_file = ["RSV", "FLU"]
 all_suppliers_in_this_test_file = ["RAVS", "EMIS"]
-all_permissions_config_content = generate_permissions_config_content(
-    generate_dict_full_permissions_all_suppliers_and_vaccine_types(
-        all_suppliers_in_this_test_file, all_vaccine_types_in_this_test_file
-    )
-)
+all_permissions_in_this_test_file = [
+    f"{vaccine_type}.CRUDS" for vaccine_type in all_vaccine_types_in_this_test_file
+]
 
 
 @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT)
@@ -68,7 +63,7 @@ def run(self, result=None):
             # Patch redis_client to use a fake redis client.
             patch("elasticache.redis_client", new=fakeredis.FakeStrictRedis()),
             # Patch the permissions config to allow all suppliers full permissions for all vaccine types.
-            patch("elasticache.redis_client.get", return_value=all_permissions_config_content),
+            patch("elasticache.redis_client.hget", return_value=json.dumps(all_permissions_in_this_test_file)),
             patch("elasticache.redis_client.hkeys", return_value=all_vaccine_types_in_this_test_file),
         ]
 
@@ -148,7 +143,7 @@ def assert_sqs_message(self, file_details: MockFileDetails) -> None:
         self.assertEqual(len(received_messages), 1)
         expected_sqs_message = {
             **file_details.sqs_message_body,
-            "permission": [f"{vaccine_type.upper()}_FULL" for vaccine_type in all_vaccine_types_in_this_test_file],
+            "permission": all_permissions_in_this_test_file,
         }
         self.assertEqual(json_loads(received_messages[0]["Body"]), expected_sqs_message)
 
@@ -356,10 +351,9 @@ def test_lambda_invalid_permissions_other_files_in_queue(self):
         add_entry_to_table(queued_file_details, FileStatus.QUEUED)
 
         # Mock the supplier permissions with a value which doesn't include the requested Flu permissions
-        permissions_config_content = generate_permissions_config_content({"EMIS": ["RSV_DELETE"]})
         with (  # noqa: E999
             patch("file_name_processor.uuid4", return_value=file_details.message_id),  # noqa: E999
-            patch("elasticache.redis_client.get", return_value=permissions_config_content),  # noqa: E999
+            patch("elasticache.redis_client.hget", return_value=None),  # noqa: E999
             patch("file_name_processor.invoke_filename_lambda") as mock_invoke_filename_lambda,  # noqa: E999
         ):  # noqa: E999
             lambda_handler(self.make_event([self.make_record(file_details.file_key)]), None)
@@ -451,117 +445,6 @@ def test_lambda_handler_multiple_records_for_same_queue(self):
         mock_invoke_filename_lambda.assert_not_called()
 
 
-@patch.dict("os.environ", MOCK_ENVIRONMENT_DICT)
-@mock_s3
-@mock_dynamodb
-@mock_sqs
-@mock_firehose
-class TestLambdaHandlerConfig(TestCase):
-    """Tests for lambda_handler when a config file is uploaded."""
-
-    config_event = {
-        "Records": [{"s3": {"bucket": {"name": BucketNames.CONFIG}, "object": {"key": PERMISSIONS_CONFIG_FILE_KEY}}}]
-    }
-
-    def setUp(self):
-        GenericSetUp(s3_client, firehose_client, sqs_client, dynamodb_client)
-
-        self.fake_redis = fakeredis.FakeStrictRedis(decode_responses=True)
-        self.fake_redis.hmset(VACCINE_TYPE_TO_DISEASES_HASH_KEY, {"RSV": "[]"})
-        redis_patcher = patch("elasticache.redis_client", new=self.fake_redis)
-        self.addCleanup(redis_patcher.stop)
-        redis_patcher.start()
-
-    def tearDown(self):
-        GenericTearDown(s3_client, firehose_client, sqs_client, dynamodb_client)
-
-    def test_elasticcache_failure_handled(self):
-        "Tests if elastic cache failure is handled when service fails to send message"
-        event = {
-            "s3": {
-                "bucket": {"name": "my-config-bucket"},  # triggers 'config' branch
-                "object": {"key": "testfile.csv"}
-            }
-        }
-
-        with (
-            patch("file_name_processor.upload_to_elasticache", side_effect=Exception("Upload failed")),
-            patch("file_name_processor.logger") as mock_logger
-        ):
-
-            result = handle_record(event)
-
-            self.assertEqual(result["statusCode"], 500)
-            self.assertEqual(result["message"], "Failed to upload file content to cache")
-            self.assertEqual(result["file_key"], "testfile.csv")
-            self.assertIn("error", result)
-
-            mock_logger.error.assert_called_once()
-            logged_msg = mock_logger.error.call_args[0][0]
-            self.assertIn("Error uploading to cache", logged_msg)
-
-    def test_successful_processing_from_configs(self):
-        """Tests that the permissions config file content is uploaded to elasticache successfully"""
-        ravs_rsv_file_details_1 = MockFileDetails.ravs_rsv_1
-        ravs_rsv_file_details_2 = MockFileDetails.ravs_rsv_2
-        s3_client.put_object(Bucket=BucketNames.SOURCE, Key=ravs_rsv_file_details_1.file_key)
-        s3_client.put_object(Bucket=BucketNames.SOURCE, Key=ravs_rsv_file_details_2.file_key)
-        record_1 = {"s3": {"bucket": {"name": BucketNames.SOURCE}, "object": {"key": ravs_rsv_file_details_1.file_key}}}
-        record_2 = {"s3": {"bucket": {"name": BucketNames.SOURCE}, "object": {"key": ravs_rsv_file_details_2.file_key}}}
-
-        ravs_rsv_permissions = {"RAVS": ["RSV_FULL"], "EMIS": ["FLU_CREATE", "FLU_UPDATE"]}
-        ravs_no_rsv_permissions = {"RAVS": ["FLU_FULL"], "EMIS": ["RSV_CREATE", "RSV_UPDATE"], "TPP": ["RSV_DELETE"]}
-
-        # Test that the permissions config file content is uploaded to elasticache successfully
-        s3_client.put_object(
-            Bucket=BucketNames.CONFIG,
-            Key=PERMISSIONS_CONFIG_FILE_KEY,
-            Body=generate_permissions_config_content(ravs_rsv_permissions),
-        )
-        lambda_handler(self.config_event, None)
-        self.assertEqual(
-            json_loads(self.fake_redis.get(PERMISSIONS_CONFIG_FILE_KEY)), {"all_permissions": ravs_rsv_permissions}
-        )
-
-        # Check that a RAVS RSV file processes successfully (as RAVS has permissions for RSV)
-        with patch("file_name_processor.uuid4", return_value=ravs_rsv_file_details_1.message_id):
-            result = handle_record(record_1)
-        expected_result = {
-            "statusCode": 200,
-            "message": "Successfully sent to SQS for further processing",
-            "file_key": ravs_rsv_file_details_1.file_key,
-            "message_id": ravs_rsv_file_details_1.message_id,
-            "vaccine_type": ravs_rsv_file_details_1.vaccine_type,
-            "supplier": ravs_rsv_file_details_1.supplier
-        }
-        self.assertEqual(result, expected_result)
-
-        # Test that the elasticache is successfully updated when the lambda is invoked with a new permissions config
-        s3_client.put_object(
-            Bucket=BucketNames.CONFIG,
-            Key=PERMISSIONS_CONFIG_FILE_KEY,
-            Body=generate_permissions_config_content(ravs_no_rsv_permissions),
-        )
-        lambda_handler(self.config_event, None)
-        self.assertEqual(
-            json_loads(self.fake_redis.get(PERMISSIONS_CONFIG_FILE_KEY)), {"all_permissions": ravs_no_rsv_permissions}
-        )
-
-        # Check that a RAVS RSV file fails to process (as RAVS now does not have permissions for RSV)
-        with patch("file_name_processor.uuid4", return_value=ravs_rsv_file_details_2.message_id):
-            result = handle_record(record_2)
-        expected_result = {
-            "statusCode": 403,
-            "message": "Infrastructure Level Response Value - Processing Error",
-            "file_key": ravs_rsv_file_details_2.file_key,
-            "message_id": ravs_rsv_file_details_2.message_id,
-            "error": "Initial file validation failed: RAVS does not have permissions for RSV",
-            "vaccine_type": ravs_rsv_file_details_2.vaccine_type,
-            "supplier": ravs_rsv_file_details_2.supplier
-        }
-        self.assertEqual(result, expected_result)
-
-
 @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT)
 @mock_s3
 @mock_dynamodb