VED-372: Query Redis for valid vaccine types.

Author: mfjarvis

filenameprocessor/src/constants.py

@@ -17,6 +17,7 @@
 AUDIT_TABLE_FILENAME_GSI = "filename_index"
 
 PERMISSIONS_CONFIG_FILE_KEY = "permissions_config.json"
+VACCINE_TYPE_TO_DISEASES_HASH_KEY = "vacc_to_diseases"
 
 ERROR_TYPE_TO_STATUS_CODE_MAP = {
     VaccineTypePermissionsError: 403,
@@ -50,9 +51,6 @@ class AuditTableKeys:
 
 class Constants:
     """Constants for the filenameprocessor lambda"""
-
-    VALID_VACCINE_TYPES = ["FLU", "COVID19", "MMR", "RSV"]
-
     VALID_VERSIONS = ["V5"]
 
     # Mappings from ODS code to supplier name.

filenameprocessor/src/elasticache.py

@@ -2,7 +2,7 @@
 
 import json
 from clients import s3_client, redis_client
-from constants import PERMISSIONS_CONFIG_FILE_KEY
+from constants import PERMISSIONS_CONFIG_FILE_KEY, VACCINE_TYPE_TO_DISEASES_HASH_KEY
 
 
 def upload_to_elasticache(file_key: str, bucket_name: str) -> None:
@@ -16,3 +16,7 @@ def upload_to_elasticache(file_key: str, bucket_name: str) -> None:
 def get_permissions_config_json_from_cache() -> dict:
     """Gets and returns the permissions config file content from ElastiCache (Redis)."""
     return json.loads(redis_client.get(PERMISSIONS_CONFIG_FILE_KEY))
+
+
+def get_valid_vaccine_types_from_cache() -> list[str]:
+    return redis_client.hkeys(VACCINE_TYPE_TO_DISEASES_HASH_KEY)

filenameprocessor/src/file_key_validation.py

@@ -3,6 +3,7 @@
 from re import match
 from datetime import datetime
 from constants import Constants
+from elasticache import get_valid_vaccine_types_from_cache
 from utils_for_filenameprocessor import identify_supplier
 from errors import InvalidFileKeyError
 
@@ -47,9 +48,11 @@ def validate_file_key(file_key: str) -> tuple[str, str]:
     extension = file_key.split(".")[1]
     supplier = identify_supplier(ods_code)
 
+    valid_vaccine_types = get_valid_vaccine_types_from_cache()
+
     # Validate each file key element
     if not (
-        vaccine_type in Constants.VALID_VACCINE_TYPES
+        vaccine_type in valid_vaccine_types
         and vaccination == "VACCINATIONS"
         and version in Constants.VALID_VERSIONS
         and supplier  # Note that if supplier could be identified, this also implies that ODS code is valid

filenameprocessor/tests/test_file_key_validation.py

@@ -17,6 +17,10 @@
 
 class TestFileKeyValidation(TestCase):
     """Tests for file_key_validation functions"""
+    def setUp(self):
+        redis_patcher = patch("clients.redis_client.hkeys", return_value=["FLU", "COVID19", "MMR", "RSV"])
+        self.addCleanup(redis_patcher.stop)
+        redis_patcher.start()
 
     def test_is_valid_datetime(self):
         "Tests that is_valid_datetime returns True for valid datetimes, and false otherwise"

filenameprocessor/tests/test_lambda_handler.py

@@ -1,4 +1,5 @@
 """Tests for lambda_handler"""
+import json
 import sys
 from unittest.mock import patch
 from unittest import TestCase
@@ -19,6 +20,10 @@
 from tests.utils_for_tests.mock_environment_variables import MOCK_ENVIRONMENT_DICT, BucketNames, Sqs
 from tests.utils_for_tests.values_for_tests import MOCK_CREATED_AT_FORMATTED_STRING, MockFileDetails
 
+from constants import VACCINE_TYPE_TO_DISEASES_HASH_KEY
+
+from elasticache import get_valid_vaccine_types_from_cache
+
 # Ensure environment variables are mocked before importing from src files
 with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT):
     from file_name_processor import lambda_handler, handle_record
@@ -67,6 +72,7 @@ def run(self, result=None):
             patch("elasticache.redis_client", new=fakeredis.FakeStrictRedis()),
             # Patch the permissions config to allow all suppliers full permissions for all vaccine types.
             patch("elasticache.redis_client.get", return_value=all_permissions_config_content),
+            patch("elasticache.redis_client.hkeys", return_value=all_vaccine_types_in_this_test_file),
         ]
 
         with ExitStack() as stack:
@@ -457,12 +463,18 @@ class TestLambdaHandlerConfig(TestCase):
     """Tests for lambda_handler when a config file is uploaded."""
 
     config_event = {
-        "Records": [{"s3": {"bucket": {"name": BucketNames.CONFIG}, "object": {"key": (PERMISSIONS_CONFIG_FILE_KEY)}}}]
+        "Records": [{"s3": {"bucket": {"name": BucketNames.CONFIG}, "object": {"key": PERMISSIONS_CONFIG_FILE_KEY}}}]
     }
 
     def setUp(self):
         GenericSetUp(s3_client, firehose_client, sqs_client, dynamodb_client)
 
+        self.fake_redis = fakeredis.FakeStrictRedis(decode_responses=True)
+        self.fake_redis.hmset(VACCINE_TYPE_TO_DISEASES_HASH_KEY, {"RSV": "[]"})
+        redis_patcher = patch("elasticache.redis_client", new=self.fake_redis)
+        self.addCleanup(redis_patcher.stop)
+        redis_patcher.start()
+
     def tearDown(self):
         GenericTearDown(s3_client, firehose_client, sqs_client, dynamodb_client)
 
@@ -475,8 +487,10 @@ def test_elasticcache_failure_handled(self):
             }
         }
 
-        with patch("file_name_processor.upload_to_elasticache", side_effect=Exception("Upload failed")), \
-             patch("file_name_processor.logger") as mock_logger:
+        with (
+            patch("file_name_processor.upload_to_elasticache", side_effect=Exception("Upload failed")),
+            patch("file_name_processor.logger") as mock_logger
+        ):
 
             result = handle_record(event)
 
@@ -491,8 +505,6 @@ def test_elasticcache_failure_handled(self):
 
     def test_successful_processing_from_configs(self):
         """Tests that the permissions config file content is uploaded to elasticache successfully"""
-        fake_redis = fakeredis.FakeStrictRedis()
-
         ravs_rsv_file_details_1 = MockFileDetails.ravs_rsv_1
         ravs_rsv_file_details_2 = MockFileDetails.ravs_rsv_2
         s3_client.put_object(Bucket=BucketNames.SOURCE, Key=ravs_rsv_file_details_1.file_key)
@@ -509,17 +521,13 @@ def test_successful_processing_from_configs(self):
             Key=PERMISSIONS_CONFIG_FILE_KEY,
             Body=generate_permissions_config_content(ravs_rsv_permissions),
         )
-        with patch("elasticache.redis_client", new=fake_redis):
-            lambda_handler(self.config_event, None)
+        lambda_handler(self.config_event, None)
         self.assertEqual(
-            json_loads(fake_redis.get(PERMISSIONS_CONFIG_FILE_KEY)), {"all_permissions": ravs_rsv_permissions}
+            json_loads(self.fake_redis.get(PERMISSIONS_CONFIG_FILE_KEY)), {"all_permissions": ravs_rsv_permissions}
         )
 
         # Check that a RAVS RSV file processes successfully (as RAVS has permissions for RSV)
-        with (
-            patch("file_name_processor.uuid4", return_value=ravs_rsv_file_details_1.message_id),
-            patch("elasticache.redis_client", new=fake_redis),
-        ):
+        with patch("file_name_processor.uuid4", return_value=ravs_rsv_file_details_1.message_id):
             result = handle_record(record_1)
         expected_result = {
             "statusCode": 200,
@@ -537,17 +545,13 @@ def test_successful_processing_from_configs(self):
             Key=PERMISSIONS_CONFIG_FILE_KEY,
             Body=generate_permissions_config_content(ravs_no_rsv_permissions),
         )
-        with patch("elasticache.redis_client", new=fake_redis):
-            lambda_handler(self.config_event, None)
+        lambda_handler(self.config_event, None)
         self.assertEqual(
-            json_loads(fake_redis.get(PERMISSIONS_CONFIG_FILE_KEY)), {"all_permissions": ravs_no_rsv_permissions}
+            json_loads(self.fake_redis.get(PERMISSIONS_CONFIG_FILE_KEY)), {"all_permissions": ravs_no_rsv_permissions}
         )
 
         # Check that a RAVS RSV file fails to process (as RAVS now does not have permissions for RSV)
-        with (
-            patch("file_name_processor.uuid4", return_value=ravs_rsv_file_details_2.message_id),
-            patch("elasticache.redis_client", new=fake_redis),
-        ):
+        with patch("file_name_processor.uuid4", return_value=ravs_rsv_file_details_2.message_id):
             result = handle_record(record_2)
         expected_result = {
             "statusCode": 403,
@@ -572,6 +576,10 @@ class TestUnexpectedBucket(TestCase):
     def setUp(self):
         GenericSetUp(s3_client, firehose_client, sqs_client, dynamodb_client)
 
+        redis_patcher = patch("elasticache.redis_client.hkeys", return_value=all_vaccine_types_in_this_test_file)
+        self.addCleanup(redis_patcher.stop)
+        redis_patcher.start()
+
     def tearDown(self):
         GenericTearDown(s3_client, firehose_client, sqs_client, dynamodb_client)
 

filenameprocessor/tests/test_logging_decorator.py

@@ -48,6 +48,10 @@ def setUp(self):
         GenericSetUp(s3_client, firehose_client, sqs_client, dynamodb_client)
         s3_client.put_object(Bucket=BucketNames.SOURCE, Key=FILE_DETAILS.file_key)
 
+        redis_patcher = patch("clients.redis_client.hkeys", return_value=["FLU", "COVID19", "MMR", "RSV"])
+        self.addCleanup(redis_patcher.stop)
+        redis_patcher.start()
+
     def tearDown(self):
         """Clean the mock AWS environment"""
         GenericTearDown(s3_client, firehose_client, sqs_client, dynamodb_client)