VED-372: Query Redis for valid vaccine types. (#612)

* VED-372: Query Redis for valid vaccine types.

* VED-372: Tidy up.

Author: mfjarvis

filenameprocessor/src/constants.py

@@ -17,6 +17,7 @@
 AUDIT_TABLE_FILENAME_GSI = "filename_index"
 
 PERMISSIONS_CONFIG_FILE_KEY = "permissions_config.json"
+VACCINE_TYPE_TO_DISEASES_HASH_KEY = "vacc_to_diseases"
 
 ERROR_TYPE_TO_STATUS_CODE_MAP = {
     VaccineTypePermissionsError: 403,
@@ -50,9 +51,6 @@ class AuditTableKeys:
 
 class Constants:
     """Constants for the filenameprocessor lambda"""
-
-    VALID_VACCINE_TYPES = ["FLU", "COVID19", "MMR", "RSV"]
-
     VALID_VERSIONS = ["V5"]
 
     # Mappings from ODS code to supplier name.

filenameprocessor/src/elasticache.py

@@ -2,7 +2,7 @@
 
 import json
 from clients import s3_client, redis_client
-from constants import PERMISSIONS_CONFIG_FILE_KEY
+from constants import PERMISSIONS_CONFIG_FILE_KEY, VACCINE_TYPE_TO_DISEASES_HASH_KEY
 
 
 def upload_to_elasticache(file_key: str, bucket_name: str) -> None:
@@ -16,3 +16,7 @@ def upload_to_elasticache(file_key: str, bucket_name: str) -> None:
 def get_permissions_config_json_from_cache() -> dict:
     """Gets and returns the permissions config file content from ElastiCache (Redis)."""
     return json.loads(redis_client.get(PERMISSIONS_CONFIG_FILE_KEY))
+
+
+def get_valid_vaccine_types_from_cache() -> list[str]:
+    return redis_client.hkeys(VACCINE_TYPE_TO_DISEASES_HASH_KEY)

filenameprocessor/src/file_key_validation.py

@@ -3,6 +3,7 @@
 from re import match
 from datetime import datetime
 from constants import Constants
+from elasticache import get_valid_vaccine_types_from_cache
 from utils_for_filenameprocessor import identify_supplier
 from errors import InvalidFileKeyError
 
@@ -47,9 +48,11 @@ def validate_file_key(file_key: str) -> tuple[str, str]:
     extension = file_key.split(".")[1]
     supplier = identify_supplier(ods_code)
 
+    valid_vaccine_types = get_valid_vaccine_types_from_cache()
+
     # Validate each file key element
     if not (
-        vaccine_type in Constants.VALID_VACCINE_TYPES
+        vaccine_type in valid_vaccine_types
         and vaccination == "VACCINATIONS"
         and version in Constants.VALID_VERSIONS
         and supplier  # Note that if supplier could be identified, this also implies that ODS code is valid

filenameprocessor/tests/test_file_key_validation.py

@@ -17,7 +17,6 @@
 
 class TestFileKeyValidation(TestCase):
     """Tests for file_key_validation functions"""
-
     def test_is_valid_datetime(self):
         "Tests that is_valid_datetime returns True for valid datetimes, and false otherwise"
         # Test case tuples are stuctured as (date_time_string, expected_result)
@@ -39,7 +38,8 @@ def test_is_valid_datetime(self):
             with self.subTest():
                 self.assertEqual(is_valid_datetime(date_time_string), expected_result)
 
-    def test_validate_file_key(self):
+    @patch("elasticache.redis_client.hkeys", return_value=["FLU", "RSV"])
+    def test_validate_file_key(self, _mock_hkeys):
         """Tests that file_key_validation returns True if all elements pass validation, and False otherwise"""
         # Test case tuples are structured as (file_key, expected_result)
         test_cases_for_success_scenarios = [
@@ -56,6 +56,7 @@ def test_validate_file_key(self):
         for file_key, expected_result in test_cases_for_success_scenarios:
             with self.subTest(f"SubTest for file key: {file_key}"):
                 self.assertEqual(validate_file_key(file_key), expected_result)
+                _mock_hkeys.assert_called_with("vacc_to_diseases")
 
         key_format_error_message = "Initial file validation failed: invalid file key format"
         invalid_file_key_error_message = "Initial file validation failed: invalid file key"
@@ -101,3 +102,4 @@ def test_validate_file_key(self):
                 with self.assertRaises(InvalidFileKeyError) as context:
                     validate_file_key(file_key)
                 self.assertEqual(str(context.exception), expected_result)
+                _mock_hkeys.assert_called_with("vacc_to_diseases")

filenameprocessor/tests/test_lambda_handler.py

@@ -5,6 +5,8 @@
 from json import loads as json_loads
 from contextlib import ExitStack
 from copy import deepcopy
+from constants import VACCINE_TYPE_TO_DISEASES_HASH_KEY
+from elasticache import get_valid_vaccine_types_from_cache
 import fakeredis
 from boto3 import client as boto3_client
 from moto import mock_s3, mock_sqs, mock_firehose, mock_dynamodb
@@ -67,6 +69,7 @@ def run(self, result=None):
             patch("elasticache.redis_client", new=fakeredis.FakeStrictRedis()),
             # Patch the permissions config to allow all suppliers full permissions for all vaccine types.
             patch("elasticache.redis_client.get", return_value=all_permissions_config_content),
+            patch("elasticache.redis_client.hkeys", return_value=all_vaccine_types_in_this_test_file),
         ]
 
         with ExitStack() as stack:
@@ -457,12 +460,18 @@ class TestLambdaHandlerConfig(TestCase):
     """Tests for lambda_handler when a config file is uploaded."""
 
     config_event = {
-        "Records": [{"s3": {"bucket": {"name": BucketNames.CONFIG}, "object": {"key": (PERMISSIONS_CONFIG_FILE_KEY)}}}]
+        "Records": [{"s3": {"bucket": {"name": BucketNames.CONFIG}, "object": {"key": PERMISSIONS_CONFIG_FILE_KEY}}}]
     }
 
     def setUp(self):
         GenericSetUp(s3_client, firehose_client, sqs_client, dynamodb_client)
 
+        self.fake_redis = fakeredis.FakeStrictRedis(decode_responses=True)
+        self.fake_redis.hmset(VACCINE_TYPE_TO_DISEASES_HASH_KEY, {"RSV": "[]"})
+        redis_patcher = patch("elasticache.redis_client", new=self.fake_redis)
+        self.addCleanup(redis_patcher.stop)
+        redis_patcher.start()
+
     def tearDown(self):
         GenericTearDown(s3_client, firehose_client, sqs_client, dynamodb_client)
 
@@ -475,8 +484,10 @@ def test_elasticcache_failure_handled(self):
             }
         }
 
-        with patch("file_name_processor.upload_to_elasticache", side_effect=Exception("Upload failed")), \
-             patch("file_name_processor.logger") as mock_logger:
+        with (
+            patch("file_name_processor.upload_to_elasticache", side_effect=Exception("Upload failed")),
+            patch("file_name_processor.logger") as mock_logger
+        ):
 
             result = handle_record(event)
 
@@ -491,8 +502,6 @@ def test_elasticcache_failure_handled(self):
 
     def test_successful_processing_from_configs(self):
         """Tests that the permissions config file content is uploaded to elasticache successfully"""
-        fake_redis = fakeredis.FakeStrictRedis()
-
         ravs_rsv_file_details_1 = MockFileDetails.ravs_rsv_1
         ravs_rsv_file_details_2 = MockFileDetails.ravs_rsv_2
         s3_client.put_object(Bucket=BucketNames.SOURCE, Key=ravs_rsv_file_details_1.file_key)
@@ -509,17 +518,13 @@ def test_successful_processing_from_configs(self):
             Key=PERMISSIONS_CONFIG_FILE_KEY,
             Body=generate_permissions_config_content(ravs_rsv_permissions),
         )
-        with patch("elasticache.redis_client", new=fake_redis):
-            lambda_handler(self.config_event, None)
+        lambda_handler(self.config_event, None)
         self.assertEqual(
-            json_loads(fake_redis.get(PERMISSIONS_CONFIG_FILE_KEY)), {"all_permissions": ravs_rsv_permissions}
+            json_loads(self.fake_redis.get(PERMISSIONS_CONFIG_FILE_KEY)), {"all_permissions": ravs_rsv_permissions}
         )
 
         # Check that a RAVS RSV file processes successfully (as RAVS has permissions for RSV)
-        with (
-            patch("file_name_processor.uuid4", return_value=ravs_rsv_file_details_1.message_id),
-            patch("elasticache.redis_client", new=fake_redis),
-        ):
+        with patch("file_name_processor.uuid4", return_value=ravs_rsv_file_details_1.message_id):
             result = handle_record(record_1)
         expected_result = {
             "statusCode": 200,
@@ -537,17 +542,13 @@ def test_successful_processing_from_configs(self):
             Key=PERMISSIONS_CONFIG_FILE_KEY,
             Body=generate_permissions_config_content(ravs_no_rsv_permissions),
         )
-        with patch("elasticache.redis_client", new=fake_redis):
-            lambda_handler(self.config_event, None)
+        lambda_handler(self.config_event, None)
         self.assertEqual(
-            json_loads(fake_redis.get(PERMISSIONS_CONFIG_FILE_KEY)), {"all_permissions": ravs_no_rsv_permissions}
+            json_loads(self.fake_redis.get(PERMISSIONS_CONFIG_FILE_KEY)), {"all_permissions": ravs_no_rsv_permissions}
         )
 
         # Check that a RAVS RSV file fails to process (as RAVS now does not have permissions for RSV)
-        with (
-            patch("file_name_processor.uuid4", return_value=ravs_rsv_file_details_2.message_id),
-            patch("elasticache.redis_client", new=fake_redis),
-        ):
+        with patch("file_name_processor.uuid4", return_value=ravs_rsv_file_details_2.message_id):
             result = handle_record(record_2)
         expected_result = {
             "statusCode": 403,
@@ -572,6 +573,10 @@ class TestUnexpectedBucket(TestCase):
     def setUp(self):
         GenericSetUp(s3_client, firehose_client, sqs_client, dynamodb_client)
 
+        redis_patcher = patch("elasticache.redis_client.hkeys", return_value=all_vaccine_types_in_this_test_file)
+        self.addCleanup(redis_patcher.stop)
+        redis_patcher.start()
+
     def tearDown(self):
         GenericTearDown(s3_client, firehose_client, sqs_client, dynamodb_client)
 

filenameprocessor/tests/test_logging_decorator.py

@@ -73,6 +73,7 @@ def run(self, result=None):
             # Time is incremented by 1.0 for each call to time.time for ease of testing.
             # Range is set to a large number (100) due to many calls being made to time.time for some tests.
             patch("logging_decorator.time.time", side_effect=[0.0 + i for i in range(100)]),
+            patch("clients.redis_client.hkeys", return_value=["FLU"])
         ]
 
         # Set up the ExitStack. Note that patches need to be explicitly started so that they will be applied even when