VED-492: Create ODS code to supplier mapping in Redis. (#682)

* VED-492: Create ODS code to supplier mapping in Redis.

* VED-492: Fix lint errors.

* VED-492: Add missing test. Use util function in more tests.

* VED-492: Fix lint error.

Author: mfjarvis

config/dev/permissions_config.json

@@ -6,7 +6,8 @@
       "FLU.CRUDS",
       "MMR.CRUDS",
       "RSV.CRUDS"
-    ]
+    ],
+    "ods_codes": ["DPSFULL"]
   },
   {
     "supplier": "DPSREDUCED",
@@ -15,7 +16,8 @@
       "FLU.CRUD",
       "MMR.CRUD",
       "RSV.CRUD"
-    ]
+    ],
+    "ods_codes": ["DPSREDUCED"]
   },
   {
     "supplier": "MAVIS",
@@ -25,26 +27,30 @@
       "MENACWY.CRUDS",
       "MMR.CRUDS",
       "FLU.CRUDS"
-    ]
+    ],
+    "ods_codes": ["V0V8L"]
   },
   {
     "supplier": "SONAR",
     "permissions": [
       "FLU.CD"
-    ]
+    ],
+    "ods_codes": ["8HK48"]
   },
   {
     "supplier": "EVA",
     "permissions": [
       "COVID19.CUD"
-    ]
+    ],
+    "ods_codes": ["8HA94"]
   },
   {
     "supplier": "RAVS",
     "permissions": [
       "RSV.CRUDS",
       "MMR.CRUDS"
-    ]
+    ],
+    "ods_codes": ["X8E5B"]
   },
   {
     "supplier": "AGEM-RAVS-Integration",
@@ -61,7 +67,8 @@
       "HPV.CRUDS",
       "3IN1.CRUDS",
       "MENACWY.CRUDS"
-    ]
+    ],
+    "ods_codes": ["YGM41", "YGJ"]
   },
   {
     "supplier": "TPP",
@@ -71,7 +78,8 @@
       "HPV.CRUDS",
       "3IN1.CRUDS",
       "MENACWY.CRUDS"
-    ]
+    ],
+    "ods_codes": ["YGA"]
   },
   {
     "supplier": "MEDICUS",
@@ -81,7 +89,8 @@
       "HPV.CRUDS",
       "3IN1.CRUDS",
       "MENACWY.CRUDS"
-    ]
+    ],
+    "ods_codes": ["YGMYW"]
   },
   {
     "supplier": "Test_App",

config/preprod/permissions_config.json

@@ -6,7 +6,8 @@
       "FLU.CRUDS",
       "MMR.CRUDS",
       "RSV.CRUDS"
-    ]
+    ],
+    "ods_codes": ["DPSFULL"]
   },
   {
     "supplier": "DPSREDUCED",
@@ -15,7 +16,8 @@
       "FLU.CRUDS",
       "MMR.CRUDS",
       "RSV.CRUDS"
-    ]
+    ],
+    "ods_codes": ["DPSREDUCED"]
   },
   {
     "supplier": "MAVIS",
@@ -25,14 +27,16 @@
       "MENACWY.CRUDS",
       "MMR.CRUDS",
       "FLU.CRUDS"
-    ]
+    ],
+    "ods_codes": ["V0V8L"]
   },
   {
     "supplier": "RAVS",
     "permissions": [
       "RSV.CRUDS",
       "MMR.CRUDS"
-    ]
+    ],
+    "ods_codes": ["X8E5B"]
   },
   {
     "supplier": "AGEM-RAVS-Integration",
@@ -49,7 +53,8 @@
       "HPV.CRUDS",
       "3IN1.CRUDS",
       "MENACWY.CRUDS"
-    ]
+    ],
+    "ods_codes": ["YGM41", "YGJ"]
   },
   {
     "supplier": "TPP",
@@ -59,7 +64,8 @@
       "HPV.CRUDS",
       "3IN1.CRUDS",
       "MENACWY.CRUDS"
-    ]
+    ],
+    "ods_codes": ["YGA"]
   },
   {
     "supplier": "MEDICUS",
@@ -69,7 +75,8 @@
       "HPV.CRUDS",
       "3IN1.CRUDS",
       "MENACWY.CRUDS"
-    ]
+    ],
+    "ods_codes": ["YGMYW"]
   },
   {
     "supplier": "Test_App",

config/prod/permissions_config.json

@@ -3,18 +3,21 @@
     "supplier": "DPSFULL",
     "permissions": [
       "RSV.CRUDS"
-    ]
+    ],
+    "ods_codes": ["DPSFULL"]
   },
   {
     "supplier": "DPSREDUCED",
     "permissions": [
       "RSV.CRUDS"
-    ]
+    ],
+    "ods_codes": ["DPSREDUCED"]
   },
   {
     "supplier": "RAVS",
     "permissions": [
       "RSV.RS"
-    ]
+    ],
+    "ods_codes": ["X8E5B"]
   }
 ]

filenameprocessor/src/constants.py

@@ -18,6 +18,7 @@
 
 SUPPLIER_PERMISSIONS_HASH_KEY = "supplier_permissions"
 VACCINE_TYPE_TO_DISEASES_HASH_KEY = "vacc_to_diseases"
+ODS_CODE_TO_SUPPLIER_SYSTEM_HASH_KEY = "ods_code_to_supplier"
 
 ERROR_TYPE_TO_STATUS_CODE_MAP = {
     VaccineTypePermissionsError: 403,
@@ -52,27 +53,3 @@ class AuditTableKeys:
 class Constants:
     """Constants for the filenameprocessor lambda"""
     VALID_VERSIONS = ["V5"]
-
-    # Mappings from ODS code to supplier name.
-    # NOTE: Any ODS code not found in this dictionary's keys is invalid for this service
-    ODS_TO_SUPPLIER_MAPPINGS = {
-        "YGM41": "EMIS",
-        "8J1100001": "PINNACLE",
-        "8HK48": "SONAR",
-        "YGA": "TPP",
-        "V0V8L": "MAVIS",
-        "X8E5B": "RAVS",
-        "0DE": "AGEM-NIVS",
-        "0DF": "NIMS",
-        "8HA94": "EVA",
-        "YGMYH": "MEDICAL_DIRECTOR",
-        "W00": "WELSH_DA_1",
-        "W000": "WELSH_DA_2",
-        "ZT001": "NORTHERN_IRELAND_DA",
-        "YA7": "SCOTLAND_DA",
-        "N2N9I": "COVID19_VACCINE_RESOLUTION_SERVICEDESK",
-        "YGJ": "EMIS",
-        "YGMYW": "MEDICUS",
-        "DPSREDUCED": "DPSREDUCED",
-        "DPSFULL": "DPSFULL",
-    }

filenameprocessor/src/elasticache.py

@@ -1,6 +1,10 @@
 import json
 from clients import redis_client
-from constants import VACCINE_TYPE_TO_DISEASES_HASH_KEY, SUPPLIER_PERMISSIONS_HASH_KEY
+from constants import (
+    VACCINE_TYPE_TO_DISEASES_HASH_KEY,
+    SUPPLIER_PERMISSIONS_HASH_KEY,
+    ODS_CODE_TO_SUPPLIER_SYSTEM_HASH_KEY
+)
 
 
 def get_supplier_permissions_from_cache(supplier_system: str) -> list[str]:
@@ -11,3 +15,7 @@ def get_supplier_permissions_from_cache(supplier_system: str) -> list[str]:
 
 def get_valid_vaccine_types_from_cache() -> list[str]:
     return redis_client.hkeys(VACCINE_TYPE_TO_DISEASES_HASH_KEY)
+
+
+def get_supplier_system_from_cache(ods_code: str) -> str:
+    return redis_client.hget(ODS_CODE_TO_SUPPLIER_SYSTEM_HASH_KEY, ods_code)

filenameprocessor/src/file_key_validation.py

@@ -3,8 +3,7 @@
 from re import match
 from datetime import datetime
 from constants import Constants
-from elasticache import get_valid_vaccine_types_from_cache
-from utils_for_filenameprocessor import identify_supplier
+from elasticache import get_valid_vaccine_types_from_cache, get_supplier_system_from_cache
 from errors import InvalidFileKeyError
 
 
@@ -46,7 +45,7 @@ def validate_file_key(file_key: str) -> tuple[str, str]:
     ods_code = file_key_parts_without_extension[3]
     timestamp = file_key_parts_without_extension[4]
     extension = file_key.split(".")[1]
-    supplier = identify_supplier(ods_code)
+    supplier = get_supplier_system_from_cache(ods_code)
 
     valid_vaccine_types = get_valid_vaccine_types_from_cache()
 

filenameprocessor/src/utils_for_filenameprocessor.py

@@ -1,7 +1,7 @@
 """Utils for filenameprocessor lambda"""
 
 import json
-from constants import Constants, SOURCE_BUCKET_NAME, FILE_NAME_PROC_LAMBDA_NAME
+from constants import SOURCE_BUCKET_NAME, FILE_NAME_PROC_LAMBDA_NAME
 from clients import s3_client, logger, lambda_client
 
 
@@ -11,14 +11,6 @@ def get_created_at_formatted_string(bucket_name: str, file_key: str) -> str:
     return response["LastModified"].strftime("%Y%m%dT%H%M%S00")
 
 
-def identify_supplier(ods_code: str) -> str:
-    """
-    Identifies the supplier from the ods code using the mapping.
-    Defaults to empty string if ODS code isn't found in the mappings.
-    """
-    return Constants.ODS_TO_SUPPLIER_MAPPINGS.get(ods_code, "")
-
-
 def move_file(bucket_name: str, source_file_key: str, destination_file_key: str) -> None:
     """Moves a file from one location to another within a single S3 bucket by copying and then deleting the file."""
     s3_client.copy_object(

filenameprocessor/tests/test_elasticache.py

@@ -7,10 +7,15 @@
 
 from tests.utils_for_tests.mock_environment_variables import MOCK_ENVIRONMENT_DICT
 from tests.utils_for_tests.generic_setup_and_teardown import GenericSetUp, GenericTearDown
+from tests.utils_for_tests.utils_for_filenameprocessor_tests import create_mock_hget
 
 # Ensure environment variables are mocked before importing from src files
 with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT):
-    from elasticache import get_supplier_permissions_from_cache, get_valid_vaccine_types_from_cache
+    from elasticache import (
+        get_supplier_permissions_from_cache,
+        get_valid_vaccine_types_from_cache,
+        get_supplier_system_from_cache
+    )
     from clients import REGION_NAME
 
 s3_client = boto3_client("s3", region_name=REGION_NAME)
@@ -29,13 +34,25 @@ def tearDown(self):
         """Tear down the S3 buckets"""
         GenericTearDown(s3_client)
 
-    @patch("elasticache.redis_client.hget", return_value=json.dumps(["COVID19.CRUDS", "RSV.CRUDS"]))
+    @patch("elasticache.redis_client.hget", side_effect=create_mock_hget(
+        {"TEST_ODS_CODE": "TEST_SUPPLIER"},
+        {}
+    ))
+    def test_get_supplier_system_from_cache(self, mock_hget):
+        result = get_supplier_system_from_cache("TEST_ODS_CODE")
+        self.assertEqual(result, "TEST_SUPPLIER")
+        mock_hget.assert_called_once_with("ods_code_to_supplier", "TEST_ODS_CODE")
+
+    @patch("elasticache.redis_client.hget", side_effect=create_mock_hget(
+        {},
+        {"TEST_SUPPLIER": json.dumps(["COVID19.CRUDS", "RSV.CRUDS"])}
+    ))
     def test_get_supplier_permissions_from_cache(self, mock_hget):
         result = get_supplier_permissions_from_cache("TEST_SUPPLIER")
         self.assertEqual(result, ["COVID19.CRUDS", "RSV.CRUDS"])
         mock_hget.assert_called_once_with("supplier_permissions", "TEST_SUPPLIER")
 
-    @patch("elasticache.redis_client.hget", return_value=None)
+    @patch("elasticache.redis_client.hget", side_effect=create_mock_hget({}, {}))
     def test_get_supplier_permissions_from_cache_not_found(self, mock_hget):
         result = get_supplier_permissions_from_cache("TEST_SUPPLIER")
         self.assertEqual(result, [])

filenameprocessor/tests/test_file_key_validation.py

@@ -5,6 +5,7 @@
 
 from tests.utils_for_tests.values_for_tests import MockFileDetails
 from tests.utils_for_tests.mock_environment_variables import MOCK_ENVIRONMENT_DICT
+from tests.utils_for_tests.utils_for_filenameprocessor_tests import MOCK_ODS_CODE_TO_SUPPLIER, create_mock_hget
 
 # Ensure environment variables are mocked before importing from src files
 with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT):
@@ -38,25 +39,27 @@ def test_is_valid_datetime(self):
             with self.subTest():
                 self.assertEqual(is_valid_datetime(date_time_string), expected_result)
 
+    @patch("elasticache.redis_client.hget", side_effect=create_mock_hget(MOCK_ODS_CODE_TO_SUPPLIER, {}))
     @patch("elasticache.redis_client.hkeys", return_value=["FLU", "RSV"])
-    def test_validate_file_key(self, _mock_hkeys):
+    def test_validate_file_key(self, mock_hkeys, mock_hget):
         """Tests that file_key_validation returns True if all elements pass validation, and False otherwise"""
         # Test case tuples are structured as (file_key, expected_result)
         test_cases_for_success_scenarios = [
             # Valid FLU/ EMIS file key (mixed case)
-            (VALID_FLU_EMIS_FILE_KEY, ("FLU", "EMIS")),
+            (VALID_FLU_EMIS_FILE_KEY, "YGM41", ("FLU", "EMIS")),
             # Valid FLU/ EMIS (all lowercase)
-            (VALID_FLU_EMIS_FILE_KEY.lower(), ("FLU", "EMIS")),
+            (VALID_FLU_EMIS_FILE_KEY.lower(), "YGM41", ("FLU", "EMIS")),
             # Valid FLU/ EMIS (all uppercase)
-            (VALID_FLU_EMIS_FILE_KEY.upper(), ("FLU", "EMIS")),
+            (VALID_FLU_EMIS_FILE_KEY.upper(), "YGM41", ("FLU", "EMIS")),
             # Valid RSV/ RAVS file key
-            (VALID_RSV_RAVS_FILE_KEY, ("RSV", "RAVS")),
+            (VALID_RSV_RAVS_FILE_KEY, "X8E5B", ("RSV", "RAVS")),
         ]
 
-        for file_key, expected_result in test_cases_for_success_scenarios:
+        for file_key, ods_code, expected_result in test_cases_for_success_scenarios:
             with self.subTest(f"SubTest for file key: {file_key}"):
                 self.assertEqual(validate_file_key(file_key), expected_result)
-                _mock_hkeys.assert_called_with("vacc_to_diseases")
+                mock_hkeys.assert_called_with("vacc_to_diseases")
+                mock_hget.assert_called_with("ods_code_to_supplier", ods_code)
 
         key_format_error_message = "Initial file validation failed: invalid file key format"
         invalid_file_key_error_message = "Initial file validation failed: invalid file key"
@@ -102,4 +105,4 @@ def test_validate_file_key(self, _mock_hkeys):
                 with self.assertRaises(InvalidFileKeyError) as context:
                     validate_file_key(file_key)
                 self.assertEqual(str(context.exception), expected_result)
-                _mock_hkeys.assert_called_with("vacc_to_diseases")
+                mock_hkeys.assert_called_with("vacc_to_diseases")