VED-790-Redis-Schema-Storage

lambdas/redis_sync/src/redis_cacher.py

@@ -24,23 +24,27 @@ def upload(bucket_name: str, file_key: str) -> dict:
 
             # Transform
             redis_mappings = transform_map(config_file_content, file_key)
-
-            redis_client = get_redis_client()
-            for key, mapping in redis_mappings.items():
-                safe_mapping = {
-                    k: json.dumps(v) if isinstance(v, list) else v
-                    for k, v in mapping.items()
-                }
-                existing_mapping = redis_client.hgetall(key)
-                logger.info("Existing mapping for %s: %s", key, existing_mapping)
-                redis_client.hmset(key, safe_mapping)
-                logger.info("New mapping for %s: %s", key, safe_mapping)
-                fields_to_delete = [k for k in existing_mapping if k not in safe_mapping]
-                if fields_to_delete:
-                    redis_client.hdel(key, *fields_to_delete)
-                    logger.info("Deleted mapping fields for %s: %s", key, fields_to_delete)
-
-            return {"status": "success", "message": f"File {file_key} uploaded to Redis cache."}
+            if redis_mappings:
+                redis_client = get_redis_client()
+                for key, mapping in redis_mappings.items():
+                    safe_mapping = {
+                        k: json.dumps(v) if isinstance(v, list) else v
+                        for k, v in mapping.items()
+                    }
+                    existing_mapping = redis_client.hgetall(key)
+                    logger.info("Existing mapping for %s: %s", key, existing_mapping)
+                    redis_client.hmset(key, safe_mapping)
+                    logger.info("New mapping for %s: %s", key, safe_mapping)
+                    fields_to_delete = [k for k in existing_mapping if k not in safe_mapping]
+                    if fields_to_delete:
+                        redis_client.hdel(key, *fields_to_delete)
+                        logger.info("Deleted mapping fields for %s: %s", key, fields_to_delete)
+
+                return {"status": "success", "message": f"File {file_key} uploaded to Redis cache."}
+            else:
+                msg = f"No valid Redis mappings found for file '{file_key}'. Nothing uploaded."
+                logger.warning(msg)
+                return {"status": "warning", "message": msg}
         except Exception:
             msg = f"Error uploading file '{file_key}' to Redis cache"
             logger.exception(msg)

lambdas/redis_sync/src/transform_configs.py

@@ -46,3 +46,12 @@ def transform_supplier_permissions(mapping):
         "supplier_permissions": supplier_permissions,
         "ods_code_to_supplier": ods_code_to_supplier
     }
+
+
+def transform_generic(data, file_type) -> dict:
+    # check for generic json file
+    if file_type.lower().endswith('.json'):
+        key = f"{file_type.split('.')[0]}_json"
+        return {key.lower(): data}
+    logger.warning(f"Unrecognized file type: {file_type}.")
+    return {}  # Default case, return empty dict if no transformation is defined

lambdas/redis_sync/src/transform_map.py

@@ -1,17 +1,19 @@
 from constants import RedisCacheKey
-from transform_configs import transform_vaccine_map, transform_supplier_permissions
+from transform_configs import transform_vaccine_map, transform_supplier_permissions, transform_generic
 from common.clients import logger
 '''
 Transform config file to format required in REDIS cache.
 '''
 
 
-def transform_map(data, file_type):
+def transform_map(data, file_type) -> dict:
     # Transform the vaccine map data as needed
     logger.info("Transforming data for file type: %s", file_type)
     if file_type == RedisCacheKey.PERMISSIONS_CONFIG_FILE_KEY:
         return transform_supplier_permissions(data)
     if file_type == RedisCacheKey.DISEASE_MAPPING_FILE_KEY:
         return transform_vaccine_map(data)
+
     logger.info("No specific transformation defined for file type: %s", file_type)
-    return data  # Default case, return data as is if no transformation is defined
+
+    return transform_generic(data, file_type)

lambdas/redis_sync/tests/test_redis_cacher.py

@@ -6,21 +6,14 @@
 class TestRedisCacher(unittest.TestCase):
 
     def setUp(self):
-        # mock s3_reader and transform_map
-        self.s3_reader_patcher = patch("redis_cacher.S3Reader")
-        self.mock_s3_reader = self.s3_reader_patcher.start()
-        self.transform_map_patcher = patch("redis_cacher.transform_map")
-        self.mock_transform_map = self.transform_map_patcher.start()
-        self.redis_client_patcher = patch("common.redis_client.redis_client")
-        self.mock_redis_client = self.redis_client_patcher.start()
-        self.logger_info_patcher = patch("logging.Logger.info")
-        self.mock_logger_info = self.logger_info_patcher.start()
+        self.mock_s3_reader = patch("redis_cacher.S3Reader").start()
+        self.mock_transform_map = patch("redis_cacher.transform_map").start()
+        self.mock_redis_client = patch("common.redis_client.redis_client").start()
+        self.mock_logger_info = patch("logging.Logger.info").start()
+        self.mock_logger_warning = patch("logging.Logger.warning").start()
 
     def tearDown(self):
-        self.s3_reader_patcher.stop()
-        self.transform_map_patcher.stop()
-        self.redis_client_patcher.stop()
-        self.logger_info_patcher.stop()
+        patch.stopall()
 
     def test_upload(self):
         mock_data = {"a": "b"}
@@ -75,3 +68,21 @@ def test_deletes_extra_fields(self):
         })
         self.mock_redis_client.hdel.assert_called_once_with("hash_name", "obsolete_key_1", "obsolete_key_2")
         self.assertEqual(result, {"status": "success", "message": f"File {file_key} uploaded to Redis cache."})
+
+    def test_unrecognised_format(self):
+        mock_data = {"a": "b"}
+
+        self.mock_s3_reader.read = unittest.mock.Mock()
+        self.mock_s3_reader.read.return_value = mock_data
+        self.mock_transform_map.return_value = {}
+
+        bucket_name = "bucket"
+        file_key = "file-key.my_yaml"
+        result = RedisCacher.upload(bucket_name, file_key)
+
+        self.mock_s3_reader.read.assert_called_once_with(bucket_name, file_key)
+        self.assertEqual(result["status"], "warning")
+        self.assertEqual(result["message"], f"No valid Redis mappings found for file '{file_key}'. Nothing uploaded.")
+        self.mock_logger_warning.assert_called_once()
+        self.mock_redis_client.hmset.assert_not_called()
+        self.mock_redis_client.hdel.assert_not_called()

lambdas/redis_sync/tests/test_transform_config.py

@@ -2,13 +2,21 @@
 import unittest
 import json
 from unittest.mock import patch
-from transform_configs import transform_vaccine_map, transform_supplier_permissions
+from transform_configs import transform_vaccine_map, transform_supplier_permissions, transform_generic
 
 
-class TestTransformConfigs(unittest.TestCase):
+class TestBase(unittest.TestCase):
     def setUp(self):
-        self.logger_info_patcher = patch("transform_configs.logger.info")
-        self.mock_logger_info = self.logger_info_patcher.start()
+        self.mock_logger_info = patch("transform_configs.logger.info").start()
+        self.mock_logger_warning = patch("transform_configs.logger.warning").start()
+
+    def tearDown(self):
+        patch.stopall()
+
+
+class TestTransformConfigs(TestBase):
+    def setUp(self):
+        super().setUp()
 
         with open("./tests/test_data/disease_mapping.json") as mapping_data:
             self.sample_map = json.load(mapping_data)
@@ -17,7 +25,7 @@ def setUp(self):
             self.supplier_data = json.load(permissions_data)
 
     def tearDown(self):
-        self.logger_info_patcher.stop()
+        super().tearDown()
 
     def test_disease_to_vacc(self):
         with open("./tests/test_data/expected_disease_to_vacc.json") as f:
@@ -49,3 +57,40 @@ def test_empty_input(self):
             "supplier_permissions": {},
             "ods_code_to_supplier": {},
         })
+
+
+class TestTransformGeneric(TestBase):
+
+    def setUp(self):
+        super().setUp()
+
+    def tearDown(self):
+        super().tearDown()
+
+    def test_json_file_transformation(self):
+        data = {"name": "test"}
+        file_type = "example.json"
+        expected = {"example_json": data}
+        result = transform_generic(data, file_type)
+        self.assertEqual(result, expected)
+
+    def test_json_file_with_uppercase_extension(self):
+        data = {"key": "value"}
+        file_type = "SamPle.JSON"
+        expected = {"sample_json": data}
+        result = transform_generic(data, file_type)
+        self.assertEqual(result, expected)
+
+    def test_unrecognized_file_type_returns_empty_dict(self):
+        data = {"key1": "value1"}
+        file_type = "example.txt"
+        result = transform_generic(data, file_type)
+        self.assertEqual(result, {})
+
+    @patch('logging.getLogger')
+    def test_warning_logged_for_unrecognized_file_type(self, mock_get_logger):
+        file_type = "unsupported.csv"
+        transform_generic({}, file_type)
+        self.mock_logger_warning.assert_called_once_with(
+            f"Unrecognized file type: {file_type}."
+        )

lambdas/redis_sync/tests/test_transform_map.py

@@ -0,0 +1,42 @@
+
+import unittest
+from unittest.mock import patch
+from transform_map import transform_map
+from constants import RedisCacheKey
+
+
+class TestTransformMap(unittest.TestCase):
+    def setUp(self):
+        self.mock_logger_info = patch("transform_map.logger.info").start()
+        self.mock_logger_warning = patch("transform_map.logger.warning").start()
+        self.mock_supplier_permissions = patch("transform_map.transform_supplier_permissions",
+                                               return_value={"result": "supplier"}).start()
+        self.mock_vaccine_map = patch("transform_map.transform_vaccine_map", return_value={"result": "vaccine"}).start()
+        self.mock_generic = patch("transform_map.transform_generic", return_value={"result": "generic"}).start()
+
+    def tearDown(self):
+        patch.stopall()
+
+    def test_permissions_config_file_key_calls_supplier_permissions(self):
+        data = {"some": "data"}
+        result = transform_map(data, RedisCacheKey.PERMISSIONS_CONFIG_FILE_KEY)
+        self.mock_supplier_permissions.assert_called_once_with(data)
+        self.assertEqual(result, {"result": "supplier"})
+        self.mock_logger_info.assert_any_call(
+            "Transforming data for file type: %s", RedisCacheKey.PERMISSIONS_CONFIG_FILE_KEY)
+
+    def test_disease_mapping_file_key_calls_vaccine_map(self):
+        data = {"other": "data"}
+        result = transform_map(data, RedisCacheKey.DISEASE_MAPPING_FILE_KEY)
+        self.mock_vaccine_map.assert_called_once_with(data)
+        self.assertEqual(result, {"result": "vaccine"})
+        self.mock_logger_info.assert_any_call(
+            "Transforming data for file type: %s", RedisCacheKey.DISEASE_MAPPING_FILE_KEY)
+
+    def test_other_file_type_calls_generic(self):
+        data = {"generic": "data"}
+        file_type = "unknown_file_type"
+        result = transform_map(data, file_type)
+        self.mock_generic.assert_called_once_with(data, file_type)
+        self.assertEqual(result, {"result": "generic"})
+        self.mock_logger_info.assert_any_call("No specific transformation defined for file type: %s", file_type)