Merge pull request #338 from NHSDigital/AMB-2382_TEST_Audit_table_file_movement_file_name_processor

Amb 2382 test audit table file movement file name processor

Author: AlexandraBenson

filenameprocessor/src/audit_table.py

@@ -57,8 +57,7 @@ def upsert_audit_table(
                 Key={AuditTableKeys.MESSAGE_ID: {"S": message_id}},
                 UpdateExpression="SET #status = :status",
                 ExpressionAttributeNames={"#status": "status"},
-                # TODO: Should this be set to file_status? The status may be 'processed' due to an exception occuring
-                ExpressionAttributeValues={":status": {"S": FileStatus.PROCESSING}},
+                ExpressionAttributeValues={":status": {"S": file_status}},
                 ConditionExpression="attribute_exists(message_id)",
             )
             logger.info("%s file set for processing, and the status successfully updated in audit table", file_key)

filenameprocessor/src/file_name_processor.py

@@ -44,10 +44,14 @@ def handle_record(record) -> dict:
         logger.error("Error obtaining file_key: %s", error)
         return {"statusCode": 500, "message": "Failed to download file key", "error": str(error)}
 
-    # The lambda is unintentionally invoked when a file is moved into a different folder in the source bucket.
-    # Excluding file keys containing a "/" is a workaround to prevent the lambda from processing files that
-    # are not in the root of the source bucket.
-    if "data-sources" in bucket_name and "/" not in file_key:
+    if "data-sources" in bucket_name:
+
+        # The lambda is unintentionally invoked when a file is moved into a different folder in the source bucket.
+        # Excluding file keys containing a "/" is a workaround to prevent the lambda from processing files that
+        # are not in the root of the source bucket.
+        if "/" in file_key:
+            message = "File skipped due to duplicate lambda invoaction"
+            return {"statusCode": 200, "message": message, "file_key": file_key}
 
         # Set default values for file-specific variables
         message_id = "Message id was not created"

filenameprocessor/tests/test_audit_table.py

@@ -37,10 +37,27 @@ def test_permissions_caching(self):
         Test that upload_to_elasticache successfully uploads the file to elasticache, which is then successfully read
         by get_permissions_config_json_from_cache
         """
-        mock_permissions = {"test_supplier_1": ["RSV_FULL"], "test_supplier_2": ["FLU_CREATE", "FLU_UPDATE"]}
-        mock_permissions_config = generate_permissions_config_content(mock_permissions)
-        s3_client.put_object(Bucket=BucketNames.CONFIG, Key=PERMISSIONS_CONFIG_FILE_KEY, Body=mock_permissions_config)
+        mock_permissions_1 = {"test_supplier_1": ["RSV_FULL"], "test_supplier_2": ["FLU_CREATE", "FLU_UPDATE"]}
+        mock_permissions_config_1 = generate_permissions_config_content(mock_permissions_1)
+
+        mock_permissions_2 = {
+            "test_supplier_1": ["FLU_FULL"],
+            "test_supplier_2": ["RSV_CREATE"],
+            "test_supplier_3": ["RSV_UPDATE"],
+        }
+        mock_permissions_config_2 = generate_permissions_config_content(mock_permissions_2)
 
         with patch("elasticache.redis_client", fakeredis.FakeStrictRedis()):
+            # Test that the permissions config is successfully uploaded to elasticache
+            s3_client.put_object(
+                Bucket=BucketNames.CONFIG, Key=PERMISSIONS_CONFIG_FILE_KEY, Body=mock_permissions_config_1
+            )
+            upload_to_elasticache(PERMISSIONS_CONFIG_FILE_KEY, BucketNames.CONFIG)
+            self.assertEqual(get_permissions_config_json_from_cache(), {"all_permissions": mock_permissions_1})
+
+            # Test that the cache is updated with the new permissions config
+            s3_client.put_object(
+                Bucket=BucketNames.CONFIG, Key=PERMISSIONS_CONFIG_FILE_KEY, Body=mock_permissions_config_2
+            )
             upload_to_elasticache(PERMISSIONS_CONFIG_FILE_KEY, BucketNames.CONFIG)
-            self.assertEqual(get_permissions_config_json_from_cache(), {"all_permissions": mock_permissions})
+            self.assertEqual(get_permissions_config_json_from_cache(), {"all_permissions": mock_permissions_2})

filenameprocessor/tests/test_elasticache.py

@@ -11,8 +11,8 @@
     from file_key_validation import is_valid_datetime, validate_file_key
     from errors import InvalidFileKeyError
 
-VALID_FLU_EMIS_FILE_KEY = MockFileDetails.flu_emis.file_key
-VALID_RSV_RAVS_FILE_KEY = MockFileDetails.rsv_ravs.file_key
+VALID_FLU_EMIS_FILE_KEY = MockFileDetails.emis_flu.file_key
+VALID_RSV_RAVS_FILE_KEY = MockFileDetails.ravs_rsv_1.file_key
 
 
 class TestFileKeyValidation(TestCase):
@@ -89,9 +89,9 @@ def test_validate_file_key(self):
             # File key with missing ODS code
             (VALID_FLU_EMIS_FILE_KEY.replace("YGM41", ""), invalid_file_key_error_message),
             # File key with invalid timestamp
-            (VALID_FLU_EMIS_FILE_KEY.replace("20240708T12130100", "20200132T12345600"), invalid_file_key_error_message),
+            (VALID_FLU_EMIS_FILE_KEY.replace("20000101T00000001", "20200132T12345600"), invalid_file_key_error_message),
             # File key with missing timestamp
-            (VALID_FLU_EMIS_FILE_KEY.replace("20240708T12130100", ""), invalid_file_key_error_message),
+            (VALID_FLU_EMIS_FILE_KEY.replace("20000101T00000001", ""), invalid_file_key_error_message),
             # File key with incorrect extension
             (VALID_FLU_EMIS_FILE_KEY.replace(".csv", ".dat"), invalid_file_key_error_message),
         ]