refactor for dps

Author: Akol125

infrastructure/instance/file_name_processor.tf

@@ -290,7 +290,8 @@ resource "aws_lambda_function" "file_processor_lambda" {
 
   environment {
     variables = {
-      ACCOUNT_ID           = var.dspp_core_account_id
+      ACCOUNT_ID           = var.immunisation_account_id
+      DPS_ACCOUNT_ID       = var.dspp_core_account_id
       SOURCE_BUCKET_NAME   = aws_s3_bucket.batch_data_source_bucket.bucket
       ACK_BUCKET_NAME      = aws_s3_bucket.batch_data_destination_bucket.bucket
       DPS_BUCKET_NAME      = "nhsd-dspp-core-ref-extended-attributes-gdp"

lambdas/filenameprocessor/src/constants.py

@@ -14,7 +14,8 @@
 
 # We have used an internal temporary bucket here and an actual dps bucket will replace this
 DPS_DESTINATION_BUCKET_NAME = os.getenv("DPS_BUCKET_NAME")
-EXPECTED_BUCKET_OWNER_ACCOUNT = os.getenv("ACCOUNT_ID")
+EXPECTED_SOURCE_BUCKET_ACCOUNT = os.getenv("ACCOUNT_ID")
+EXPECTED_DPS_DESTINATION_ACCOUNT = os.getenv("DPS_ACCOUNT_ID")
 AUDIT_TABLE_NAME = os.getenv("AUDIT_TABLE_NAME")
 AUDIT_TABLE_TTL_DAYS = os.getenv("AUDIT_TABLE_TTL_DAYS")
 VALID_VERSIONS = ["V5"]

lambdas/filenameprocessor/src/file_name_processor.py

@@ -21,7 +21,8 @@
     DPS_DESTINATION_BUCKET_NAME,
     DPS_DESTINATION_PREFIX,
     ERROR_TYPE_TO_STATUS_CODE_MAP,
-    EXPECTED_BUCKET_OWNER_ACCOUNT,
+    EXPECTED_DPS_DESTINATION_ACCOUNT,
+    EXPECTED_SOURCE_BUCKET_ACCOUNT,
     EXTENDED_ATTRIBUTES_ARCHIVE_PREFIX,
     EXTENDED_ATTRIBUTES_FILE_PREFIX,
     SOURCE_BUCKET_NAME,
@@ -270,8 +271,8 @@ def handle_extended_attributes_file(
             file_key,
             DPS_DESTINATION_BUCKET_NAME,
             dest_file_key,
-            EXPECTED_BUCKET_OWNER_ACCOUNT,
-            EXPECTED_BUCKET_OWNER_ACCOUNT,
+            EXPECTED_DPS_DESTINATION_ACCOUNT,
+            EXPECTED_SOURCE_BUCKET_ACCOUNT,
         )
 
         move_file(bucket_name, file_key, f"{EXTENDED_ATTRIBUTES_ARCHIVE_PREFIX}/{file_key}")

lambdas/filenameprocessor/tests/test_lambda_handler.py

@@ -270,6 +270,12 @@ def test_lambda_handler_extended_attributes_success(self, mock_get_redis_client)
         mock_redis.hkeys = Mock(return_value=["COVID", *all_vaccine_types_in_this_test_file])
         mock_get_redis_client.return_value = mock_redis
 
+        # Ensure DPS destination bucket exists in moto so copy succeeds
+        try:
+            s3_client.create_bucket(Bucket=BucketNames.DPS_DESTINATION)
+        except Exception:
+            pass
+
         # Patch uuid4 (message id), and prevent external copy issues by simulating move
         with (
             patch("file_name_processor.validate_permissions_for_extended_attributes_files", return_value="X8E5B_COVID"),
@@ -288,10 +294,15 @@ def test_lambda_handler_extended_attributes_success(self, mock_get_redis_client)
         self.assertEqual(item[AuditTableKeys.STATUS]["S"], "Processed")
         self.assertEqual(item[AuditTableKeys.TIMESTAMP]["S"], test_cases[0].created_at_formatted_string)
         self.assertEqual(item[AuditTableKeys.EXPIRES_AT]["N"], str(test_cases[0].expires_at))
-        # File should be moved to destination/
-        dest_key = f"dps_destination/{test_cases[0].file_key}"
-        retrieved = s3_client.get_object(Bucket=BucketNames.DESTINATION, Key=dest_key)
-        self.assertIsNotNone(retrieved)
+        # File should be moved to source extended-attributes-archive/<file_key>
+        archived_key = f"extended-attributes-archive/{test_cases[0].file_key}"
+        archived_obj = s3_client.get_object(Bucket=BucketNames.SOURCE, Key=archived_key)
+        self.assertIsNotNone(archived_obj)
+
+        # Also verify file copied to DPS destination bucket under dps_destination/<file_key>
+        dps_key = f"dps_destination/{test_cases[0].file_key}"
+        copied_obj = s3_client.get_object(Bucket=BucketNames.DPS_DESTINATION, Key=dps_key)
+        self.assertIsNotNone(copied_obj)
 
         # No SQS and no ack file
         self.assert_no_sqs_message()
@@ -443,6 +454,12 @@ def test_lambda_handler_extended_attributes_extension_checks(self, mock_get_redi
         # .CSV accepted
         csv_key = MockFileDetails.extended_attributes_file.file_key
         s3_client.put_object(Bucket=BucketNames.SOURCE, Key=csv_key, Body=MOCK_EXTENDED_ATTRIBUTES_FILE_CONTENT)
+        # Ensure DPS destination bucket exists so copy can succeed
+        try:
+            s3_client.create_bucket(Bucket=BucketNames.DPS_DESTINATION)
+        except Exception:
+            pass
+
         with (
             patch("file_name_processor.validate_permissions_for_extended_attributes_files", return_value="X8E5B_COVID"),
             # Ensure EA DAT case passes permission validation by returning CUDS for X8E5B
@@ -453,8 +470,11 @@ def test_lambda_handler_extended_attributes_extension_checks(self, mock_get_redi
             patch("file_name_processor.uuid4", return_value="EA_csv_id"),
         ):
             lambda_handler(self.make_event([self.make_record(csv_key)]), None)
-        # Ensure processed path hit by checking destination (implementation currently uses single slash)
-        s3_client.get_object(Bucket=BucketNames.DESTINATION, Key=f"dps_destination/{csv_key}")
+
+        # Ensure processed path hit by checking archive move in source bucket
+        s3_client.get_object(Bucket=BucketNames.SOURCE, Key=f"extended-attributes-archive/{csv_key}")
+        # And verify copy to DPS destination
+        s3_client.get_object(Bucket=BucketNames.DPS_DESTINATION, Key=f"dps_destination/{csv_key}")
 
         # .DAT accepted
         dat_key = MockFileDetails.extended_attributes_file.file_key[:-3] + "dat"
@@ -464,7 +484,8 @@ def test_lambda_handler_extended_attributes_extension_checks(self, mock_get_redi
             patch("file_name_processor.uuid4", return_value="EA_dat_id"),
         ):
             lambda_handler(self.make_event([self.make_record(dat_key)]), None)
-        s3_client.get_object(Bucket=BucketNames.DESTINATION, Key=f"dps_destination/{dat_key}")
+        s3_client.get_object(Bucket=BucketNames.SOURCE, Key=f"extended-attributes-archive/{dat_key}")
+        s3_client.get_object(Bucket=BucketNames.DPS_DESTINATION, Key=f"dps_destination/{dat_key}")
 
         # Invalid extension fails
         bad_ext_key = csv_key[:-3] + "txt"

lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py

@@ -12,6 +12,7 @@ class BucketNames:
     CONFIG = "immunisation-batch-internal-dev-data-configs"
     SOURCE = "immunisation-batch-internal-dev-data-sources"
     DESTINATION = "immunisation-batch-internal-dev-data-destinations"
+    DPS_DESTINATION = "nhsd-dspp-core-ref-extended-attributes-gdp"
     # Mock firehose bucket used for testing only (due to limitations of the moto testing package)
     MOCK_FIREHOSE = "mock-firehose-bucket"
 
@@ -36,7 +37,9 @@ class Sqs:
 MOCK_ENVIRONMENT_DICT = {
     "SOURCE_BUCKET_NAME": BucketNames.SOURCE,
     "ACK_BUCKET_NAME": BucketNames.DESTINATION,
+    "DPS_BUCKET_NAME": BucketNames.DESTINATION,
     "ACCOUNT_ID": MOCK_ACCOUNT_ID,
+    "DPS_ACCOUNT_ID": MOCK_ACCOUNT_ID,
     "QUEUE_URL": "https://sqs.eu-west-2.amazonaws.com/123456789012/imms-batch-file-created-queue.fifo",
     "REDIS_HOST": "localhost",
     "REDIS_PORT": "6379",

lambdas/filenameprocessor/tests/utils_for_tests/utils_for_filenameprocessor_tests.py

@@ -91,6 +91,7 @@ def __init__(
             for bucket_name in [
                 BucketNames.SOURCE,
                 BucketNames.DESTINATION,
+                BucketNames.DPS_DESTINATION,
                 BucketNames.CONFIG,
                 BucketNames.MOCK_FIREHOSE,
             ]:
@@ -136,6 +137,7 @@ def __init__(
             for bucket_name in [
                 BucketNames.SOURCE,
                 BucketNames.DESTINATION,
+                BucketNames.DPS_DESTINATION,
                 BucketNames.CONFIG,
                 BucketNames.MOCK_FIREHOSE,
             ]: