change env vars and use moto for test

Author: Akol125

infrastructure/instance/file_name_processor.tf

@@ -162,6 +162,19 @@ resource "aws_iam_policy" "filenameprocessor_lambda_exec_policy" {
           "firehose:PutRecordBatch"
         ],
         "Resource" : "arn:aws:firehose:*:*:deliverystream/${module.splunk.firehose_stream_name}"
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "s3:GetObject",
+          "s3:PutObject",
+          "s3:DeleteObject",
+          "s3:ListBucket"
+        ],
+        "Resource" : [
+          "arn:aws:s3:::nhsd-dspp-core-ref-extended-attributes-gdp",
+          "arn:aws:s3:::nhsd-dspp-core-ref-extended-attributes-gdp/*"
+        ]
       }
     ]
   })
@@ -277,9 +290,10 @@ resource "aws_lambda_function" "file_processor_lambda" {
 
   environment {
     variables = {
-      ACCOUNT_ID           = var.immunisation_account_id
+      ACCOUNT_ID           = var.dspp_core_account_id
       SOURCE_BUCKET_NAME   = aws_s3_bucket.batch_data_source_bucket.bucket
       ACK_BUCKET_NAME      = aws_s3_bucket.batch_data_destination_bucket.bucket
+      DPS_BUCKET_NAME      = "nhsd-dspp-core-ref-extended-attributes-gdp"
       QUEUE_URL            = aws_sqs_queue.batch_file_created.url
       REDIS_HOST           = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address
       REDIS_PORT           = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].port

lambdas/filenameprocessor/src/constants.py

@@ -13,7 +13,7 @@
 SOURCE_BUCKET_NAME = os.getenv("SOURCE_BUCKET_NAME")
 
 # We have used an internal temporary bucket here and an actual dps bucket will replace this
-DPS_DESTINATION_BUCKET_NAME = os.getenv("ACK_BUCKET_NAME")
+DPS_DESTINATION_BUCKET_NAME = os.getenv("DPS_BUCKET_NAME")
 EXPECTED_BUCKET_OWNER_ACCOUNT = os.getenv("ACCOUNT_ID")
 AUDIT_TABLE_NAME = os.getenv("AUDIT_TABLE_NAME")
 AUDIT_TABLE_TTL_DAYS = os.getenv("AUDIT_TABLE_TTL_DAYS")
@@ -22,7 +22,10 @@
 VACCINE_TYPE_TO_DISEASES_HASH_KEY = "vacc_to_diseases"
 ODS_CODE_TO_SUPPLIER_SYSTEM_HASH_KEY = "ods_code_to_supplier"
 EXTENDED_ATTRIBUTES_FILE_PREFIX = "Vaccination_Extended_Attributes"
+
+# Currently only COVID extended attributes files are supported, might be extended in future for other vaccine types
 EXTENDED_ATTRIBUTES_VACC_TYPE = "COVID"
+
 DPS_DESTINATION_PREFIX = "dps_destination"
 EXTENDED_ATTRIBUTES_ARCHIVE_PREFIX = "extended-attributes-archive"
 VALID_EA_VERSIONS = ["V1_5"]

lambdas/filenameprocessor/tests/test_lambda_handler.py

@@ -274,26 +274,6 @@ def test_lambda_handler_extended_attributes_success(self, mock_get_redis_client)
         with (
             patch("file_name_processor.validate_permissions_for_extended_attributes_files", return_value="X8E5B_COVID"),
             patch("file_name_processor.uuid4", return_value=test_cases[0].message_id),
-            patch(
-                "file_name_processor.copy_file_to_external_bucket",
-                side_effect=lambda src_bucket, key, dst_bucket, dst_key, exp_owner, exp_src_owner: (
-                    s3_client.put_object(
-                        Bucket=BucketNames.DESTINATION,
-                        Key=dst_key,
-                        Body=s3_client.get_object(Bucket=src_bucket, Key=key)["Body"].read(),
-                    ),
-                ),
-            ),
-            patch(
-                "file_name_processor.move_file",
-                side_effect=lambda source_bucket, source_key, destination_key: (
-                    s3_client.put_object(
-                        Bucket=source_bucket,
-                        Key=destination_key,
-                        Body=s3_client.get_object(Bucket=source_bucket, Key=source_key)["Body"].read(),
-                    ),
-                ),
-            ),
         ):
             lambda_handler(self.make_event([self.make_record(test_cases[0].file_key)]), None)
 
@@ -310,7 +290,6 @@ def test_lambda_handler_extended_attributes_success(self, mock_get_redis_client)
         self.assertEqual(item[AuditTableKeys.EXPIRES_AT]["N"], str(test_cases[0].expires_at))
         # File should be moved to destination/
         dest_key = f"dps_destination/{test_cases[0].file_key}"
-        print(f" destination file is at {s3_client.list_objects(Bucket=BucketNames.DESTINATION)}")
         retrieved = s3_client.get_object(Bucket=BucketNames.DESTINATION, Key=dest_key)
         self.assertIsNotNone(retrieved)
 
@@ -352,17 +331,6 @@ def test_lambda_handler_extended_attributes_failure(self, mock_get_redis_client)
             patch("file_name_processor.validate_permissions_for_extended_attributes_files", return_value="X8E5B_COVID"),
             patch("file_name_processor.uuid4", return_value=test_cases[0].message_id),
             patch("file_name_processor.copy_file_to_external_bucket", side_effect=Exception("Test ClientError")),
-            patch(
-                "file_name_processor.move_file",
-                side_effect=lambda bucket, key, dst_key: (
-                    s3_client.put_object(
-                        Bucket=bucket,
-                        Key=dst_key,
-                        Body=s3_client.get_object(Bucket=bucket, Key=key)["Body"].read(),
-                    ),
-                    s3_client.delete_object(Bucket=bucket, Key=key),
-                ),
-            ),
         ):
             lambda_handler(self.make_event([self.make_record(test_cases[0].file_key)]), None)
 
@@ -384,7 +352,6 @@ def test_lambda_handler_extended_attributes_failure(self, mock_get_redis_client)
         self.assertEqual(item[AuditTableKeys.EXPIRES_AT]["N"], str(test_cases[0].expires_at))
         # File should be moved to source under archive/
         dest_key = f"extended-attributes-archive/{test_cases[0].file_key}"
-        print(f" destination file is at {s3_client.list_objects(Bucket=BucketNames.SOURCE)}")
         retrieved = s3_client.get_object(Bucket=BucketNames.SOURCE, Key=dest_key)
         self.assertIsNotNone(retrieved)
 
@@ -484,27 +451,6 @@ def test_lambda_handler_extended_attributes_extension_checks(self, mock_get_redi
                 return_value=["COVID.CUDS"],
             ),
             patch("file_name_processor.uuid4", return_value="EA_csv_id"),
-            patch(
-                "file_name_processor.copy_file_to_external_bucket",
-                side_effect=lambda src_bucket, key, dst_bucket, dst_key, exp_owner, exp_src_owner: (
-                    s3_client.put_object(
-                        Bucket=BucketNames.DESTINATION,
-                        Key=dst_key,
-                        Body=s3_client.get_object(Bucket=src_bucket, Key=key)["Body"].read(),
-                    ),
-                ),
-            ),
-            patch(
-                "file_name_processor.move_file",
-                side_effect=lambda bucket, key, dst_key: (
-                    s3_client.put_object(
-                        Bucket=bucket,
-                        Key=dst_key,
-                        Body=s3_client.get_object(Bucket=bucket, Key=key)["Body"].read(),
-                    ),
-                    s3_client.delete_object(Bucket=bucket, Key=key),
-                ),
-            ),
         ):
             lambda_handler(self.make_event([self.make_record(csv_key)]), None)
         # Ensure processed path hit by checking destination (implementation currently uses single slash)
@@ -516,27 +462,6 @@ def test_lambda_handler_extended_attributes_extension_checks(self, mock_get_redi
         with (
             patch("file_name_processor.validate_permissions_for_extended_attributes_files", return_value="X8E5B_COVID"),
             patch("file_name_processor.uuid4", return_value="EA_dat_id"),
-            patch(
-                "file_name_processor.copy_file_to_external_bucket",
-                side_effect=lambda src_bucket, key, dst_bucket, dst_key, exp_owner, exp_src_owner: (
-                    s3_client.put_object(
-                        Bucket=BucketNames.DESTINATION,
-                        Key=dst_key,
-                        Body=s3_client.get_object(Bucket=src_bucket, Key=key)["Body"].read(),
-                    ),
-                ),
-            ),
-            patch(
-                "file_name_processor.move_file",
-                side_effect=lambda bucket, key, dst_key: (
-                    s3_client.put_object(
-                        Bucket=bucket,
-                        Key=dst_key,
-                        Body=s3_client.get_object(Bucket=bucket, Key=key)["Body"].read(),
-                    ),
-                    s3_client.delete_object(Bucket=bucket, Key=key),
-                ),
-            ),
         ):
             lambda_handler(self.make_event([self.make_record(dat_key)]), None)
         s3_client.get_object(Bucket=BucketNames.DESTINATION, Key=f"dps_destination/{dat_key}")
@@ -569,10 +494,6 @@ def test_lambda_handler_extended_attributes_extension_checks(self, mock_get_redi
             Key=invalid_file_key,
             Body=MOCK_EXTENDED_ATTRIBUTES_FILE_CONTENT,
         )
-
-        # TODO: rewrite the bucket patches to use moto
-
-        # Patch uuid4 (message id), and don't move the file
         with (
             patch("file_name_processor.uuid4", return_value=test_cases[0].message_id),
             patch(

lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py

@@ -36,6 +36,7 @@ class Sqs:
 MOCK_ENVIRONMENT_DICT = {
     "SOURCE_BUCKET_NAME": BucketNames.SOURCE,
     "ACK_BUCKET_NAME": BucketNames.DESTINATION,
+    "ACCOUNT_ID": MOCK_ACCOUNT_ID,
     "QUEUE_URL": "https://sqs.eu-west-2.amazonaws.com/123456789012/imms-batch-file-created-queue.fifo",
     "REDIS_HOST": "localhost",
     "REDIS_PORT": "6379",