VED-470: Add missing permission. Move processed files to an archive prefix.

Author: mfjarvis

mesh_processor/src/converter.py

@@ -133,6 +133,10 @@ def process_record(record: dict) -> None:
 
     logger.info(f"Transfer complete for {file_key}")
 
+    move_file(bucket_name, file_key, bucket_name, f"archive/{file_key}")
+
+    logger.info(f"Archived {file_key}")
+
 
 def lambda_handler(event: dict, _context: dict) -> dict:
     success = True

mesh_processor/tests/test_converter.py

@@ -48,10 +48,17 @@ def test_non_multipart_content_type(self):
         result = invoke_lambda("test-csv-file.csv")
         self.assertEqual(result["statusCode"], 200)
 
-        response = s3.get_object(Bucket="destination-bucket", Key="overridden-filename.csv")
-        body = response["Body"].read().decode("utf-8")
+        get_target_response = s3.get_object(Bucket="destination-bucket", Key="overridden-filename.csv")
+        body = get_target_response["Body"].read().decode("utf-8")
         assert body == "some CSV content"
 
+        with self.assertRaises(ClientError) as e:
+            s3.head_object(Bucket="source-bucket", Key="test-csv-file.csv")
+        self.assertEqual(e.exception.response["Error"]["Code"], "404")
+
+        head_archive_response = s3.head_object(Bucket="source-bucket", Key="archive/test-csv-file.csv")
+        assert head_archive_response["ResponseMetadata"]["HTTPStatusCode"] == 200
+
     def test_non_multipart_content_type_no_mesh_metadata(self):
         s3 = boto3.client("s3", region_name="eu-west-2")
         s3.put_object(

terraform/mesh_processor.tf

@@ -137,7 +137,8 @@ resource "aws_iam_policy" "mesh_processor_lambda_exec_policy" {
           "s3:GetObject",
           "s3:ListBucket",
           "s3:PutObject",
-          "s3:CopyObject"
+          "s3:CopyObject",
+          "s3:DeleteObject"
         ]
         Resource = [
           aws_s3_bucket.batch_data_source_bucket.arn,