Merge pull request #128 from aws-samples/fix/kms-key

PCA Workflow failure with KMS encrypted recording / BulkMove CopyObject AccessDenied issue with tagged recordings

Author: rstrahan

pca-server/cfn/lib/pca.template

@@ -299,4 +299,5 @@ Outputs:
         - !Sub '"${SFFinalProcessingRole.Arn}"'
         - !Sub '"${SFCTRGenesysRole.Arn}"'
         - !Sub '"${SFTranscribeFailedRole.Arn}"'
+        - !Sub '"${SFPostCTRProcessingRole.Arn}"'
 

pca-server/src/pca/pca-aws-sf-bulk-move-files.py

@@ -24,27 +24,41 @@ def lambda_handler(event, context):
 
     # Get as many files from S3 as we can move this time (minimum of queueSpace and dripRate)
     s3Client = boto3.client('s3')
+    s3 = boto3.resource('s3')
     maxKeys = min(dripRate, queueSpace) + 10 # list a few additional keys to allow for some folder objects that won't be moved
     response = s3Client.list_objects_v2(Bucket=sourceBucket, MaxKeys=maxKeys)
     if "Contents" in response:
         # We now have a list of objects that we can use
-        sourcePrefix = "/" + sourceBucket + "/"
         keyPrefix = targetAudioKey
         if keyPrefix != "":
             keyPrefix += "/"
-        files = [f for f in response["Contents"] if not f["Key"].endswith("/")][:dripRate] # ignore folder objects
+        files = [f for f in response["Contents"] if not f["Key"].endswith("/")][:dripRate]
+        folders = [f for f in response["Contents"] if f["Key"].endswith("/")]
         for audioFile in files:
             try:
                 # Copy and delete file
-                print(f'Copying: Bucket={targetBucket}, CopySource={(sourcePrefix + audioFile["Key"])}, Key={(keyPrefix + audioFile["Key"])}')
-                copyResponse = s3Client.copy_object(Bucket=targetBucket,
-                                                    CopySource=(sourcePrefix + audioFile["Key"]),
-                                                    Key=(keyPrefix + audioFile["Key"]))
+                copyDestnKey = keyPrefix + audioFile["Key"]
+                copy_source = {
+                    'Bucket': sourceBucket,
+                    'Key': audioFile["Key"]
+                }
+                print(f'Copying: copy_source={copy_source}, targetBucket={targetBucket}, copyDestnKey={copyDestnKey}')
+                s3.meta.client.copy(copy_source, targetBucket, copyDestnKey)
                 deleteResponse = s3Client.delete_object(Bucket=sourceBucket, Key=audioFile["Key"])
                 movedFiles += 1
             except Exception as e:
                 print("Failed to move audio file {}".format(audioFile["Key"]))
                 print(e)
+                raise
+        # Delete any folder objects so they do not appear in subsequent object lists - usually these are created in S3 console only
+        for folder in folders:
+            try:
+                print(f'Deleting folder object: Bucket={sourceBucket}, Key={folder["Key"]}')
+                deleteResponse = s3Client.delete_object(Bucket=sourceBucket, Key=folder["Key"])
+            except Exception as e:
+                print("Failed to delete folder file {}".format(folder["Key"]))
+                print(e)
+                raise        
 
     # Increase our counter, remove the queue value and return
     sfData["filesProcessed"] += movedFiles