correct role handling and S3FS bypass (#130)

* correct role handling

* enable separate source and destination buckets

* fix complete s3fs bybass

* add upload flags

* update to AWS CLI2

Author: ErinWeisbart

config.py

@@ -9,7 +9,10 @@
 AWS_REGION = 'us-east-1'
 AWS_PROFILE = 'default'                 # The same profile used by your AWS CLI installation
 SSH_KEY_NAME = 'your-key-file.pem'      # Expected to be in ~/.ssh
-AWS_BUCKET = 'your-bucket-name'
+AWS_BUCKET = 'your-bucket-name'         # Bucket to use for logging
+SOURCE_BUCKET = 'bucket-name'           # Bucket to download files from
+DESTINATION_BUCKET = 'bucket-name'      # Bucket to upload files to
+UPLOAD_FLAGS = ''                       # Any flags needed for upload to destination bucket
 
 # EC2 AND ECS INFORMATION:
 ECS_CLUSTER = 'default'
@@ -32,7 +35,7 @@
 SQS_DEAD_LETTER_QUEUE = 'arn:aws:sqs:some-region:111111100000:DeadMessages'
 
 # LOG GROUP INFORMATION:
-LOG_GROUP_NAME = APP_NAME 
+LOG_GROUP_NAME = APP_NAME
 
 # REDUNDANCY CHECKS
 CHECK_IF_DONE_BOOL = 'False'  #True or False- should it check if there are a certain number of non-empty files and delete the job if yes?

files/exampleJob.json

@@ -1,7 +1,8 @@
 {
-  "_comment1": "Paths in this file are relative to the root of your S3 bucket",
-  "pipeline": "projects/analysis.cppipe", 
-  "data_file": "projects/list_of_images.csv", 
+  "_comment1": "Paths in this file are relative to the root of S3 buckets",
+  "_comment2": "pipeline, data_file, and input are relative to SOURCE_BUCKET; output to DESTINATION_BUCKET",
+  "pipeline": "projects/analysis.cppipe",
+  "data_file": "projects/list_of_images.csv",
   "input": "projects/input/",
   "output": "projects/output/",
   "output_structure": "Metadata_Plate-Metadata_Well-Metadata_Site",
@@ -11,4 +12,3 @@
     {"Metadata": "Metadata_Plate=Plate1,Metadata_Well=A01,Metadata_Site=2"}
   ]
 }
-