Check minimum file size, misc updates for 1.2.1 (#97)

* Add min file size, bump DCP versions

* Update Dockerfile

* Update Makefile

* Update Makefile

* Update fabfile.py

* Update cp-worker to check for a minimum file size

If not specified, use 1 byte

* Don't write bad code

* Don't write bad code

* Don't check alarms at midnight

Author: bethac07

config.py

@@ -36,3 +36,4 @@
 # REDUNDANCY CHECKS
 CHECK_IF_DONE_BOOL = 'True'  #True or False- should it check if there are a certain number of non-empty files and delete the job if yes?
 EXPECTED_NUMBER_FILES = 7    #What is the number of files that trigger skipping a job?
+MIN_FILE_SIZE_BYTES = 1      #What is the minimal number of bytes an object should be to "count"?

fabfile.py

@@ -154,6 +154,10 @@ def generate_task_definition():
 	{
 	    "name": "SECONDS_TO_START",
 	    "value": str(SECONDS_TO_START)
+	},
+	{
+	    "name": "MIN_FILE_SIZE_BYTES",
+	    "value": str(MIN_FILE_SIZE_BYTES)
 	}
     ]
     return task_definition

run.py

@@ -305,7 +305,8 @@ def monitor():
 	#This is slooooooow, which is why we don't just do it at the end
         curtime=datetime.datetime.now().strftime('%H%M')
         if curtime[-2:]=='00':
-            killdeadAlarms(fleetId,monitorapp)
+	    if curtime[:2]!='00':
+                killdeadAlarms(fleetId,monitorapp)
 	#Once every 10 minutes, check if all jobs are in process, and if so scale the spot fleet size to match
 	#the number of jobs still in process WITHOUT force terminating them.
 	#This can help keep costs down if, for example, you start up 100+ machines to run a large job, and

worker/Dockerfile

@@ -5,7 +5,7 @@
 # This wraps the cellprofiler docker registry
 #
 
-FROM cellprofiler/cellprofiler:3.1.5
+FROM cellprofiler/cellprofiler:3.1.9
 
 # Install S3FS 
 

worker/Makefile

@@ -1,6 +1,6 @@
 user = cellprofiler
 project = distributed-cellprofiler
-tag = latest
+tag = 1.2.1_3.1.9
 
 .DEFAULT_GOAL: build
 build:

worker/cp-worker.py

@@ -22,6 +22,10 @@
 LOG_GROUP_NAME= os.environ['LOG_GROUP_NAME']
 CHECK_IF_DONE_BOOL= os.environ['CHECK_IF_DONE_BOOL']
 EXPECTED_NUMBER_FILES= os.environ['EXPECTED_NUMBER_FILES']
+if 'MIN_FILE_SIZE_BYTES' not in os.environ:
+    MIN_FILE_SIZE_BYTES = 1
+else:
+    MIN_FILE_SIZE_BYTES = int(os.environ['MIN_FILE_SIZE_BYTES'])
 
 #################################
 # CLASS TO HANDLE THE SQS QUEUE
@@ -132,9 +136,9 @@ def runCellProfiler(message):
 		s3client=boto3.client('s3')
 		bucketlist=s3client.list_objects(Bucket=AWS_BUCKET,Prefix=remoteOut+'/')
 		objectsizelist=[k['Size'] for k in bucketlist['Contents']]
+		objectsizelist = [i for i in objectsizelist if i >= MIN_FILE_SIZE_BYTES]
 		if len(objectsizelist)>=int(EXPECTED_NUMBER_FILES):
-		    if 0 not in objectsizelist:
-			return 'SUCCESS'
+		    return 'SUCCESS'
 	except KeyError: #Returned if that folder does not exist
 		pass