BD2KGenomics · jnkrupp · Dec 4, 2017 · Dec 6, 2017 · Dec 7, 2017 · Dec 7, 2017
diff --git a/.travis.yml b/.travis.yml
@@ -0,0 +1,17 @@
+sudo: required
+
+services:
+ - docker
+
+language: python
+
+before_install:
+ - sudo apt-get update
+ - git clone https://github.com/jnkrupp/dcc-action-service.git
+ - cd dcc-action-service
+ - git checkout feature/add-travisci
+script:
+# - sudo docker-compose -f docker-compose.yml up --build -d
+# - sudo docker-compose -f docker-compose.yml down
+ - make testme
+
diff --git a/Makefile b/Makefile
@@ -0,0 +1,32 @@
+build:
+	docker build -t quay.io/ucsc_cgl/dcc-action-service .
+
+run:
+	# Apply migrations and then run using the built image in daemon mode
+	sudo docker-compose up -d --build --force-recreate
+
+populate:
+	# Populate the ElasticSearch index
+	sudo docker-compose exec dcc-action-service /app/test/populator.sh
+
+unittests:
+	# Run pytest inside the running container from run
+	sudo docker-compose exec dcc-action-service py.test -p no:cacheprovider -s -x
+
+stop:
+	sudo docker-compose down --rmi 'all'
+
+# This is to set up the environment for some quick prototyping. This will setup ElasticSearch and Kibana bounded on the
+# local host loaded with some testing files
+play: stop reset run
+	sleep 60
+	echo "Finished Sleeping 60 Seconds"
+	$(MAKE) populate
+#	echo "Kibana is now available on localhost:5601"
+
+# Clears the whole system, reloads indexes, and runs the tests
+testme: play unittests
+
+reset:
+	sudo docker-compose stop
+	sudo docker-compose rm -f
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -0,0 +1,36 @@
+version: '2'
+
+services:
+  dcc-action-service:
+    environment:
+      ES_SERVICE: "elasticsearch1"
+      ES_DOMAIN: "elasticsearch1"
+      ES_PORT: "9200"
+      ES_PROTOCOL: "http"
+      ES_DONOR_INDEX: "analysis_index"
+    build:
+      context: ./luigi_task_executor
+      dockerfile: Dockerfile_decider
+    ports:
+      - "9000:80"
+    networks:
+      - esnet
+    volumes:
+      - ~/dcc-action-service/logs:/home/ubuntu/logs
+      - ~/dcc-action-service/pipeline_deciders:/home/ubuntu/pipeline_deciders
+
+  elasticsearch1:
+    image: docker.elastic.co/elasticsearch/elasticsearch:5.6.3
+    container_name: "elasticsearch1"
+    environment:
+      - "discovery.type=single-node"
+      - "xpack.security.enabled=false"
+    ports:
+       - "9200:9200"
+       - "9300:9300"
+    networks:
+      - esnet
+
+networks:
+  esnet:
+    driver: bridge    
diff --git a/luigi_task_executor/Dockerfile_decider b/luigi_task_executor/Dockerfile_decider
@@ -111,18 +111,25 @@ COPY Protect.py .
 RUN chown ubuntu:ubuntu Protect.py
 RUN chmod a+x Protect.py
 
-COPY Fusion.py . 
+COPY Fusion.py .
 RUN chown ubuntu:ubuntu Fusion.py
 RUN chmod a+x Fusion.py
 
 COPY base_decider.py . 
 RUN chown ubuntu:ubuntu base_decider.py
 RUN chmod a+x base_decider.py
 
+COPY test/ /app/test
+RUN chown -R ubuntu:ubuntu /app/test
+RUN chmod -R a+x /app/test
+
 
 #allow ubuntu (and all other users) to execute sudo without a password
 RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
+#install pip
+RUN pip install -U pytest
+
 #run everything from now on as ubuntu
 USER ubuntu
 
@@ -136,10 +143,10 @@ RUN python3 get-pip.py --user
 RUN pip install awscli --upgrade --user
 ENV PATH /home/ubuntu/.local/bin:$PATH
 
-
 CMD ["/home/ubuntu/setup_decider_jobs.sh"]
 
 # Run the command on container startup in test mode
 #CMD sudo cron && sudo tail -f /tmp/decider_log
 ##CMD sudo cron && sudo tail -f /var/log/cron.log
 
+WORKDIR /app/
diff --git a/luigi_task_executor/base_decider.py b/luigi_task_executor/base_decider.py
@@ -48,6 +48,9 @@ class ConsonanceTask(luigi.Task):
     #Consonance will not be called in test mode
     test_mode = luigi.BoolParameter(default = False)
 
+    #Use local storage for TravisCI testing
+    use_local_storage = luigi.BoolParameter(default = True)
+
     def run(self):
         print("\n\n\n** TASK RUN **")
 
@@ -199,17 +202,23 @@ def run(self):
         print("\n\n\n\n** TASK RUN DONE **")
 
     def save_metadata_json(self):
+        if(self.use_local_storage == True):
+            return luigi.LocalTarget(self.s3_metadata_json_file_path)
         return S3Target(self.s3_metadata_json_file_path)
 
-    def save_dockstore_tool_runner_json_local(self):
+    def save_dockstore_tool_runner_json_local(self): 
         return luigi.LocalTarget(self.local_dockstore_tool_runner_json_file_path)
 
     def save_dockstore_tool_runner_json(self):
+        if(self.use_local_storage == True):
+            return luigi.LocalTarget(self.s3_dockstore_tool_runner_json_file_path) 
         return S3Target(self.s3_dockstore_tool_runner_json_file_path)
 
     def output(self):
 #        return S3Target(self.s3_finished_json_file_path)
 #        print("output target:{}".format("s3://" + self.touch_file_path + "/" +  self.file_prefix + "_finished.json"))
+        if(self.use_local_storage == True):
+            return luigi.LocalTarget("s3://" + self.touch_file_path + "/" +  self.file_prefix + "_finished.json")
         return S3Target("s3://" + self.touch_file_path + "/" +  self.file_prefix + "_finished.json")
 
 
@@ -234,6 +243,9 @@ class base_Coordinator(luigi.Task):
     #Consonance will not be called in test mode
     test_mode = luigi.BoolParameter(default = False)
 
+    #Use local storage for TravisCI testing
+    use_local_storage = luigi.BoolParameter(default = True)
+
     center = luigi.Parameter(default = "")
     program = luigi.Parameter(default = "")
     project = luigi.Parameter(default = "")
@@ -598,7 +610,8 @@ def requires(self):
                     metadata_json_file_name = job['metadata_json_file_name'],
                     touch_file_path = job['touch_file_path'],
                     file_prefix = job['file_prefix'],
-                    test_mode=self.test_mode))
+                    test_mode=self.test_mode,
+                    use_local_storage=self.use_local_storage))
 
 
         print("total of {} jobs; max jobs allowed is {}\n\n".format(str(len(listOfJobs)), self.max_jobs))
@@ -622,6 +635,9 @@ def output(self):
         ts = time.time()
         ts_str = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H:%M:%S')
         workflow_version_dir = self.workflow_version.replace('.', '_')
+        if(self.use_local_storage == True):
+            return luigi.LocalTarget('s3://'+ self.touch_file_bucket + '/consonance-jobs/{}_Coordinator/{}/{}-{}.txt'.format( \
+                                  self.get_pipeline_name(), workflow_version_dir, self.get_pipeline_name(), ts_str)) 
         return S3Target('s3://'+ self.touch_file_bucket + '/consonance-jobs/{}_Coordinator/{}/{}-{}.txt'.format( \
                                   self.get_pipeline_name(), workflow_version_dir, self.get_pipeline_name(), ts_str))
 

diff --git a/luigi_task_executor/run_Luigi_Deciders.sh b/luigi_task_executor/run_Luigi_Deciders.sh
@@ -62,14 +62,15 @@ echo "Running Luigi deciders" >> ${LOG_FILE_PATH}/cron_decider_log.txt
 # run the decider
 #use the '--test-mode' switch to skip running Consonance
 #This will be the new run commmand:
-PYTHONPATH="${DECIDER_SOURCE_PATH}" luigi --module RNA-Seq RNASeqCoordinator --dockstore-tool-running-dockstore-tool "quay.io/ucsc_cgl/dockstore-tool-runner:1.0.22" --workflow-version "3.3.4-1.12.3" --touch-file-bucket ${TOUCH_FILE_DIRECTORY} --redwood-host ${STORAGE_SERVER} --redwood-token ${STORAGE_ACCESS_TOKEN} --es-index-host ${ELASTIC_SEARCH_SERVER} --es-index-port ${ELASTIC_SEARCH_PORT} --vm-instance-type "c4.8xlarge" --vm-region ${AWS_REGION} --program Treehouse --tmp-dir /datastore --test-mode > "${LOG_FILE_PATH}"/cron_log_RNA-Seq_decider_stdout.txt 2> "${LOG_FILE_PATH}"/cron_log_RNA-Seq_decider_stderr.txt
+#PYTHONPATH="${DECIDER_SOURCE_PATH}" luigi --module RNA-Seq RNASeqCoordinator --dockstore-tool-running-dockstore-tool "quay.io/ucsc_cgl/dockstore-tool-runner:1.0.21" --workflow-version "3.3.4-1.12.3" --touch-file-bucket ${TOUCH_FILE_DIRECTORY} --redwood-host ${STORAGE_SERVER} --redwood-token ${STORAGE_ACCESS_TOKEN} --es-index-host ${ELASTIC_SEARCH_SERVER} --es-index-port ${ELASTIC_SEARCH_PORT} --vm-instance-type "c4.8xlarge" --vm-region ${AWS_REGION} --program Treehouse --tmp-dir /datastore --test-mode  > "${LOG_FILE_PATH}"/cron_log_RNA-Seq_decider_stdout.txt 2> "${LOG_FILE_PATH}"/cron_log_RNA-Seq_decider_stderr.txt
 
 #Run the ProTECT decider:
-PYTHONPATH="${DECIDER_SOURCE_PATH}" luigi --module Protect ProtectCoordinator --dockstore-tool-running-dockstore-tool "quay.io/ucsc_cgl/dockstore-tool-runner:1.0.22" --workflow-version "2.5.5-1.12.3" --touch-file-bucket ${TOUCH_FILE_DIRECTORY} --redwood-host ${STORAGE_SERVER} --redwood-token ${STORAGE_ACCESS_TOKEN} --es-index-host ${ELASTIC_SEARCH_SERVER} --es-index-port ${ELASTIC_SEARCH_PORT} --vm-instance-type "c4.8xlarge" --vm-region ${AWS_REGION} --project TARGET --tmp-dir /datastore --test-mode  > "${LOG_FILE_PATH}"/cron_log_Protect_decider_stdout.txt 2> "${LOG_FILE_PATH}"/cron_log_Protect_decider_stderr.txt
+#PYTHONPATH="${DECIDER_SOURCE_PATH}" luigi --module Protect ProtectCoordinator --dockstore-tool-running-dockstore-tool "quay.io/ucsc_cgl/dockstore-tool-runner:1.0.21" --workflow-version "2.5.5-1.12.3" --touch-file-bucket ${TOUCH_FILE_DIRECTORY} --redwood-host ${STORAGE_SERVER} --redwood-token ${STORAGE_ACCESS_TOKEN} --es-index-host ${ELASTIC_SEARCH_SERVER} --es-index-port ${ELASTIC_SEARCH_PORT} --vm-instance-type "c4.8xlarge" --vm-region ${AWS_REGION} --project TARGET --tmp-dir /datastore --test-mode  > "${LOG_FILE_PATH}"/cron_log_Protect_decider_stdout.txt 2> "${LOG_FILE_PATH}"/cron_log_Protect_decider_stderr.txt
 
-PYTHONPATH="${DECIDER_SOURCE_PATH}" luigi --module CNV CNVCoordinator --dockstore-tool-running-dockstore-tool "quay.io/ucsc_cgl/dockstore-tool-runner:1.0.22" --workflow-version "v1.0.0" --touch-file-bucket ${TOUCH_FILE_DIRECTORY} --redwood-host ${STORAGE_SERVER} --redwood-token ${STORAGE_ACCESS_TOKEN} --es-index-host ${ELASTIC_SEARCH_SERVER} --es-index-port ${ELASTIC_SEARCH_PORT} --vm-instance-type "c4.8xlarge" --vm-region ${AWS_REGION} --project WCDT --tmp-dir /datastore --test-mode  > "${LOG_FILE_PATH}"/cron_log_CNV_decider_stdout.txt 2> "${LOG_FILE_PATH}"/cron_log_CNV_decider_stderr.txt
 
-PYTHONPATH="${DECIDER_SOURCE_PATH}" luigi --module Fusion FusionCoordinator --dockstore-tool-running-dockstore-tool "quay.io/ucsc_cgl/dockstore-tool-runner:1.0.22" --workflow-version "0.2.1" --touch-file-bucket ${TOUCH_FILE_DIRECTORY} --redwood-host ${STORAGE_SERVER} --redwood-token ${STORAGE_ACCESS_TOKEN} --es-index-host ${ELASTIC_SEARCH_SERVER} --es-index-port ${ELASTIC_SEARCH_PORT} --vm-instance-type "r4.8xlarge" --vm-region ${AWS_REGION} --program Fusion --tmp-dir /datastore --test-mode > "${LOG_FILE_PATH}"/cron_log_Fusion_decider_stdout.txt 2> "${LOG_FILE_PATH}"/cron_log_Fusion_decider_stderr.txt
+#PYTHONPATH="${DECIDER_SOURCE_PATH}" luigi --module CNV CNVCoordinator --dockstore-tool-running-dockstore-tool "quay.io/ucsc_cgl/dockstore-tool-runner:1.0.21" --workflow-version "v1.0.0" --touch-file-bucket ${TOUCH_FILE_DIRECTORY} --redwood-host ${STORAGE_SERVER} --redwood-token ${STORAGE_ACCESS_TOKEN} --es-index-host ${ELASTIC_SEARCH_SERVER} --es-index-port ${ELASTIC_SEARCH_PORT} --vm-instance-type "c4.8xlarge" --vm-region ${AWS_REGION} --project WCDT --tmp-dir /datastore --test-mode  > "${LOG_FILE_PATH}"/cron_log_CNV_decider_stdout.txt 2> "${LOG_FILE_PATH}"/cron_log_CNV_decider_stderr.txt
+
+#PYTHONPATH="${DECIDER_SOURCE_PATH}" luigi --module Fusion FusionCoordinator --dockstore-tool-running-dockstore-tool "quay.io/ucsc_cgl/dockstore-tool-runner:1.0.21" --workflow-version "0.2.1" --touch-file-bucket ${TOUCH_FILE_DIRECTORY} --redwood-host ${STORAGE_SERVER} --redwood-token ${STORAGE_ACCESS_TOKEN} --es-index-host ${ELASTIC_SEARCH_SERVER} --es-index-port ${ELASTIC_SEARCH_PORT} --vm-instance-type "c4.8xlarge" --vm-region ${AWS_REGION} --program Fusion --tmp-dir /datastore --test-mode --use-local-storage > "${LOG_FILE_PATH}"/cron_log_Fusion_decider_stdout.txt 2> "${LOG_FILE_PATH}"/cron_log_Fusion_decider_stderr.txt
 
 #execute commands to run addtional deciders in the extra folder
 #if [ -f ${DECIDER_SOURCE_PATH}/extra/run_addtional_deciders.sh ]
@@ -79,20 +80,20 @@ PYTHONPATH="${DECIDER_SOURCE_PATH}" luigi --module Fusion FusionCoordinator --do
 
 #These are log file messages used for testing: 
 
-#echo -e "\n\n"
-#echo "${now} DEBUG!! run of luigi decider!!!" >> ${LOG_FILE_PATH}/logfile.txt
-#echo "executing consonance --version test" >> ${LOG_FILE_PATH}/logfile.txt
+echo -e "\n\n"
+echo "${now} DEBUG!! run of luigi decider!!!" >> ${LOG_FILE_PATH}/logfile.txt
+echo "executing consonance --version test" >> ${LOG_FILE_PATH}/logfile.txt
 #consonance --version >> ${LOG_FILE_PATH}/logfile.txt
 
-#echo "redwood server is ${STORAGE_SERVER}" >> ${LOG_FILE_PATH}/logfile.txt
-#echo "redwood token is ${STORAGE_ACCESS_TOKEN}" >> ${LOG_FILE_PATH}/logfile.txt
+echo "redwood server is ${STORAGE_SERVER}" >> ${LOG_FILE_PATH}/logfile.txt
+echo "redwood token is ${STORAGE_ACCESS_TOKEN}" >> ${LOG_FILE_PATH}/logfile.txt
 
-#echo "elastic search server is ${ELASTIC_SEARCH_SERVER}" >> ${LOG_FILE_PATH}/logfile.txt
-#echo "elastic search port is ${ELASTIC_SEARCH_PORT}" >> ${LOG_FILE_PATH}/logfile.txt
+echo "elastic search server is ${ELASTIC_SEARCH_SERVER}" >> ${LOG_FILE_PATH}/logfile.txt
+echo "elastic search port is ${ELASTIC_SEARCH_PORT}" >> ${LOG_FILE_PATH}/logfile.txt
 
-#echo "touch file directory is ${TOUCH_FILE_DIRECTORY}" >> ${LOG_FILE_PATH}/logfile.txt
+echo "touch file directory is ${TOUCH_FILE_DIRECTORY}" >> ${LOG_FILE_PATH}/logfile.txt
 
-#echo "executing java -version test" >> ${LOG_FILE_PATH}/logfile.txt
+echo "executing java -version test" >> ${LOG_FILE_PATH}/logfile.txt
 #java -version >> ${LOG_FILE_PATH}/logfile.txt 2>&1
 
 #echo "executing aws test" >> ${LOG_FILE_PATH}/logfile.txt

diff --git a/luigi_task_executor/test/__init__.py b/luigi_task_executor/test/__init__.py