Merge pull request #232 from BD2KGenomics/hotfix/1.1

Hotfix 1.1 (resolves #229)

Author: hannes-ucsc

src/toil_scripts/batch_alignment/launch_bwa_hg19.sh

@@ -2,14 +2,11 @@
 # John Vivian
 #
 # Please read the associated README.md before attempting to use.
-#
-# Precautionary step: Create location where jobStore and tmp files will exist
-mkdir -p ${HOME}/toil_mnt
-# Execution of pipeline
-python bwa_alignment.py \
-${HOME}/toil_mnt/jobStore \
---retryCount 3 \
---config bwa_config.csv \
+export PYTHONPATH=$(python -c 'from os.path import abspath as a, dirname as d;import sys;print d(d(d(a(sys.argv[1]))))' $0)
+python -m toil_scripts.batch_alignment.bwa_alignment \
+/data/alignment-jobStore \
+--retryCount 2 \
+--config bwa-config.csv \
 --lb KapaHyper \
 --ref https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/alignment/hg19.fa \
 --amb https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/alignment/hg19.fa.amb \
@@ -18,9 +15,7 @@ ${HOME}/toil_mnt/jobStore \
 --pac https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/alignment/hg19.fa.pac \
 --sa https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/alignment/hg19.fa.sa \
 --fai https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/alignment/hg19.fa.fai \
---workDir ${HOME}/toil_mnt \
---ssec ${HOME}/master.key \
---output_dir ${HOME} \
---s3_dir cgl-driver-projects/test/alignment \
---sudo \
-#--restart
+--workDir /data \
+--ssec /data/master.key \
+--output_dir /data \
+--use_bwakit

src/toil_scripts/batch_alignment/launch_bwa_hg19_mesos.sh

@@ -3,10 +3,11 @@
 #
 # Please read the associated README.md before attempting to use.
 #
-python bwa_alignment.py \
-aws:us-west-2:alignment-wcdt-run-1 \
---retryCount 3 \
---config bwa_config.csv \
+export PYTHONPATH=$(python -c 'from os.path import abspath as a, dirname as d;import sys;print d(d(d(a(sys.argv[1]))))' $0)
+python -m toil_scripts.batch_alignment.bwa_alignment \
+aws:us-west-2:alignment-run-1 \
+--retryCount 2 \
+--config bwa-config.csv \
 --lb KapaHyper \
 --ref https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/alignment/hg19.fa \
 --amb https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/alignment/hg19.fa.amb \
@@ -16,10 +17,9 @@ aws:us-west-2:alignment-wcdt-run-1 \
 --sa https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/alignment/hg19.fa.sa \
 --fai https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/alignment/hg19.fa.fai \
 --ssec /home/mesosbox/shared/master.key \
---output_dir /home/mesosbox/shared \
 --s3_dir cgl-driver-projects/test/alignment \
 --sseKey=/home/mesosbox/shared/master.key \
 --batchSystem="mesos" \
 --mesosMaster mesos-master:5050 \
 --workDir=/var/lib/toil \
-#--restart
+--use_bwakit

src/toil_scripts/batch_alignment/launch_bwa_hg38.sh

@@ -2,14 +2,11 @@
 # John Vivian
 #
 # Please read the associated README.md before attempting to use.
-#
-# Precautionary step: Create location where jobStore and tmp files will exist
-mkdir -p ${HOME}/toil_mnt
-# Execution of pipeline
-python bwa_alignment.py \
-${HOME}/toil_mnt/jobStore \
---retryCount 3 \
---config bwa_config.csv \
+export PYTHONPATH=$(python -c 'from os.path import abspath as a, dirname as d;import sys;print d(d(d(a(sys.argv[1]))))' $0)
+python -m toil_scripts.batch_alignment.bwa_alignment \
+/data/alignment-jobStore \
+--retryCount 2 \
+--config bwa-config.csv \
 --lb KapaHyper \
 --ref https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/alignment/hg38.fa \
 --amb https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/alignment/hg38.fa.amb \
@@ -18,9 +15,7 @@ ${HOME}/toil_mnt/jobStore \
 --pac https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/alignment/hg38.fa.pac \
 --sa https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/alignment/hg38.fa.sa \
 --fai https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/alignment/hg38.fa.fai \
---workDir ${HOME}/toil_mnt \
---ssec ${HOME}/master.key \
---output_dir ${HOME} \
---s3_dir cgl-driver-projects/test/alignment \
---sudo \
-#--restart
+--workDir /data \
+--ssec /data/master.key \
+--output_dir /data \
+--use_bwakit

src/toil_scripts/batch_alignment/launch_bwa_hg38_no_alt_mesos.sh

@@ -4,9 +4,9 @@
 # Please read the associated README.md before attempting to use.
 export PYTHONPATH=$(python -c 'from os.path import abspath as a, dirname as d;import sys;print d(d(d(a(sys.argv[1]))))' $0)
 python -m toil_scripts.batch_alignment.bwa_alignment \
-aws:us-west-2:bwa-1 \
---retryCount 3 \
---config /home/mesosbox/shared/config.txt \
+aws:us-west-2:alignment-run-1 \
+--retryCount 2 \
+--config /home/mesosbox/shared/bwa-config.csv \
 --lb KapaHyper \
 --ref https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/alignment/hg38_no_alt.fa \
 --amb https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/alignment/hg38_no_alt.fa.amb \
@@ -20,4 +20,4 @@ aws:us-west-2:bwa-1 \
 --batchSystem="mesos" \
 --mesosMaster mesos-master:5050 \
 --s3_dir cgl-driver-projects/test/alignment \
-#--restart
+--use_bwakit

src/toil_scripts/batch_alignment/launch_bwakit_grch38.sh

@@ -3,13 +3,11 @@
 #
 # Pipeline for alt-aware alignment against the GRCh38 build used in the 1000G vs. b38 recompute.
 # Precautionary step: Create location where jobStore and tmp files will exist and set TOIL_HOME.
-TOIL_HOME=FIXME
-mkdir -p ${TOIL_HOME}/toil_mnt
-# Execution of pipeline
+export PYTHONPATH=$(python -c 'from os.path import abspath as a, dirname as d;import sys;print d(d(d(a(sys.argv[1]))))' $0)
 python -m toil_scripts.batch_alignment.bwa_alignment \
-${TOIL_HOME}/toil_mnt/jobStore \
---retryCount 3 \
---config bwa_config.csv \
+/data/alignment-jobStore \
+--retryCount 2 \
+--config bwa-config.csv \
 --lb LIB \
 --ref https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/variant_grch38/GRCh38_full_analysis_set_plus_decoy_hla.fa \
 --amb https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/variant_grch38/GRCh38_full_analysis_set_plus_decoy_hla.fa.amb \
@@ -20,7 +18,5 @@ ${TOIL_HOME}/toil_mnt/jobStore \
 --fai https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/variant_grch38/GRCh38_full_analysis_set_plus_decoy_hla.fa.fai \
 --alt https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/variant_grch38/GRCh38_full_analysis_set_plus_decoy_hla.fa.alt \
 --use_bwakit \
---workDir ${TOIL_HOME}/toil_mnt \
---output_dir ${TOIL_HOME} \
---s3_dir cgl-driver-projects/test/alignment \
---sudo
+--workDir /data \
+--output_dir /data

src/toil_scripts/exome_variant_pipeline/exome_variant_pipeline.py

@@ -416,7 +416,7 @@ def pipeline_launchpoint(job, job_vars):
     job_vars: tuple     Contains the input_args and ids dictionaries
     """
     pre_processing = job.wrapJobFn(index_bams, job_vars).encapsulate()
-    run_mutect = job.wrapJobFn(mutect, job_vars, pre_processing.rv())
+    run_mutect = job.wrapJobFn(mutect, job_vars, pre_processing.rv(), disk='75G')
     # run_pindel = job.wrapJobFn(pindel, job_vars, pre_processing.rv())
     # run_muse = job.wrapJobFn(muse, job_vars, pre_processing.rv())
     consolidate = job.wrapJobFn(consolidate_output, job_vars, run_mutect.rv())#, run_pindel.rv(), run_muse.rv())

src/toil_scripts/exome_variant_pipeline/launch_variant_hg19.sh

@@ -2,23 +2,16 @@
 # John Vivian
 #
 # Please read the associated README.md before attempting to use.
-#
-# Precautionary step: Create location where jobStore and tmp files will exist
-mkdir -p ${HOME}/toil_mnt
-# Execution of pipeline
 export PYTHONPATH=$(python -c 'from os.path import abspath as a, dirname as d;import sys;print d(d(d(a(sys.argv[1]))))' $0)
 python toil_scripts.exome_variant_pipeline.exome_variant_pipeline \
-${HOME}/toil_mnt/jstore \
---retryCount 1 \
---config "exome_variant_config.csv" \
+/data/exome-jobStore\
+--retryCount 2 \
+--config "exome-config.csv" \
 --reference 'https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/variant_hg19/hg19.fa' \
 --phase 'https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/variant_hg19/1000G_phase1.indels.hg19.sites.vcf' \
 --mills 'https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/variant_hg19/Mills_and_1000G_gold_standard.indels.hg19.sites.vcf' \
 --dbsnp 'https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/variant_hg19/dbsnp_138.hg19.vcf' \
 --cosmic 'https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/variant_hg19/cosmic.hg19.vcf' \
---output_dir ${HOME}/ \
---ssec ${HOME}/master.key \
---s3_dir 'cgl-driver-projects/test/variants/' \
---workDir ${HOME}/toil_mnt \
---sudo \
-#--restart
+--output_dir /data \
+--ssec /data/master.key \
+--workDir /data

src/toil_scripts/exome_variant_pipeline/launch_variant_hg19_mesos.sh

@@ -4,9 +4,9 @@
 # Please read the associated README.md before attempting to use.
 export PYTHONPATH=$(python -c 'from os.path import abspath as a, dirname as d;import sys;print d(d(d(a(sys.argv[1]))))' $0)
 python -m toil_scripts.exome_variant_pipeline.exome_variant_pipeline \
-aws:us-west-2:jvivian-releases-exome-1 \
+aws:us-west-2:exome-run-1 \
 --retryCount 1 \
---config /home/mesosbox/shared/config.txt \
+--config /home/mesosbox/shared/exome-config.csv \
 --reference 'https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/variant_hg19/hg19.fa' \
 --phase 'https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/variant_hg19/1000G_phase1.indels.hg19.sites.vcf' \
 --mills 'https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/variant_hg19/Mills_and_1000G_gold_standard.indels.hg19.sites.vcf' \
@@ -16,5 +16,4 @@ aws:us-west-2:jvivian-releases-exome-1 \
 --s3_dir cgl-driver-projects/test/variants/ \
 --workDir /var/lib/toil \
 --batchSystem="mesos" \
---mesosMaster mesos-master:5050 \
-#--restart
+--mesosMaster mesos-master:5050

src/toil_scripts/rnaseq_cgl/launch_cgl_mesos.sh

@@ -6,13 +6,12 @@
 # Execution of pipeline
 export PYTHONPATH=$(python -c 'from os.path import abspath as a, dirname as d;import sys;print d(d(d(a(sys.argv[1]))))' $0)
 python -m toil_scripts.rnaseq_cgl.rnaseq_cgl_pipeline \
-aws:us-west-2:cgl-pipeline-run-1 \
---config /home/mesosbox/shared/config.txt \
---retryCount 1 \
+aws:us-west-2:rnaseq-run-1 \
+--config /home/mesosbox/shared/rnaseq-config.txt \
+--retryCount 2 \
 --ssec /home/mesosbox/shared/master.key \
 --s3_dir cgl-driver-projects/test/ \
 --sseKey=/home/mesosbox/shared/master.key \
 --batchSystem="mesos" \
 --mesosMaster mesos-master:5050 \
---workDir=/var/lib/toil \
-#--restart
+--workDir=/var/lib/toil

src/toil_scripts/rnaseq_cgl/launch_cgl_sm.sh

@@ -2,15 +2,11 @@
 # John Vivian
 #
 # Please read the associated README.md before attempting to use.
-#
-# Precautionary step: Create location where jobStore and tmp files will exist
-# Execution of pipeline
 export PYTHONPATH=$(python -c 'from os.path import abspath as a, dirname as d;import sys;print d(d(d(a(sys.argv[1]))))' $0)
 python -m toil_scripts.rnaseq_cgl.rnaseq_cgl_pipeline \
 /data/rnseq-jstore \
---config /data/config.txt \
+--config /data/rnaseq-config.csv \
 --retryCount 1 \
 --ssec /data/master.key \
---s3_dir cgl-driver-projects/test/releases \
+--output_dir /data \
 --workDir /data
-#--restart