Merge pull request #264 from ENCODE-DCC/dev

v2.1.4

Author: leepc12

.circleci/config.yml

@@ -7,7 +7,7 @@ defaults: &defaults
 
 machine_defaults: &machine_defaults
   machine: 
-    image: ubuntu-1604:202007-01
+    image: ubuntu-2004:202010-01
   working_directory: ~/chip-seq-pipeline2
 
 make_tag: &make_tag
@@ -16,37 +16,17 @@ make_tag: &make_tag
     echo "export TAG=encodedcc/chip-seq-pipeline:${CIRCLE_BRANCH}_${CIRCLE_WORKFLOW_ID}" > ${BASH_ENV}
 
 commands:
-  download_task_test_data:
-    description: "Download task test data. This is based on py2 so run this before installing py3."
-    steps:
-      - run:
-          command: |
-            cd dev/test/test_task/
-            rm -rf chip-seq-pipeline-test-data
-            export BOTO_CONFIG=/dev/null
-            gsutil -m cp -r gs://encode-pipeline-test-samples/encode-chip-seq-pipeline/chip-seq-pipeline-test-data .
-
   install_python3_caper_gcs:
     description: "Install py3, caper and gcs. Set py3 as default python."
     steps:      
       - run:
           command: |
             sudo apt-get update && sudo apt-get install software-properties-common git wget curl -y
+            sudo apt-get install -y software-properties-common git wget curl python3 python3-pip default-jre
 
-            # install java 11
-            sudo add-apt-repository ppa:openjdk-r/ppa -y
-            sudo apt-get update && sudo apt-get install openjdk-11-jdk -y
-            # automatically set 11 as default java
-            sudo update-java-alternatives -a
-
-            sudo add-apt-repository ppa:deadsnakes/ppa -y
-            sudo apt-get update && sudo apt-get install python3.6 -y
-            sudo wget --no-check-certificate https://bootstrap.pypa.io/get-pip.py
-            sudo python3.6 get-pip.py
-            sudo ln -s /usr/bin/python3.6 /usr/local/bin/python3
-
-            pip3 install --upgrade pip
-            pip3 install caper google-cloud-storage
+            sudo python3 -m pip install --upgrade pip
+            sudo pip3 install PyYAML --ignore-installed
+            sudo pip3 install caper google-cloud-storage
 
   run_workflow_test:
     description: "Run workflow test. This requires an env var INPUT as input JSON's basename."
@@ -86,7 +66,7 @@ jobs:
           name: build image
           command: |
             source ${BASH_ENV}
-            export DOCKER_CACHE_TAG=v1.8.1
+            export DOCKER_CACHE_TAG=v2.1.0
             echo "pulling ${DOCKER_CACHE_TAG}!"
             docker pull encodedcc/chip-seq-pipeline:${DOCKER_CACHE_TAG}
             docker login -u=${DOCKERHUB_USER} -p=${DOCKERHUB_PASS}
@@ -97,7 +77,6 @@ jobs:
     <<: *machine_defaults
     steps:
       - checkout
-      - download_task_test_data
       - install_python3_caper_gcs
       - run: *make_tag
       - run:
@@ -106,6 +85,9 @@ jobs:
             source ${BASH_ENV}
             cd dev/test/test_task/
 
+            echo ${GCLOUD_SERVICE_ACCOUNT_SECRET_JSON} > tmp_secret_key.json
+            export GOOGLE_APPLICATION_CREDENTIALS=$PWD/tmp_secret_key.json
+
             # unittest for wrapper .py
             docker run ${TAG} bash -c "cd /software/chip-seq-pipeline/dev/test/test_py/; pytest -vv"
 

.gitignore

@@ -125,3 +125,5 @@ tmp_db*
 temp_db*
 
 cromwell.out
+cromwell.out.*
+

README.md

@@ -127,8 +127,8 @@ You cannot use these input JSON files directly. Go to the destination directory
 We have a separate project on DNANexus to provide example FASTQs and `genome_tsv` for `hg38` and `mm10` (also chr19-only version of those two. Use chr19-only versions for testing). We recommend to make copies of these directories on your own project.
 
 `genome_tsv`
-- AWS: https://platform.dnanexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/pipeline-genome-data/genome_tsv/v3
-- Azure: https://platform.dnanexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/pipeline-genome-data/genome_tsv/v3
+- AWS: https://platform.dnanexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/pipeline-genome-data/genome_tsv/v4
+- Azure: https://platform.dnanexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/pipeline-genome-data/genome_tsv/v4
 
 Example FASTQs
 - AWS: https://platform.dnanexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq_subsampled

chip.wdl

@@ -7,10 +7,10 @@ struct RuntimeEnvironment {
 }
 
 workflow chip {
-    String pipeline_ver = 'v2.1.2'
+    String pipeline_ver = 'v2.1.4'
 
     meta {
-        version: 'v2.1.2'
+        version: 'v2.1.4'
 
         author: 'Jin wook Lee'
         email: '[email protected]'
@@ -19,8 +19,8 @@ workflow chip {
 
         specification_document: 'https://docs.google.com/document/d/1lG_Rd7fnYgRpSIqrIfuVlAz2dW1VaSQThzk836Db99c/edit?usp=sharing'
 
-        default_docker: 'encodedcc/chip-seq-pipeline:v2.1.2'
-        default_singularity: 'https://encode-pipeline-singularity-image.s3.us-west-2.amazonaws.com/chip-seq-pipeline_v2.1.2.sif'
+        default_docker: 'encodedcc/chip-seq-pipeline:v2.1.4'
+        default_singularity: 'https://encode-pipeline-singularity-image.s3.us-west-2.amazonaws.com/chip-seq-pipeline_v2.1.4.sif'
         croo_out_def: 'https://storage.googleapis.com/encode-pipeline-output-definition/chip.croo.v5.json'
 
         parameter_group: {
@@ -71,8 +71,8 @@ workflow chip {
     }
     input {
         # group: runtime_environment
-        String docker = 'encodedcc/chip-seq-pipeline:v2.1.2'
-        String singularity = 'https://encode-pipeline-singularity-image.s3.us-west-2.amazonaws.com/chip-seq-pipeline_v2.1.2.sif'
+        String docker = 'encodedcc/chip-seq-pipeline:v2.1.4'
+        String singularity = 'https://encode-pipeline-singularity-image.s3.us-west-2.amazonaws.com/chip-seq-pipeline_v2.1.4.sif'
         String conda = 'encode-chip-seq-pipeline'
         String conda_macs2 = 'encode-chip-seq-pipeline-macs2'
         String conda_spp = 'encode-chip-seq-pipeline-spp'
@@ -2096,6 +2096,7 @@ workflow chip {
         ctl_paired_ends = ctl_paired_end_,
         pipeline_type = pipeline_type,
         aligner = aligner_,
+        no_dup_removal = no_dup_removal,
         peak_caller = peak_caller_,
         cap_num_peak = cap_num_peak_,
         idr_thresh = idr_thresh,
@@ -3046,6 +3047,7 @@ task qc_report {
         Array[Boolean] ctl_paired_ends
         String pipeline_type
         String aligner
+        Boolean no_dup_removal
         String peak_caller
         Int cap_num_peak
         Float idr_thresh
@@ -3105,6 +3107,7 @@ task qc_report {
     command {
         set -e
         python3 $(which encode_task_qc_report.py) \
+            --pipeline-prefix chip \
             ${'--pipeline-ver ' + pipeline_ver} \
             ${"--title '" + sub(title,"'","_") + "'"} \
             ${"--desc '" + sub(description,"'","_") + "'"} \
@@ -3114,6 +3117,7 @@ task qc_report {
             --ctl-paired-ends ${sep=' ' ctl_paired_ends} \
             --pipeline-type ${pipeline_type} \
             --aligner ${aligner} \
+            ${if (no_dup_removal) then '--no-dup-removal ' else ''} \
             --peak-caller ${peak_caller} \
             ${'--cap-num-peak ' + cap_num_peak} \
             --idr-thresh ${idr_thresh} \