Skip to content

Commit 9461e6b

Browse files
leepc12leepc12
authored
Merge pull request #199 from ENCODE-DCC/dev
v1.6.1
2 parents f3ab828 + ba7bbca commit 9461e6b

14 files changed

+495
-27
lines changed

‎.circleci/config.yml‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ defaults: &defaults
77

88
machine_defaults: &machine_defaults
99
machine:
10-
image: circleci/classic:201808-01
10+
image: ubuntu-1604:202007-01
1111
working_directory: ~/chip-seq-pipeline2
1212

1313
make_tag: &make_tag
@@ -31,9 +31,14 @@ commands:
3131
steps:
3232
- run:
3333
command: |
34-
pyenv global 3.7.0
35-
pip install --upgrade pip
36-
pip install caper google-cloud-storage
34+
sudo apt-get update && sudo apt-get install software-properties-common git wget curl default-jre -y
35+
sudo add-apt-repository ppa:deadsnakes/ppa -y
36+
sudo apt-get update && sudo apt-get install python3.6 -y
37+
sudo wget --no-check-certificate https://bootstrap.pypa.io/get-pip.py
38+
sudo python3.6 get-pip.py
39+
sudo ln -s /usr/bin/python3.6 /usr/local/bin/python3
40+
pip3 install --upgrade pip
41+
pip3 install caper google-cloud-storage
3742
3843
run_workflow_test:
3944
description: "Run workflow test. This requires an env var INPUT as input JSON's basename."

chip.croo.v4.json renamed to chip.croo.v5.json

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,10 @@
177177
"path": "peak/rep${i+1}/${basename}",
178178
"table": "Peak/Replicate ${i+1}/Blacklist-filtered narrowpeak (BigBed)"
179179
},
180+
"bfilt_peak_starch": {
181+
"path": "peak/rep${i+1}/${basename}",
182+
"table": "Peak/Replicate ${i+1}/Blacklist-filtered narrowpeak (starch)"
183+
},
180184
"bfilt_peak_hammock": {
181185
"path": "peak/rep${i+1}/${basename}",
182186
"table": "Peak/Replicate ${i+1}/Blacklist-filtered narrowpeak (hammock)"
@@ -199,6 +203,10 @@
199203
"path": "peak/rep${i+1}/pseudorep1/${basename}",
200204
"table": "Peak/Replicate ${i+1}/Pseudoreplicate 1/Blacklist-filtered narrowpeak (BigBed)"
201205
},
206+
"bfilt_peak_starch": {
207+
"path": "peak/rep${i+1}/pseudorep1/${basename}",
208+
"table": "Peak/Replicate ${i+1}/Pseudoreplicate 1/Blacklist-filtered narrowpeak (starch)"
209+
},
202210
"bfilt_peak_hammock": {
203211
"path": "peak/rep${i+1}/pseudorep1/${basename}",
204212
"table": "Peak/Replicate ${i+1}/Pseudoreplicate 1/Blacklist-filtered narrowpeak (hammock)"
@@ -221,6 +229,10 @@
221229
"path": "peak/rep${i+1}/pseudorep2/${basename}",
222230
"table": "Peak/Replicate ${i+1}/Pseudoreplicate 2/Blacklist-filtered narrowpeak (BigBed)"
223231
},
232+
"bfilt_peak_starch": {
233+
"path": "peak/rep${i+1}/pseudorep2/${basename}",
234+
"table": "Peak/Replicate ${i+1}/Pseudoreplicate 2/Blacklist-filtered narrowpeak (starch)"
235+
},
224236
"bfilt_peak_hammock": {
225237
"path": "peak/rep${i+1}/pseudorep2/${basename}",
226238
"table": "Peak/Replicate ${i+1}/Pseudoreplicate 2/Blacklist-filtered narrowpeak (hammock)"
@@ -247,6 +259,10 @@
247259
"path": "peak/pooled-rep/${basename}",
248260
"table": "Peak/Pooled replicate/Blacklist-filtered narrowpeak (BigBed)"
249261
},
262+
"bfilt_peak_starch": {
263+
"path": "peak/pooled-rep/${basename}",
264+
"table": "Peak/Pooled replicate/Blacklist-filtered narrowpeak (starch)"
265+
},
250266
"bfilt_peak_hammock": {
251267
"path": "peak/pooled-rep/${basename}",
252268
"table": "Peak/Pooled replicate/Blacklist-filtered narrowpeak (hammock)"
@@ -269,6 +285,10 @@
269285
"path": "peak/pooled-rep/pseudorep1/${basename}",
270286
"table": "Peak/Pooled replicate/Pseudoreplicate 1/Blacklist-filtered narrowpeak (BigBed)"
271287
},
288+
"bfilt_peak_starch": {
289+
"path": "peak/pooled-rep/pseudorep1/${basename}",
290+
"table": "Peak/Pooled replicate/Pseudoreplicate 1/Blacklist-filtered narrowpeak (starch)"
291+
},
272292
"bfilt_peak_hammock": {
273293
"path": "peak/pooled-rep/pseudorep1/${basename}",
274294
"table": "Peak/Pooled replicate/Pseudoreplicate 1/Blacklist-filtered narrowpeak (hammock)"
@@ -291,6 +311,10 @@
291311
"path": "peak/pooled-rep/pseudorep2/${basename}",
292312
"table": "Peak/Pooled replicate/Pseudoreplicate 2/Blacklist-filtered narrowpeak (BigBed)"
293313
},
314+
"bfilt_peak_starch": {
315+
"path": "peak/pooled-rep/pseudorep2/${basename}",
316+
"table": "Peak/Pooled replicate/Pseudoreplicate 2/Blacklist-filtered narrowpeak (starch)"
317+
},
294318
"bfilt_peak_hammock": {
295319
"path": "peak/pooled-rep/pseudorep2/${basename}",
296320
"table": "Peak/Pooled replicate/Pseudoreplicate 2/Blacklist-filtered narrowpeak (hammock)"
@@ -370,6 +394,10 @@
370394
"path": "peak/${basename.split('.')[0].replace('_vs_','_').replace('_','_vs_').replace('-','_vs_')}/${basename}",
371395
"table": "Peak/${basename.split('.')[0].replace('_vs_','_').replace('_',' vs. ').replace('-',' vs. ').capitalize()}/Blacklist-filtered IDR peak (BigBed)"
372396
},
397+
"bfilt_idr_peak_starch": {
398+
"path": "peak/${basename.split('.')[0].replace('_vs_','_').replace('_','_vs_').replace('-','_vs_')}/${basename}",
399+
"table": "Peak/${basename.split('.')[0].replace('_vs_','_').replace('_',' vs. ').replace('-',' vs. ').capitalize()}/Blacklist-filtered IDR peak (starch)"
400+
},
373401
"bfilt_idr_peak_hammock": {
374402
"path": "peak/${basename.split('.')[0].replace('_vs_','_').replace('_','_vs_').replace('-','_vs_')}/${basename}",
375403
"table": "Peak/${basename.split('.')[0].replace('_vs_','_').replace('_',' vs. ').replace('-',' vs. ').capitalize()}/Blacklist-filtered IDR peak (hammock)"
@@ -402,6 +430,10 @@
402430
"path": "peak/pooled-pseudorep1_vs_2/${basename}",
403431
"table": "Peak/Pooled pseudoreplicate 1 vs. 2/Blacklist-filtered IDR peak (BigBed)"
404432
},
433+
"bfilt_idr_peak_starch": {
434+
"path": "peak/pooled-pseudorep1_vs_2/${basename}",
435+
"table": "Peak/Pooled pseudoreplicate 1 vs. 2/Blacklist-filtered IDR peak (starch)"
436+
},
405437
"bfilt_idr_peak_hammock": {
406438
"path": "peak/pooled-pseudorep1_vs_2/${basename}",
407439
"table": "Peak/Pooled pseudoreplicate 1 vs. 2/Blacklist-filtered IDR peak (hammock)"
@@ -433,6 +465,10 @@
433465
"path": "peak/rep${i+1}-pseudorep1_vs_2/${basename}",
434466
"table": "Peak/Replicate ${i+1}/Pseudorep 1 vs. pseudorep 2/Blacklist-filtered IDR peak (BigBed)"
435467
},
468+
"bfilt_idr_peak_starch": {
469+
"path": "peak/rep${i+1}-pseudorep1_vs_2/${basename}",
470+
"table": "Peak/Replicate ${i+1}/Pseudorep 1 vs. pseudorep 2/Blacklist-filtered IDR peak (starch)"
471+
},
436472
"bfilt_idr_peak_hammock": {
437473
"path": "peak/rep${i+1}-pseudorep1_vs_2/${basename}",
438474
"table": "Peak/Replicate ${i+1}/Pseudorep 1 vs. pseudorep 2/Blacklist-filtered IDR peak (hammock)"
@@ -464,6 +500,10 @@
464500
"path": "peak/${basename.split('.')[0].replace('_vs_','_').replace('_','_vs_').replace('-','_vs_')}/${basename}",
465501
"table": "Peak/${basename.split('.')[0].replace('_vs_','_').replace('_',' vs. ').replace('-',' vs. ').capitalize()}/Blacklist-filtered overlap peak (BigBed)"
466502
},
503+
"bfilt_overlap_peak_starch": {
504+
"path": "peak/${basename.split('.')[0].replace('_vs_','_').replace('_','_vs_').replace('-','_vs_')}/${basename}",
505+
"table": "Peak/${basename.split('.')[0].replace('_vs_','_').replace('_',' vs. ').replace('-',' vs. ').capitalize()}/Blacklist-filtered overlap peak (starch)"
506+
},
467507
"bfilt_overlap_peak_hammock": {
468508
"path": "peak/${basename.split('.')[0].replace('_vs_','_').replace('_','_vs_').replace('-','_vs_')}/${basename}",
469509
"table": "Peak/${basename.split('.')[0].replace('_vs_','_').replace('_',' vs. ').replace('-',' vs. ').capitalize()}/Blacklist-filtered overlap peak (hammock)"
@@ -483,6 +523,10 @@
483523
"path": "peak/pooled-pseudorep1_vs_2/${basename}",
484524
"table": "Peak/Pooled pseudoreplicate 1 vs. 2/Blacklist-filtered overlap peak (BigBed)"
485525
},
526+
"bfilt_overlap_peak_starch": {
527+
"path": "peak/pooled-pseudorep1_vs_2/${basename}",
528+
"table": "Peak/Pooled pseudoreplicate 1 vs. 2/Blacklist-filtered overlap peak (starch)"
529+
},
486530
"bfilt_overlap_peak_hammock": {
487531
"path": "peak/pooled-pseudorep1_vs_2/${basename}",
488532
"table": "Peak/Pooled pseudoreplicate 1 vs. 2/Blacklist-filtered overlap peak (hammock)"
@@ -501,6 +545,10 @@
501545
"path": "peak/rep${i+1}-pseudorep1_vs_2/${basename}",
502546
"table": "Peak/Replicate ${i+1}/Pseudorep 1 vs. pseudorep 2/Blacklist-filtered overlap peak (BigBed)"
503547
},
548+
"bfilt_overlap_peak_starch": {
549+
"path": "peak/rep${i+1}-pseudorep1_vs_2/${basename}",
550+
"table": "Peak/Replicate ${i+1}/Pseudorep 1 vs. pseudorep 2/Blacklist-filtered overlap peak (starch)"
551+
},
504552
"bfilt_overlap_peak_hammock": {
505553
"path": "peak/rep${i+1}-pseudorep1_vs_2/${basename}",
506554
"table": "Peak/Replicate ${i+1}/Pseudorep 1 vs. pseudorep 2/Blacklist-filtered overlap peak (hammock)"
@@ -521,6 +569,10 @@
521569
"table": "Peak/IDR reproducibility/Optimal peak (BigBed)",
522570
"ucsc_track": "track type=bigBed name=\"Optimal IDR peak\" priority=100 color=0,0,255 visibility=full"
523571
},
572+
"optimal_peak_starch": {
573+
"path": "peak/idr_reproducibility/${basename}",
574+
"table": "Peak/IDR reproducibility/Optimal peak (starch)"
575+
},
524576
"optimal_peak_hammock": {
525577
"path": "peak/idr_reproducibility/${basename}",
526578
"table": "Peak/IDR reproducibility/Optimal peak (hammock)"
@@ -534,6 +586,10 @@
534586
"table": "Peak/IDR reproducibility/Conservative peak (BigBed)",
535587
"ucsc_track": "track type=bigBed name=\"Conservative IDR peak\" priority=100 color=0,0,255 visibility=full"
536588
},
589+
"conservative_peak_starch": {
590+
"path": "peak/idr_reproducibility/${basename}",
591+
"table": "Peak/IDR reproducibility/Conservative peak (starch)"
592+
},
537593
"conservative_peak_hammock": {
538594
"path": "peak/idr_reproducibility/${basename}",
539595
"table": "Peak/IDR reproducibility/Conservative peak (hammock)"
@@ -568,6 +624,10 @@
568624
"table": "Peak/Overlap reproducibility/Conservative peak (BigBed)",
569625
"ucsc_track": "track type=bigBed name=\"Conservative overlap peak\" priority=100 color=0,0,255 visibility=full"
570626
},
627+
"conservative_peak_starch": {
628+
"path": "peak/overlap_reproducibility/${basename}",
629+
"table": "Peak/Overlap reproducibility/Conservative peak (starch)"
630+
},
571631
"conservative_peak_hammock": {
572632
"path": "peak/overlap_reproducibility/${basename}",
573633
"table": "Peak/Overlap reproducibility/Conservative peak (hammock)"

‎chip.wdl‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
11
version 1.0
22

33
workflow chip {
4-
String pipeline_ver = 'v1.6.0'
4+
String pipeline_ver = 'v1.6.1'
55

66
meta {
77
author: 'Jin wook Lee (leepc12@gmail.com) at ENCODE-DCC'
88
description: 'ENCODE TF/Histone ChIP-Seq pipeline'
99
specification_document: 'https://docs.google.com/document/d/1lG_Rd7fnYgRpSIqrIfuVlAz2dW1VaSQThzk836Db99c/edit?usp=sharing'
1010

11-
caper_docker: 'encodedcc/chip-seq-pipeline:v1.6.0'
12-
caper_singularity: 'docker://encodedcc/chip-seq-pipeline:v1.6.0'
13-
croo_out_def: 'https://storage.googleapis.com/encode-pipeline-output-definition/chip.croo.v4.json'
11+
caper_docker: 'encodedcc/chip-seq-pipeline:v1.6.1'
12+
caper_singularity: 'docker://encodedcc/chip-seq-pipeline:v1.6.1'
13+
croo_out_def: 'https://storage.googleapis.com/encode-pipeline-output-definition/chip.croo.v5.json'
1414

1515
parameter_group: {
1616
pipeline_metadata: {
@@ -2507,6 +2507,7 @@ task call_peak {
25072507
# generated by post_call_peak py
25082508
File bfilt_peak = glob('*.bfilt.'+peak_type+'.gz')[0]
25092509
File bfilt_peak_bb = glob('*.bfilt.'+peak_type+'.bb')[0]
2510+
File bfilt_peak_starch = glob('*.bfilt.'+peak_type+'.starch')[0]
25102511
File bfilt_peak_hammock = glob('*.bfilt.'+peak_type+'.hammock.gz*')[0]
25112512
File bfilt_peak_hammock_tbi = glob('*.bfilt.'+peak_type+'.hammock.gz*')[1]
25122513
File frip_qc = glob('*.frip.qc')[0]
@@ -2599,6 +2600,7 @@ task idr {
25992600
File idr_peak = glob('*[!.][!b][!f][!i][!l][!t].'+peak_type+'.gz')[0]
26002601
File bfilt_idr_peak = glob('*.bfilt.'+peak_type+'.gz')[0]
26012602
File bfilt_idr_peak_bb = glob('*.bfilt.'+peak_type+'.bb')[0]
2603+
File bfilt_idr_peak_starch = glob('*.bfilt.'+peak_type+'.starch')[0]
26022604
File bfilt_idr_peak_hammock = glob('*.bfilt.'+peak_type+'.hammock.gz*')[0]
26032605
File bfilt_idr_peak_hammock_tbi = glob('*.bfilt.'+peak_type+'.hammock.gz*')[1]
26042606
File idr_plot = glob('*.txt.png')[0]
@@ -2648,6 +2650,7 @@ task overlap {
26482650
File overlap_peak = glob('*[!.][!b][!f][!i][!l][!t].'+peak_type+'.gz')[0]
26492651
File bfilt_overlap_peak = glob('*.bfilt.'+peak_type+'.gz')[0]
26502652
File bfilt_overlap_peak_bb = glob('*.bfilt.'+peak_type+'.bb')[0]
2653+
File bfilt_overlap_peak_starch = glob('*.bfilt.'+peak_type+'.starch')[0]
26512654
File bfilt_overlap_peak_hammock = glob('*.bfilt.'+peak_type+'.hammock.gz*')[0]
26522655
File bfilt_overlap_peak_hammock_tbi = glob('*.bfilt.'+peak_type+'.hammock.gz*')[1]
26532656
File frip_qc = if defined(ta) then glob('*.frip.qc')[0] else glob('null')[0]
@@ -2686,10 +2689,12 @@ task reproducibility {
26862689
output {
26872690
File optimal_peak = glob('*optimal_peak.*.gz')[0]
26882691
File optimal_peak_bb = glob('*optimal_peak.*.bb')[0]
2692+
File optimal_peak_starch = glob('*optimal_peak.*.starch')[0]
26892693
File optimal_peak_hammock = glob('*optimal_peak.*.hammock.gz*')[0]
26902694
File optimal_peak_hammock_tbi = glob('*optimal_peak.*.hammock.gz*')[1]
26912695
File conservative_peak = glob('*conservative_peak.*.gz')[0]
26922696
File conservative_peak_bb = glob('*conservative_peak.*.bb')[0]
2697+
File conservative_peak_starch = glob('*conservative_peak.*.starch')[0]
26932698
File conservative_peak_hammock = glob('*conservative_peak.*.hammock.gz*')[0]
26942699
File conservative_peak_hammock_tbi = glob('*conservative_peak.*.hammock.gz*')[1]
26952700
File reproducibility_qc = glob('*reproducibility.qc')[0]

‎dev/docker_image/Dockerfile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,10 @@ RUN wget http://www.usadellab.org/cms/uploads/supplementary/Trimmomatic/Trimmoma
111111
# Install pytest for testing environment
112112
RUN pip3 install --no-cache-dir pytest
113113

114+
# Install bedops 2.4.39
115+
RUN mkdir bedops_2.4.39 && cd bedops_2.4.39 && wget https://github.com/bedops/bedops/releases/download/v2.4.39/bedops_linux_x86_64-v2.4.39.tar.bz2 && tar -xvjf bedops_linux_x86_64-v2.4.39.tar.bz2 && rm -f bedops_linux_x86_64-v2.4.39.tar.bz2
116+
ENV PATH="/software/bedops_2.4.39/bin:${PATH}"
117+
114118
# Prevent conflict with locally installed python outside of singularity container
115119
ENV PYTHONNOUSERSITE=True
116120

0 commit comments

Comments
 (0)