Merge pull request #69 from cancerit/feature/docker

keiranmraine · web-flow · commit ac49f01c93b7 · 2019-05-15T13:49:49.000+01:00
Feature/docker
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,10 @@
+/LICENCE
+/prerelease.sh
+/README.md
+/INSTALL
+/LICENCE
+/CHANGES.md
+/.gitignore
+/.git
+/perl/docs
+/perl/docs.tar.gz
diff --git a/.travis.yml b/.travis.yml
@@ -2,58 +2,30 @@ notifications:
   slack: wtsi-cgpit:ptUMR1tkNyZJYd9TpGoss8WR
   email: false
 
-env:
- - CC=gcc
+sudo: false
 
-addons:
- apt:
-  update: true
-  packages:
-   - build-essential
-   - autoconf
-   - bsdtar
-   - time
-   - curl
-   - libcurl4-openssl-dev
-   - nettle-dev
-   - zlib1g-dev
-   - libncurses5-dev
-   - libpstreams-dev
-   - unzip
-   - libpng12-dev
-   - libexpat1-dev
-   - libgoogle-perftools-dev
-   - lsof
-   - libbz2-dev
-   - liblzma-dev
-   - libgnutls-dev
-   - libtasn1-6-dev
-   - p11-kit
-   - libxml2-dev
-   - libgd-dev
-   - psmisc
-   - libdb-dev
-
-install: true
-
-language: perl
-
-perl:
- - "5.22-shrplib"
+services:
+  - docker
 
 script:
  - set -e
- - export PATH=$HOME/install/bin:$HOME/install/biobambam2/bin:$PATH
- - git clone --depth 1 --single-branch --branch develop https://github.com/cancerit/cgpBigWig.git
- - cd cgpBigWig
- - ./setup.sh $HOME/PCAP-opt
- - cd ../
- - git clone --depth 1 --single-branch --branch develop https://github.com/cancerit/PCAP-core.git
- - cd PCAP-core
- - ./setup.sh $HOME/PCAP-opt
- - cd ../
- - git clone --depth 1 --single-branch --branch dev https://github.com/cancerit/cgpVcf.git
- - cd cgpVcf
- - ./setup.sh $HOME/PCAP-opt
- - cd ../
- - ./setup.sh $HOME/PCAP-opt
+ - echo 'Build and check docker image'
+ - docker build -t cgppindel .
+ - docker images | grep -c cgppindel
+ - echo 'Verify program(s) from each inherited base image - dockstore-cgpbigwig'
+ - docker run -t --rm cgppindel bwjoin --version
+ - echo 'Verify program(s) from each inherited base image - dockstore-cgpmap'
+ - docker run -t --rm cgppindel bwa_mem.pl -version
+ - docker run -t --rm cgppindel bammarkduplicates2 --version
+ - docker run -t --rm cgppindel samtools --version
+ - docker run -t --rm cgppindel bash -c 'bwa 2>&1 | grep Version'
+ - echo 'Verify program(s) from this repo'
+ - docker run -t --rm cgppindel pindel.pl --version
+ - docker run -t --rm cgppindel pindel_input_gen.pl --version
+ - docker run -t --rm cgppindel pindel_2_combined_vcf.pl --version
+ - docker run -t --rm cgppindel FlagVcf.pl --version
+ - docker run -t --rm cgppindel pindel_merge_vcf_bam.pl --version
+ - docker run -t --rm cgppindel pindel_np_from_vcf.pl --version
+ - docker run -t --rm cgppindel pindel_germ_bed.pl --version
+ - docker run -t --rm cgppindel which pindel
+ - docker run -t --rm cgppindel which filter_pindel_reads
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,9 @@
 # CHANGES
 
+## NEXT
+
+* Added Dockerfile and docker documentation
+
 ## 3.2.0
 
 * Tabix search for high depth/excluded regions now performed in memory using IntervalTrees
@@ -153,15 +157,4 @@ Comparing sites in VCF files...
 Found 15321 SNPs common to both files.
 Found 0 SNPs only in main file.
 Found 0 SNPs only in second file.
-After filtering, kept 16309 out of a possible 16309 Sites
-Run Time = 6.00 seconds
-```
-
-## 2.0.4
-
-* Permits empty results files
-
-## 2.0.0
-
-* Migrates all Tabix and Bio::DB::Sam to Bio::DB::HTS::Tabix and Bio::DB::Sam
-* Cleans up install
+After
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,81 @@
+FROM  quay.io/wtsicgp/dockstore-cgpmap:3.1.4 as builder
+
+USER  root
+
+# ALL tool versions used by opt-build.sh
+ENV VER_CGPVCF="v2.2.1"
+ENV VER_VCFTOOLS="0.1.16"
+
+RUN apt-get -yq update
+RUN apt-get install -yq --no-install-recommends \
+locales \
+g++ \
+make \
+gcc \
+pkg-config \
+zlib1g-dev
+
+RUN locale-gen en_US.UTF-8
+RUN update-locale LANG=en_US.UTF-8
+
+ENV OPT /opt/wtsi-cgp
+ENV PATH $OPT/bin:$OPT/biobambam2/bin:$PATH
+ENV PERL5LIB $OPT/lib/perl5
+ENV LD_LIBRARY_PATH $OPT/lib
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+
+# build tools from other repos
+ADD build/opt-build.sh build/
+RUN bash build/opt-build.sh $OPT
+
+# build the tools in this repo, separate to reduce build time on errors
+COPY . .
+RUN bash build/opt-build-local.sh $OPT
+
+FROM ubuntu:16.04
+
+LABEL maintainer="cgphelp@sanger.ac.uk" \
+      uk.ac.sanger.cgp="Cancer, Ageing and Somatic Mutation, Wellcome Trust Sanger Institute" \
+      version="1.0.0" \
+      description="cgpPindel docker"
+
+RUN apt-get -yq update
+RUN apt-get install -yq --no-install-recommends \
+apt-transport-https \
+locales \
+curl \
+ca-certificates \
+libperlio-gzip-perl \
+bzip2 \
+psmisc \
+time \
+zlib1g \
+liblzma5 \
+libncurses5 \
+p11-kit \
+unattended-upgrades && \
+unattended-upgrade -d -v && \
+apt-get remove -yq unattended-upgrades && \
+apt-get autoremove -yq
+
+RUN locale-gen en_US.UTF-8
+RUN update-locale LANG=en_US.UTF-8
+
+ENV OPT /opt/wtsi-cgp
+ENV PATH $OPT/bin:$OPT/biobambam2/bin:$PATH
+ENV PERL5LIB $OPT/lib/perl5
+ENV LD_LIBRARY_PATH $OPT/lib
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+
+RUN mkdir -p $OPT
+COPY --from=builder $OPT $OPT
+
+## USER CONFIGURATION
+RUN adduser --disabled-password --gecos '' ubuntu && chsh -s /bin/bash && mkdir -p /home/ubuntu
+
+USER    ubuntu
+WORKDIR /home/ubuntu
+
+CMD ["/bin/bash"]
diff --git a/README.md b/README.md
@@ -2,6 +2,8 @@
 
 cgpPindel contains the Cancer Genome Projects workflow for [Pindel][pindel-core].
 
+[![Quay Badge][quay-status]][quay-repo]
+
 | Master                                        | Develop                                         |
 | --------------------------------------------- | ----------------------------------------------- |
 | [![Master Badge][travis-master]][travis-base] | [![Develop Badge][travis-develop]][travis-base] |
@@ -18,25 +20,30 @@ Contents:
 
 <!-- TOC depthFrom:2 depthTo:6 withLinks:1 updateOnSave:1 orderedList:0 -->
 
-- [Docker, Singularity and Dockstore](#docker-singularity-and-dockstore)
-- [Dependencies/Install](#dependenciesinstall)
-- [Creating a release](#creating-a-release)
-	- [Preparation](#preparation)
-	- [Cutting the release](#cutting-the-release)
-- [LICENCE](#licence)
+- [cgpPindel](#cgppindel)
+  - [Docker, Singularity and Dockstore](#docker-singularity-and-dockstore)
+  - [Dependencies/Install](#dependenciesinstall)
+  - [Creating a release](#creating-a-release)
+    - [Preparation](#preparation)
+    - [Release process](#release-process)
+      - [Code changes](#code-changes)
+      - [Docker image](#docker-image)
+      - [Cutting the release](#cutting-the-release)
+  - [LICENCE](#licence)
 
 <!-- /TOC -->
 
 ## Docker, Singularity and Dockstore
 
 There are pre-built images containing this codebase on quay.io.
 
+* [cgpPindel][cgpPindel-git]: Contained within this repository - contains the cgpPindel package
 * [dockstore-cgpwxs][ds-cgpwxs-git]: Contains tools specific to WXS analysis.
 * [dockstore-cgpwgs][ds-cgpwgs-git]: Contains additional tools for WGS analysis.
 
 These were primarily designed for use with dockstore.org but can be used as normal containers.
 
-The docker images are know to work correctly after import into a singularity image.
+The docker images are known to work correctly after import into a singularity image.
 
 ## Dependencies/Install
 
@@ -66,15 +73,34 @@ Please be aware that this expects basic C compilation libraries and tools to be
 * Commit/push all relevant changes.
 * Pull a clean version of the repo and use this for the following steps.
 
-### Cutting the release
+### Release process
+
+This project is maintained using HubFlow.
+
+#### Code changes
+
+1. Make appropriate changes
+2. Update `perl/lib/Sanger/CGP/Pindel.pm` to the correct version (adding rc/beta to end if applicable).
+3. Update `CHANGES.md` to show major items.
+4. Run `./prerelease.sh`
+5. Check all tests and coverage reports are acceptable.
+6. Commit the updated docs and updated module/version.
+7. Push commits.
+
+#### Docker image
 
-1. Update `perl/lib/Sanger/CGP/Pindel.pm` to the correct version (adding rc/beta to end if applicable).
-1. Update `CHANGES.md` to show major items.
-1. Run `./prerelease.sh`
-1. Check all tests and coverage reports are acceptable.
-1. Commit the updated docs and updated module/version.
-1. Push commits.
 1. Use the GitHub tools to draft a release.
+2. Build image locally
+3. Run example inputs and verify any changes are acceptable
+4. Bump version in `Dockerfile`
+5. Push changes
+
+#### Cutting the release
+
+1. Check state on Travis
+2. Generate the release (add notes to GitHub)
+3. Confirm that image has been built on [quay.io][quay-builds]
+4. Update the [dockstore][dockstore-cgpPindel] entry, see [their docs][dockstore-get-started].
 
 ## LICENCE
 
@@ -114,9 +140,18 @@ identical to a statement that reads ‘Copyright (c) 2005, 2006, 2007, 2008,
 [pcap-core-rel]: https://github.com/cancerit/PCAP-core/releases
 [ds-cgpwxs-git]: https://github.com/cancerit/dockstore-cgpwxs
 [ds-cgpwgs-git]: https://github.com/cancerit/dockstore-cgpwgs
+[cgpPindel-git]: https://github.com/cancerit/cgpPindel
 [pindel-core]: http://gmt.genome.wustl.edu/pindel/current
 
 <!-- Travis -->
 [travis-base]: https://travis-ci.org/cancerit/cgpPindel
 [travis-master]: https://travis-ci.org/cancerit/cgpPindel.svg?branch=master
 [travis-develop]: https://travis-ci.org/cancerit/cgpPindel.svg?branch=dev
+
+<!-- Quay.io -->
+[quay-status]: https://quay.io/repository/wtsicgp/cgpPindel/status
+[quay-repo]: https://quay.io/repository/wtsicgp/cgpPindel
+[quay-builds]: https://quay.io/repository/wtsicgp/cgpPindel?tab=builds
+
+<!-- Dockstore -->
+[dockstore-cgpPindel]: https://dockstore.org/containers/quay.io/wtsicgp/cgpPindel
diff --git a/build/opt-build-local.sh b/build/opt-build-local.sh
@@ -0,0 +1,65 @@
+#! /bin/bash
+
+set -xe
+
+if [[ -z "${TMPDIR}" ]]; then
+  TMPDIR=/tmp
+fi
+
+set -u
+
+if [ "$#" -lt "1" ] ; then
+  echo "Please provide an installation path such as /opt/ICGC"
+  exit 1
+fi
+
+# get path to this script
+SCRIPT_PATH=`dirname $0`;
+SCRIPT_PATH=`(cd $SCRIPT_PATH && pwd)`
+
+# get the location to install to
+INST_PATH=$1
+mkdir -p $1
+INST_PATH=`(cd $1 && pwd)`
+echo $INST_PATH
+
+# get current directory
+INIT_DIR=`pwd`
+
+CPU=`grep -c ^processor /proc/cpuinfo`
+if [ $? -eq 0 ]; then
+  if [ "$CPU" -gt "6" ]; then
+    CPU=6
+  fi
+else
+  CPU=1
+fi
+echo "Max compilation CPUs set to $CPU"
+
+SETUP_DIR=$INIT_DIR/install_tmp
+mkdir -p $SETUP_DIR/distro # don't delete the actual distro directory until the very end
+mkdir -p $INST_PATH/bin
+cd $SETUP_DIR
+
+# make sure tools installed can see the install loc of libraries
+set +u
+export LD_LIBRARY_PATH=`echo $INST_PATH/lib:$LD_LIBRARY_PATH | perl -pe 's/:\$//;'`
+export PATH=`echo $INST_PATH/bin:$PATH | perl -pe 's/:\$//;'`
+export MANPATH=`echo $INST_PATH/man:$INST_PATH/share/man:$MANPATH | perl -pe 's/:\$//;'`
+export PERL5LIB=`echo $INST_PATH/lib/perl5:$PERL5LIB | perl -pe 's/:\$//;'`
+set -u
+
+## cgpPindel - should be the build root
+if [ ! -e $SETUP_DIR/cgpPindel.success ]; then
+  cd $INIT_DIR
+  if [ ! -e $SETUP_DIR/cgpPindel_c.success ]; then
+    g++ -O3 -o $INST_PATH/bin/pindel c++/pindel.cpp
+    g++ -O3 -o $INST_PATH/bin/filter_pindel_reads c++/filter_pindel_reads.cpp
+    touch $SETUP_DIR/cgpPindel_c.success
+  fi
+  cd perl
+  cpanm --no-interactive --notest --mirror http://cpan.metacpan.org --notest -l $INST_PATH --installdeps .
+  cpanm -v --no-interactive --mirror http://cpan.metacpan.org -l $INST_PATH .
+  cd $SETUP_DIR
+  touch $SETUP_DIR/cgpPindel.success
+fi
diff --git a/build/opt-build.sh b/build/opt-build.sh
