Merge branch 'fix/CircleTests' of github.com:oesteban/nipype into fix/CircleTests

Author: oesteban

.circle/codecov.sh

@@ -9,15 +9,17 @@ set -e         # Exit immediately if a command exits with a non-zero status.
 set -u         # Treat unset variables as an error when substituting.
 set -x         # Print command traces before executing command.
 
-mkdir -p ${CIRCLE_TEST_REPORTS}/unittests ${CIRCLE_TEST_REPORTS}/smoketest
-cp ${SCRATCH}/coverage*.xml ${CIRCLE_TEST_REPORTS}/unittests/
-cp ${SCRATCH}/smoketests*.xml ${CIRCLE_TEST_REPORTS}/smoketest/
-
 # Send coverage data to codecov.io
 curl -so codecov.io https://codecov.io/bash
 chmod 755 codecov.io
 
-find "${CIRCLE_TEST_REPORTS}/unittests" -name '*.xml' -print0 | \
+find "${SCRATCH}/" -name 'coverage*.xml' -maxdepth 1 -print0 | \
   xargs -0 -I file ./codecov.io -f file -t "${CODECOV_TOKEN}" -F unittests
-find "${CIRCLE_TEST_REPORTS}/smoketest" -name '*.xml' -print0 | \
+find "${SCRATCH}/" -name 'smoketest*.xml' -maxdepth 1 -print0 | \
   xargs -0 -I file ./codecov.io -f file -t "${CODECOV_TOKEN}" -F smoketests
+
+# Place test and coverage in the tests folder
+mkdir -p ${CIRCLE_TEST_REPORTS}/tests/
+cp ${SCRATCH}/pytest*.xml ${CIRCLE_TEST_REPORTS}/tests/ || true
+cp ${SCRATCH}/coverage*.xml ${CIRCLE_TEST_REPORTS}/tests/ || true
+cp ${SCRATCH}/smoketest*.xml ${CIRCLE_TEST_REPORTS}/tests/ || true

Dockerfile

@@ -30,157 +30,37 @@
 #
 # Based on https://github.com/poldracklab/fmriprep/blob/9c92a3de9112f8ef1655b876de060a2ad336ffb0/Dockerfile
 #
-FROM ubuntu:xenial-20161213
+FROM nipype/base:latest
 MAINTAINER The nipype developers https://github.com/nipy/nipype
 
-ARG DEBIAN_FRONTEND=noninteractive
-
-# Pre-cache neurodebian key
-COPY docker/files/neurodebian.gpg /root/.neurodebian.gpg
-
-# Prepare environment
-RUN apt-key add /root/.neurodebian.gpg && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends curl bzip2 ca-certificates xvfb && \
-    curl -sSL http://neuro.debian.net/lists/xenial.us-ca.full >> /etc/apt/sources.list.d/neurodebian.sources.list && \
-    apt-key adv --refresh-keys --keyserver hkp://ha.pool.sks-keyservers.net 0xA5D32F012649A5A9 || true; \
-    apt-get update
-
-# Installing freesurfer
-RUN curl -sSL https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.0/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.0.tar.gz | tar zxv -C /opt \
-    --exclude='freesurfer/trctrain' \
-    --exclude='freesurfer/subjects/fsaverage_sym' \
-    --exclude='freesurfer/subjects/fsaverage3' \
-    --exclude='freesurfer/subjects/fsaverage4' \
-    --exclude='freesurfer/subjects/fsaverage5' \
-    --exclude='freesurfer/subjects/fsaverage6' \
-    --exclude='freesurfer/subjects/cvs_avg35' \
-    --exclude='freesurfer/subjects/cvs_avg35_inMNI152' \
-    --exclude='freesurfer/subjects/bert' \
-    --exclude='freesurfer/subjects/V1_average' \
-    --exclude='freesurfer/average/mult-comp-cor' \
-    --exclude='freesurfer/lib/cuda' \
-    --exclude='freesurfer/lib/qt'
-
-ENV FSL_DIR=/usr/share/fsl/5.0 \
-    OS=Linux \
-    FS_OVERRIDE=0 \
-    FIX_VERTEX_AREA= \
-    FSF_OUTPUT_FORMAT=nii.gz \
-    FREESURFER_HOME=/opt/freesurfer
-ENV SUBJECTS_DIR=$FREESURFER_HOME/subjects \
-    FUNCTIONALS_DIR=$FREESURFER_HOME/sessions \
-    MNI_DIR=$FREESURFER_HOME/mni \
-    LOCAL_DIR=$FREESURFER_HOME/local \
-    FSFAST_HOME=$FREESURFER_HOME/fsfast \
-    MINC_BIN_DIR=$FREESURFER_HOME/mni/bin \
-    MINC_LIB_DIR=$FREESURFER_HOME/mni/lib \
-    MNI_DATAPATH=$FREESURFER_HOME/mni/data \
-    FMRI_ANALYSIS_DIR=$FREESURFER_HOME/fsfast
-ENV PERL5LIB=$MINC_LIB_DIR/perl5/5.8.5 \
-    MNI_PERL5LIB=$MINC_LIB_DIR/perl5/5.8.5 \
-    PATH=$FREESURFER_HOME/bin:$FSFAST_HOME/bin:$FREESURFER_HOME/tktools:$MINC_BIN_DIR:$PATH
-RUN echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh
-
-# Installing Neurodebian packages (FSL, AFNI, git)
-RUN apt-get install -y --no-install-recommends \
-                    fsl-core=5.0.9-1~nd+1+nd16.04+1 \
-                    fsl-mni152-templates=5.0.7-2 \
-                    afni=16.2.07~dfsg.1-2~nd16.04+1
-
-ENV FSLDIR=/usr/share/fsl/5.0 \
-    FSLOUTPUTTYPE=NIFTI_GZ \
-    FSLMULTIFILEQUIT=TRUE \
-    POSSUMDIR=/usr/share/fsl/5.0 \
-    LD_LIBRARY_PATH=/usr/lib/fsl/5.0:$LD_LIBRARY_PATH \
-    FSLTCLSH=/usr/bin/tclsh \
-    FSLWISH=/usr/bin/wish \
-    AFNI_MODELPATH=/usr/lib/afni/models \
-    AFNI_IMSAVE_WARNINGS=NO \
-    AFNI_TTATLAS_DATASET=/usr/share/afni/atlases \
-    AFNI_PLUGINPATH=/usr/lib/afni/plugins \
-    PATH=/usr/lib/fsl/5.0:/usr/lib/afni/bin:$PATH
-
-# Installing and setting up ANTs
-RUN mkdir -p /opt/ants && \
-    curl -sSL "https://github.com/stnava/ANTs/releases/download/v2.1.0/Linux_Ubuntu14.04.tar.bz2" \
-    | tar -xjC /opt/ants --strip-components 1
-
-ENV ANTSPATH=/opt/ants \
-    PATH=$ANTSPATH:$PATH
-
-# Installing and setting up c3d
-RUN mkdir -p /opt/c3d && \
-    curl -sSL "http://downloads.sourceforge.net/project/c3d/c3d/1.0.0/c3d-1.0.0-Linux-x86_64.tar.gz" \
-    | tar -xzC /opt/c3d --strip-components 1
-
-ENV C3DPATH=/opt/c3d/ \
-    PATH=$C3DPATH/bin:$PATH
-
-# Install some other required tools
-RUN apt-get install -y --no-install-recommends \
-                    git=1:2.7.4-0ubuntu1 \
-                    graphviz=2.38.0-12ubuntu2 \
-                    unzip \
-                    apt-utils \
-                    fusefat \
-                    make \
-                    ruby=1:2.3.0+1 && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-# Install fake-S3
-ENV GEM_HOME /usr/lib/ruby/gems/2.3
-ENV BUNDLE_PATH="$GEM_HOME" \
-    BUNDLE_BIN="$GEM_HOME/bin" \
-    BUNDLE_SILENCE_ROOT_WARNING=1 \
-    BUNDLE_APP_CONFIG="$GEM_HOME"
-ENV PATH $BUNDLE_BIN:$PATH
-RUN mkdir -p "$GEM_HOME" "$BUNDLE_BIN" && \
-    chmod 777 "$GEM_HOME" "$BUNDLE_BIN"
-
-RUN gem install fakes3
-
-# Install Matlab MCR: from the good old install_spm_mcr.sh of @chrisfilo
-WORKDIR /opt
-RUN echo "destinationFolder=/opt/mcr" > mcr_options.txt && \
-    echo "agreeToLicense=yes" >> mcr_options.txt && \
-    echo "outputFile=/tmp/matlabinstall_log" >> mcr_options.txt && \
-    echo "mode=silent" >> mcr_options.txt && \
-    mkdir -p matlab_installer && \
-    curl -sSL http://www.mathworks.com/supportfiles/downloads/R2015a/deployment_files/R2015a/installers/glnxa64/MCR_R2015a_glnxa64_installer.zip \
-         -o matlab_installer/installer.zip && \
-    unzip matlab_installer/installer.zip -d matlab_installer/ && \
-    matlab_installer/install -inputFile mcr_options.txt && \
-    rm -rf matlab_installer mcr_options.txt
-
-# Install SPM
-RUN curl -sSL http://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/dev/spm12_r6472_Linux_R2015a.zip -o spm12.zip && \
-    unzip spm12.zip && \
-    rm -rf spm12.zip
-
-ENV MATLABCMD="/opt/mcr/v85/toolbox/matlab" \
-    SPMMCRCMD="/opt/spm12/run_spm12.sh /opt/mcr/v85/ script" \
-    FORCE_SPMMCR=1
-
+ARG BUILD_DATE
+ARG VCS_REF
+ARG VERSION
+ARG PYTHON_VERSION_MAJOR=3
+ARG PYTHON_VERSION_MINOR=5
 
 # Installing and setting up miniconda
-RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-4.2.12-Linux-x86_64.sh && \
-    bash Miniconda3-4.2.12-Linux-x86_64.sh -b -p /usr/local/miniconda && \
-    rm Miniconda3-4.2.12-Linux-x86_64.sh
+RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION_MAJOR}-4.2.12-Linux-x86_64.sh && \
+    bash Miniconda${PYTHON_VERSION_MAJOR}-4.2.12-Linux-x86_64.sh -b -p /usr/local/miniconda && \
+    rm Miniconda${PYTHON_VERSION_MAJOR}-4.2.12-Linux-x86_64.sh
 
 ENV PATH=/usr/local/miniconda/bin:$PATH \
     LANG=C.UTF-8 \
     LC_ALL=C.UTF-8 \
-    ACCEPT_INTEL_PYTHON_EULA=yes
+    ACCEPT_INTEL_PYTHON_EULA=yes \
+    MKL_NUM_THREADS=1 \
+    OMP_NUM_THREADS=1
+# MKL/OMP_NUM_THREADS: unless otherwise specified, each process should
+# only use one thread - nipype will handle parallelization
 
 # Installing precomputed python packages
 RUN conda config --add channels conda-forge --add channels intel && \
     chmod +x /usr/local/miniconda/bin/* && \
     conda config --set always_yes yes --set changeps1 no && \
     conda update -q conda && \
     chmod +x /usr/local/miniconda/bin/*; sync && \
-    conda install -y mkl=2017.0.1 \
+    conda install -y python=${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR} \
+                     mkl=2017.0.1 \
                      numpy=1.11.2 \
                      scipy=0.18.1 \
                      scikit-learn=0.17.1 \
@@ -194,14 +74,9 @@ RUN conda config --add channels conda-forge --add channels intel && \
     find /usr/local/miniconda/ -exec chmod 775 {} +
 
 # matplotlib cleanups: set default backend, precaching fonts
-RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python3.5/site-packages/matplotlib/mpl-data/matplotlibrc && \
+RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}/site-packages/matplotlib/mpl-data/matplotlibrc && \
     python -c "from matplotlib import font_manager"
 
-# Unless otherwise specified each process should only use one thread - nipype
-# will handle parallelization
-ENV MKL_NUM_THREADS=1 \
-    OMP_NUM_THREADS=1
-
 # Installing dev requirements (packages that are not in pypi)
 WORKDIR /root/
 COPY requirements.txt requirements.txt
@@ -214,11 +89,18 @@ RUN cd /root/src/nipype && \
     pip install -e .[all] && \
     rm -rf ~/.cache/pip
 
+# Install CI scripts
+COPY docker/files/run_* /usr/bin/
+RUN chmod +x /usr/bin/run_*
+
+# Replace imglob with a Python3 compatible version
+COPY nipype/external/fsl_imglob.py /usr/bin/fsl_imglob.py
+RUN rm -rf ${FSLDIR}/bin/imglob && \
+    chmod +x /usr/bin/fsl_imglob.py && \
+    ln -s /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob
+
 WORKDIR /root/
 
-ARG BUILD_DATE
-ARG VCS_REF
-ARG VERSION
 LABEL org.label-schema.build-date=$BUILD_DATE \
       org.label-schema.name="NIPYPE" \
       org.label-schema.description="NIPYPE - Neuroimaging in Python: Pipelines and Interfaces" \

README.rst

@@ -78,8 +78,8 @@ Support and Communication
 -------------------------
 
 If you have a problem or would like to ask a question about how to do something in Nipype please open an issue to
-`NeuroStars.org <http://neurostars.org>`_ with a *nipype* tag. `NeuroStars.org <http://neurostars.org>`_  is a 
-platform similar to StackOverflow but dedicated to neuroinformatics. 
+`NeuroStars.org <http://neurostars.org>`_ with a *nipype* tag. `NeuroStars.org <http://neurostars.org>`_  is a
+platform similar to StackOverflow but dedicated to neuroinformatics.
 
 To participate in the Nipype development related discussions please use the following mailing list::
 

circle.yml

@@ -33,13 +33,13 @@ dependencies:
     - if [[ ! -d ~/examples/feeds ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O fsl-5.0.9-feeds.tar.gz "${DATA_NIPYPE_FSL_FEEDS}" && tar xzf fsl-5.0.9-feeds.tar.gz -C ~/examples/; fi
     - docker images
     - sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py
-    - e=1 && for i in {1..5}; do docker build --rm=false -t nipype/nipype:latest --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
+    - e=1 && for i in {1..5}; do docker build -f docker/Dockerfile_base --rm=false -t nipype/base:latest --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
         timeout: 21600
-    - e=1 && for i in {1..5}; do docker build --rm=false -f docker/Dockerfile_py27 -t nipype/nipype_test:py27 . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
-        timeout: 1600
-    - e=1 && for i in {1..5}; do docker build --rm=false -f docker/Dockerfile_py35 -t nipype/nipype_test:py35 . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
-        timeout: 1600
-    - docker save -o $HOME/docker/cache.tar nipype/nipype:latest nipype/nipype_test:py27 nipype/nipype_test:py35 :
+    - e=1 && for i in {1..5}; do docker build --rm=false -t nipype/nipype:latest -t nipype/nipype:py35 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
+        timeout: 6000
+    - e=1 && for i in {1..5}; do docker build --rm=false -t nipype/nipype:py27 --build-arg PYTHON_VERSION_MAJOR=2 --build-arg PYTHON_VERSION_MINOR=7 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
+        timeout: 6000
+    - docker save -o $HOME/docker/cache.tar ubuntu:xenial-20161213 nipype/base:latest nipype/nipype:latest nipype/nipype_test:py27 nipype/nipype_test:py35 :
         timeout: 6000
 
 test:
@@ -60,7 +60,7 @@ deployment:
   production:
     tag: /.*/
     commands:
-      # Deploy to docker hub      
+      # Deploy to docker hub
       - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker push nipype/nipype:latest; fi :
           timeout: 21600
       - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker tag nipype/nipype nipype/nipype:$CIRCLE_TAG && docker push nipype/nipype:$CIRCLE_TAG; fi :