Merge remote-tracking branch 'origin/master' into HEAD

Author: alexsavio

.circle/tests.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Balance nipype testing workflows across CircleCI build nodes
+#
+
+# Setting      # $ help set
+set -e         # Exit immediately if a command exits with a non-zero status.
+set -u         # Treat unset variables as an error when substituting.
+set -x         # Print command traces before executing command.
+
+if [ "${CIRCLE_NODE_TOTAL:-}" != "4" ]; then
+  echo "These tests were designed to be run at 4x parallelism."
+  exit 1
+fi
+
+# These tests are manually balanced based on previous build timings.
+# They may need to be rebalanced in the future.
+case ${CIRCLE_NODE_INDEX} in
+  0)
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_pytests.sh && \
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_pytests.sh && \
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -v $WORKDIR:/work -w /src/nipype/doc nipype/nipype:py35 /usr/bin/run_builddocs.sh && \
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d && \
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d
+    ;;
+  1)
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ level1 && \
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ l2pipeline
+    ;;
+  2)
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 && \
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline
+    ;;
+  3)
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 && \
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline && \
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow
+    ;;
+esac

.coveragerc

@@ -1,8 +1,2 @@
 [run]
 branch = True
-source = nipype
-include = */nipype/*
-omit =
-    */nipype/external/*
-    */nipype/fixes/*
-    */setup.py

.dockerignore

@@ -1,6 +1,7 @@
 # python cache
-__pycache__/**/*
 __pycache__
+**/__pycache__
+**/*.pyc
 *.pyc
 
 # python distribution
@@ -23,12 +24,14 @@ src/
 # other
 docs/**/*
 docs/
-.coverage
-.coveragerc
-codecov.yml
-rtd_requirements.txt
+.cache/
+.circle/**/*
+.circle/
 circle.yml
+rtd_requirements.txt
 Vagrantfile
 .travis.yml
-.noserc
+.mailmap
 
+# Previous coverage results
+.coverage

.noserc

@@ -42,7 +42,7 @@ before_install:
 install:
 - travis_retry pip install -e .[$NIPYPE_EXTRAS]
 script:
-- py.test --doctest-modules nipype
+- py.test -v --doctest-modules nipype
 deploy:
   provider: pypi
   user: satra

.travis.yml

@@ -1,13 +1,31 @@
-Upcoming release 0.13
+Upcoming Release
 =====================
 
+* ENH: Added non-steady state detector for EPI data (https://github.com/nipy/nipype/pull/1839)
+* ENH: Enable new BBRegister init options for FSv6+ (https://github.com/nipy/nipype/pull/1811)
+* REF: Splits nipype.interfaces.utility into base, csv, and wrappers (https://github.com/nipy/nipype/pull/1828)
+* FIX: Makespec now runs with nipype in current directory (https://github.com/nipy/nipype/pull/1813)
+* FIX: Flexible nifti opening with mmap if Numpy < 1.12.0 (https://github.com/nipy/nipype/pull/1796 + https://github.com/nipy/nipype/pull/1831)
 * ENH: DVARS includes intensity normalization feature - turned on by default (https://github.com/nipy/nipype/pull/1827)
 * FIX: DVARS is correctly using sum of squares instead of standard deviation (https://github.com/nipy/nipype/pull/1827)
 * ENH: Refactoring of nipype.interfaces.utility (https://github.com/nipy/nipype/pull/1828)
 * FIX: CircleCI were failing silently. Some fixes to tests (https://github.com/nipy/nipype/pull/1833)
 * FIX: Issues in Docker image permissions, and docker documentation (https://github.com/nipy/nipype/pull/1825)
 * ENH: Revised all Dockerfiles and automated deployment to Docker Hub
        from CircleCI (https://github.com/nipy/nipype/pull/1815)
+* ENH: Update ReconAll interface for FreeSurfer v6.0.0 (https://github.com/nipy/nipype/pull/1790)
+* FIX: Cast DVARS float outputs to avoid memmap error (https://github.com/nipy/nipype/pull/1777)
+* FIX: FSL FNIRT intensity mapping files (https://github.com/nipy/nipype/pull/1799)
+* ENH: Additional outputs generated by FSL EDDY (https://github.com/nipy/nipype/pull/1793)
+* TST: Parallelize CircleCI build across 4 containers (https://github.com/nipy/nipype/pull/1769)
+
+
+0.13.0-rc1 (January 4, 2017)
+===============================
+ 
+* FIX: Compatibility with traits 4.6 (https://github.com/nipy/nipype/pull/1770)
+* FIX: Multiproc deadlock (https://github.com/nipy/nipype/pull/1756)
+* TST: Replace nose and unittest with pytest (https://github.com/nipy/nipype/pull/1722, https://github.com/nipy/nipype/pull/1751)
 * FIX: Semaphore capture using MultiProc plugin (https://github.com/nipy/nipype/pull/1689)
 * REF: Refactor AFNI interfaces (https://github.com/nipy/nipype/pull/1678, https://github.com/nipy/nipype/pull/1680)
 * ENH: Move nipype commands to group command using click (https://github.com/nipy/nipype/pull/1608)
@@ -36,7 +54,6 @@ Upcoming release 0.13
 * ENH: Added support for custom job submission check in SLURM (https://github.com/nipy/nipype/pull/1582)
 * ENH: Added ANTs interface CreateJacobianDeterminantImage; replaces deprecated JacobianDeterminant
        (https://github.com/nipy/nipype/pull/1654)
-* ENH: Update ReconAll interface for FreeSurfer v6.0.0 (https://github.com/nipy/nipype/pull/1790)
 
 Release 0.12.1 (August 3, 2016)
 ===============================

CHANGES

@@ -30,191 +30,74 @@
 #
 # Based on https://github.com/poldracklab/fmriprep/blob/9c92a3de9112f8ef1655b876de060a2ad336ffb0/Dockerfile
 #
-FROM ubuntu:xenial-20161213
+FROM nipype/base:latest
 MAINTAINER The nipype developers https://github.com/nipy/nipype
 
-ARG DEBIAN_FRONTEND=noninteractive
-
-# Pre-cache neurodebian key
-COPY docker/files/neurodebian.gpg /root/.neurodebian.gpg
-
-# Prepare environment
-RUN apt-key add /root/.neurodebian.gpg && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends curl bzip2 ca-certificates xvfb && \
-    curl -sSL http://neuro.debian.net/lists/xenial.us-ca.full >> /etc/apt/sources.list.d/neurodebian.sources.list && \
-    apt-key adv --refresh-keys --keyserver hkp://ha.pool.sks-keyservers.net 0xA5D32F012649A5A9 || true; \
-    apt-get update
-
-# Installing freesurfer
-RUN curl -sSL https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.0/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.0.tar.gz | tar zxv -C /opt \
-    --exclude='freesurfer/trctrain' \
-    --exclude='freesurfer/subjects/fsaverage_sym' \
-    --exclude='freesurfer/subjects/fsaverage3' \
-    --exclude='freesurfer/subjects/fsaverage4' \
-    --exclude='freesurfer/subjects/fsaverage5' \
-    --exclude='freesurfer/subjects/fsaverage6' \
-    --exclude='freesurfer/subjects/cvs_avg35' \
-    --exclude='freesurfer/subjects/cvs_avg35_inMNI152' \
-    --exclude='freesurfer/subjects/bert' \
-    --exclude='freesurfer/subjects/V1_average' \
-    --exclude='freesurfer/average/mult-comp-cor' \
-    --exclude='freesurfer/lib/cuda' \
-    --exclude='freesurfer/lib/qt'
-
-ENV FSL_DIR=/usr/share/fsl/5.0 \
-    OS=Linux \
-    FS_OVERRIDE=0 \
-    FIX_VERTEX_AREA= \
-    FSF_OUTPUT_FORMAT=nii.gz \
-    FREESURFER_HOME=/opt/freesurfer
-ENV SUBJECTS_DIR=$FREESURFER_HOME/subjects \
-    FUNCTIONALS_DIR=$FREESURFER_HOME/sessions \
-    MNI_DIR=$FREESURFER_HOME/mni \
-    LOCAL_DIR=$FREESURFER_HOME/local \
-    FSFAST_HOME=$FREESURFER_HOME/fsfast \
-    MINC_BIN_DIR=$FREESURFER_HOME/mni/bin \
-    MINC_LIB_DIR=$FREESURFER_HOME/mni/lib \
-    MNI_DATAPATH=$FREESURFER_HOME/mni/data \
-    FMRI_ANALYSIS_DIR=$FREESURFER_HOME/fsfast
-ENV PERL5LIB=$MINC_LIB_DIR/perl5/5.8.5 \
-    MNI_PERL5LIB=$MINC_LIB_DIR/perl5/5.8.5 \
-    PATH=$FREESURFER_HOME/bin:$FSFAST_HOME/bin:$FREESURFER_HOME/tktools:$MINC_BIN_DIR:$PATH
-RUN echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh
-
-# Installing Neurodebian packages (FSL, AFNI, git)
-RUN apt-get install -y --no-install-recommends \
-                    fsl-core=5.0.9-1~nd+1+nd16.04+1 \
-                    fsl-mni152-templates=5.0.7-2 \
-                    afni=16.2.07~dfsg.1-2~nd16.04+1
-
-ENV FSLDIR=/usr/share/fsl/5.0 \
-    FSLOUTPUTTYPE=NIFTI_GZ \
-    FSLMULTIFILEQUIT=TRUE \
-    POSSUMDIR=/usr/share/fsl/5.0 \
-    LD_LIBRARY_PATH=/usr/lib/fsl/5.0:$LD_LIBRARY_PATH \
-    FSLTCLSH=/usr/bin/tclsh \
-    FSLWISH=/usr/bin/wish \
-    AFNI_MODELPATH=/usr/lib/afni/models \
-    AFNI_IMSAVE_WARNINGS=NO \
-    AFNI_TTATLAS_DATASET=/usr/share/afni/atlases \
-    AFNI_PLUGINPATH=/usr/lib/afni/plugins \
-    PATH=/usr/lib/fsl/5.0:/usr/lib/afni/bin:$PATH
-
-# Installing and setting up ANTs
-RUN mkdir -p /opt/ants && \
-    curl -sSL "https://github.com/stnava/ANTs/releases/download/v2.1.0/Linux_Ubuntu14.04.tar.bz2" \
-    | tar -xjC /opt/ants --strip-components 1
-
-ENV ANTSPATH=/opt/ants \
-    PATH=$ANTSPATH:$PATH
-
-# Installing and setting up c3d
-RUN mkdir -p /opt/c3d && \
-    curl -sSL "http://downloads.sourceforge.net/project/c3d/c3d/1.0.0/c3d-1.0.0-Linux-x86_64.tar.gz" \
-    | tar -xzC /opt/c3d --strip-components 1
-
-ENV C3DPATH=/opt/c3d/ \
-    PATH=$C3DPATH/bin:$PATH
-
-# Install some other required tools
-RUN apt-get install -y --no-install-recommends \
-                    git=1:2.7.4-0ubuntu1 \
-                    graphviz=2.38.0-12ubuntu2 \
-                    unzip \
-                    apt-utils \
-                    fusefat \
-                    make \
-                    ruby=1:2.3.0+1 && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-# Install fake-S3
-ENV GEM_HOME /usr/lib/ruby/gems/2.3
-ENV BUNDLE_PATH="$GEM_HOME" \
-    BUNDLE_BIN="$GEM_HOME/bin" \
-    BUNDLE_SILENCE_ROOT_WARNING=1 \
-    BUNDLE_APP_CONFIG="$GEM_HOME"
-ENV PATH $BUNDLE_BIN:$PATH
-RUN mkdir -p "$GEM_HOME" "$BUNDLE_BIN" && \
-    chmod 777 "$GEM_HOME" "$BUNDLE_BIN"
-
-RUN gem install fakes3
-
-# Install Matlab MCR: from the good old install_spm_mcr.sh of @chrisfilo
-WORKDIR /opt
-RUN echo "destinationFolder=/opt/mcr" > mcr_options.txt && \
-    echo "agreeToLicense=yes" >> mcr_options.txt && \
-    echo "outputFile=/tmp/matlabinstall_log" >> mcr_options.txt && \
-    echo "mode=silent" >> mcr_options.txt && \
-    mkdir -p matlab_installer && \
-    curl -sSL http://www.mathworks.com/supportfiles/downloads/R2015a/deployment_files/R2015a/installers/glnxa64/MCR_R2015a_glnxa64_installer.zip \
-         -o matlab_installer/installer.zip && \
-    unzip matlab_installer/installer.zip -d matlab_installer/ && \
-    matlab_installer/install -inputFile mcr_options.txt && \
-    rm -rf matlab_installer mcr_options.txt
-
-# Install SPM
-RUN curl -sSL http://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/dev/spm12_r6472_Linux_R2015a.zip -o spm12.zip && \
-    unzip spm12.zip && \
-    rm -rf spm12.zip
-
-ENV MATLABCMD="/opt/mcr/v85/toolbox/matlab" \
-    SPMMCRCMD="/opt/spm12/run_spm12.sh /opt/mcr/v85/ script" \
-    FORCE_SPMMCR=1
-
+ARG PYTHON_VERSION_MAJOR=3
 
 # Installing and setting up miniconda
-RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-4.2.12-Linux-x86_64.sh && \
-    bash Miniconda3-4.2.12-Linux-x86_64.sh -b -p /usr/local/miniconda && \
-    rm Miniconda3-4.2.12-Linux-x86_64.sh
+RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION_MAJOR}-4.2.12-Linux-x86_64.sh && \
+    bash Miniconda${PYTHON_VERSION_MAJOR}-4.2.12-Linux-x86_64.sh -b -p /usr/local/miniconda && \
+    rm Miniconda${PYTHON_VERSION_MAJOR}-4.2.12-Linux-x86_64.sh
 
 ENV PATH=/usr/local/miniconda/bin:$PATH \
     LANG=C.UTF-8 \
     LC_ALL=C.UTF-8 \
-    ACCEPT_INTEL_PYTHON_EULA=yes
+    ACCEPT_INTEL_PYTHON_EULA=yes \
+    MKL_NUM_THREADS=1 \
+    OMP_NUM_THREADS=1
+# MKL/OMP_NUM_THREADS: unless otherwise specified, each process should
+# only use one thread - nipype will handle parallelization
 
 # Installing precomputed python packages
-RUN conda config --add channels conda-forge --add channels intel && \
-    chmod +x /usr/local/miniconda/bin/* && \
-    conda config --set always_yes yes --set changeps1 no && \
-    conda update -q conda && \
-    chmod +x /usr/local/miniconda/bin/*; sync && \
-    conda install -y mkl=2017.0.1 \
-                     numpy=1.11.2 \
-                     scipy=0.18.1 \
-                     scikit-learn=0.17.1 \
-                     matplotlib=1.5.3 \
-                     pandas=0.19.0 \
-                     libxml2=2.9.4 \
-                     libxslt=1.1.29 \
+ARG PYTHON_VERSION_MINOR=5
+RUN conda config --add channels conda-forge; sync && \
+    conda config --set always_yes yes --set changeps1 no; sync && \
+    conda install -y python=${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR} \
+                     mkl \
+                     numpy \
+                     scipy \
+                     scikit-learn \
+                     matplotlib \
+                     pandas \
+                     libxml2 \
+                     libxslt \
                      traits=4.6.0 \
-                     psutil=5.0.1 \
+                     psutil \
                      icu=58.1 && \
-    find /usr/local/miniconda/ -exec chmod 775 {} +
+    sync;
 
 # matplotlib cleanups: set default backend, precaching fonts
-RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python3.5/site-packages/matplotlib/mpl-data/matplotlibrc && \
+RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}/site-packages/matplotlib/mpl-data/matplotlibrc && \
     python -c "from matplotlib import font_manager"
 
-# Unless otherwise specified each process should only use one thread - nipype
-# will handle parallelization
-ENV MKL_NUM_THREADS=1 \
-    OMP_NUM_THREADS=1
+# Install codecov inside container
+RUN curl -so /usr/bin/codecov.io https://codecov.io/bash && \
+    chmod 755 /usr/bin/codecov.io
+
+# Install CI scripts
+COPY docker/files/run_* /usr/bin/
+RUN chmod +x /usr/bin/run_*
+
+# Replace imglob with a Python3 compatible version
+COPY nipype/external/fsl_imglob.py /usr/bin/fsl_imglob.py
+RUN rm -rf ${FSLDIR}/bin/imglob && \
+    chmod +x /usr/bin/fsl_imglob.py && \
+    ln -s /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob
 
 # Installing dev requirements (packages that are not in pypi)
-WORKDIR /root/
+WORKDIR /src/
 COPY requirements.txt requirements.txt
 RUN pip install -r requirements.txt && \
     rm -rf ~/.cache/pip
 
 # Installing nipype
-COPY . /root/src/nipype
-RUN cd /root/src/nipype && \
+COPY . /src/nipype
+RUN cd /src/nipype && \
     pip install -e .[all] && \
     rm -rf ~/.cache/pip
 
-WORKDIR /root/
+WORKDIR /work/
 
 ARG BUILD_DATE
 ARG VCS_REF