Merge remote-tracking branch 'upstream/master' into enh/FSLb0calc

Author: oesteban

.travis.yml

@@ -8,6 +8,7 @@ python:
 env:
 - INSTALL_DEB_DEPENDECIES=true
 - INSTALL_DEB_DEPENDECIES=false
+- INSTALL_DEB_DEPENDECIES=true DUECREDIT_ENABLE=yes
 before_install:
 - wget http://repo.continuum.io/miniconda/Miniconda${TRAVIS_PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh
   -O /home/travis/.cache/miniconda.sh
@@ -32,6 +33,7 @@ install:
 #   conda install -y vtk mayavi; fi
 - conda install -y nipype
 - pip install python-coveralls coverage doctest-ignore-unicode
+- if [ ! -z "$DUECREDIT_ENABLE"]; then pip install duecredit; fi
 - rm -r /home/travis/miniconda/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype*
 - pip install -r requirements.txt
 - pip install -e .

bin/nipype_crash_search

@@ -1,11 +1,12 @@
-#!/usr/bin/env python
+#!python
 """Search for tracebacks inside a folder of nipype crash
 log files that match a given regular expression.
 
 Examples:
 nipype_crash_search -d nipype/wd/log -r '.*subject123.*'
 """
 import re
+import sys
 import os.path as op
 from glob import glob
 
@@ -60,6 +61,7 @@ def display_crash_search(logdir, regex):
 
 if __name__ == "__main__":
     from argparse import ArgumentParser, RawTextHelpFormatter
+
     defstr = ' (default %(default)s)'
     parser = ArgumentParser(prog='nipype_crash_search',
                             description=__doc__,
@@ -71,6 +73,10 @@ if __name__ == "__main__":
                         default='*',
                         help='Regular expression to be searched in each traceback.' + defstr)
 
-    args = parser.parse_args()
+    if len(sys.argv) == 1:
+        parser.print_help()
+        exit(0)
 
+    args = parser.parse_args()
     display_crash_search(args.logdir, args.regex)
+    exit(0)

bin/nipype_display_pklz

@@ -0,0 +1,36 @@
+#!python
+"""Prints the content of any .pklz file in your working directory.
+
+Examples:
+
+nipype_print_pklz _inputs.pklz
+nipype_print_pklz _node.pklz
+"""
+
+def pprint_pklz_file(pklz_file):
+    """ Print the content of the pklz_file. """
+    from pprint import pprint
+    from nipype.utils.filemanip import loadpkl
+
+    pkl_data = loadpkl(pklz_file)
+    pprint(pkl_data)
+
+
+if __name__ == "__main__":
+
+    import sys
+    from argparse import ArgumentParser, RawTextHelpFormatter
+
+    defstr = ' (default %(default)s)'
+    parser = ArgumentParser(prog='nipype_print_pklz',
+                            description=__doc__,
+                            formatter_class=RawTextHelpFormatter)
+    parser.add_argument('pklzfile', metavar='f', type=str,
+                        help='pklz file to display')
+
+    if len(sys.argv) == 1:
+        parser.print_help()
+        exit(0)
+
+    args = parser.parse_args()
+    pprint_pklz_file(args.pklzfile)

circle.yml

@@ -21,13 +21,18 @@ dependencies:
     - if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi
     - docker build -f docker/nipype_test/Dockerfile_py35 -t nipype/nipype_test:py35 . :
         timeout: 1600
-    - mkdir -p ~/docker; docker save nipype/nipype_test:py35 > ~/docker/image.tar :
+    - docker build -f docker/nipype_test/Dockerfile_py27 -t nipype/nipype_test:py27 . :
         timeout: 1600
+    - mkdir -p ~/docker; docker save nipype/nipype_test:py27 > ~/docker/image.tar :
+        timeout: 1600
+    - pip install xunitmerge
 
 test:
   override:
     - docker run -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /root/src/nipype/doc nipype/nipype_test:py35 /usr/bin/run_builddocs.sh
-    - docker run -v /etc/localtime:/etc/localtime:ro -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /root/src/nipype nipype/nipype_test:py35 /usr/bin/run_nosetests.sh :
+    - docker run -v /etc/localtime:/etc/localtime:ro -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /root/src/nipype nipype/nipype_test:py35 /usr/bin/run_nosetests.sh py35 :
+        timeout: 2600
+    - docker run -v /etc/localtime:/etc/localtime:ro -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /root/src/nipype nipype/nipype_test:py27 /usr/bin/run_nosetests.sh py27 :
         timeout: 2600
     - docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow3d :
         timeout: 1600
@@ -39,15 +44,19 @@ test:
     - docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ l2pipeline :
         timeout: 1600
     - docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_fsl_reuse Linear /root/examples/ level1_workflow
-    - docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_nested Linear /root/examples/ level1
-    - docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_nested Linear /root/examples/ l2pipeline
+    - docker run -v /etc/localtime:/etc/localtime:ro -e NIPYPE_NUMBER_OF_CPUS=4 -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ level1
+    - docker run -v /etc/localtime:/etc/localtime:ro -e NIPYPE_NUMBER_OF_CPUS=4 -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ level1
+    - docker run -v /etc/localtime:/etc/localtime:ro -e NIPYPE_NUMBER_OF_CPUS=4 -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ l2pipeline
+
   post:
     - bash docker/circleci/teardown.sh
 
 general:
   artifacts:
     - "~/docs"
     - "~/logs"
-    - "~/coverage.xml"
-    - "~/nosetests.xml"
+    - "~/coverage_py27.xml"
+    - "~/coverage_py35.xml"
+    - "~/nosetests_py27.xml"
+    - "~/nosetests_py35.xml"
     - "~/scratch"

doc/devel/index.rst

@@ -19,6 +19,7 @@ conventions documented in the `NIPY Developers Guide
    architecture
    provenance
    software_using_nipype
+   testing_nipype
 
 
 .. include:: ../links_names.txt

doc/devel/testing_nipype.rst

@@ -0,0 +1,73 @@
+==============
+Testing nipype
+==============
+
+In order to ensure the stability of each release of Nipype, the project uses two
+continuous integration services: `CircleCI <https://circleci.com/gh/nipy/nipype/tree/master>`_
+and `Travis CI <https://travis-ci.org/nipy/nipype>`_.
+If both batteries of tests are passing, the following badges should be shown in green color:
+
+.. image:: https://travis-ci.org/nipy/nipype.png?branch=master
+  :target: https://travis-ci.org/nipy/nipype
+
+.. image:: https://circleci.com/gh/nipy/nipype/tree/master.svg?style=svg
+  :target: https://circleci.com/gh/nipy/nipype/tree/master
+
+
+Tests implementation
+--------------------
+
+Nipype testing framework is built upon `nose <http://nose.readthedocs.io/en/latest/>`_.
+By the time these guidelines are written, Nipype implements 17638 tests.
+
+To run the tests locally, first get nose installed::
+
+  pip install nose
+
+
+Then, after nipype is `installed in developer mode <../users/install.html#nipype-for-developers>`_,
+the tests can be run with the following simple command::
+
+  make tests
+
+
+Skip tests
+----------
+
+Nipype will skip some tests depending on the currently available software and data
+dependencies. Installing software dependencies and downloading the necessary data
+will reduce the number of skip tests.
+
+Some tests in Nipype make use of some images distributed within the `FSL course data
+<http://fsl.fmrib.ox.ac.uk/fslcourse/>`_. This reduced version of the package can be downloaded `here 
+<https://3552243d5be815c1b09152da6525cb8fe7b900a6.googledrive.com/host/0BxI12kyv2olZVUswazA3NkFvOXM/nipype-fsl_course_data.tar.gz>`_.
+To enable the tests depending on these data, just unpack the targz file and set the :code:`FSL_COURSE_DATA` environment
+variable to point to that folder.
+
+
+Testing Nipype using Docker
+---------------------------
+
+As of :code:`nipype-0.13`, Nipype is tested inside Docker containers. Once the developer
+`has installed the Docker Engine <https://docs.docker.com/engine/installation/>`_, testing
+Nipype is as easy as follows::
+
+  cd path/to/nipype/
+  docker build -f docker/nipype_test/Dockerfile_py27 -t nipype/nipype_test:py27
+  docker run -it --rm -v /etc/localtime:/etc/localtime:ro \
+                      -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" \
+                      -v ~/examples:/root/examples:ro \
+                      -v ~/scratch:/scratch \
+                      -w /root/src/nipype \
+                      nipype/nipype_test:py27 /usr/bin/run_nosetests.sh
+
+For running nipype in Python 3.5::
+
+  cd path/to/nipype/
+  docker build -f docker/nipype_test/Dockerfile_py35 -t nipype/nipype_test:py35
+  docker run -it --rm -v /etc/localtime:/etc/localtime:ro \
+                      -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" \
+                      -v ~/examples:/root/examples:ro \
+                      -v ~/scratch:/scratch \
+                      -w /root/src/nipype \
+                      nipype/nipype_test:py35 /usr/bin/run_nosetests.sh

docker/circleci/run_nosetests.sh

@@ -3,20 +3,35 @@ set -e
 set -x
 set -u
 
+PYTHON_VERSION=$( python -c "import sys; print('{}{}'.format(sys.version_info[0], sys.version_info[1]))" )
+
 # Create necessary directories
-mkdir -p /scratch/nose /scratch/crashfiles
+mkdir -p /scratch/nose /scratch/crashfiles /scratch/logs/py${PYTHON_VERSION}
 
 # Create a nipype config file
 mkdir -p /root/.nipype
 echo '[logging]' > /root/.nipype/nipype.cfg
 echo 'log_to_file = true' >> /root/.nipype/nipype.cfg
-echo 'log_directory = /scratch/logs/' >> /root/.nipype/nipype.cfg
+echo "log_directory = /scratch/logs/py${PYTHON_VERSION}" >> /root/.nipype/nipype.cfg
 
+# Enable profile_runtime tests only for python 2.7
+if [[ "${PYTHON_VERSION}" -lt "30" ]]; then
+    echo '[execution]' >> /root/.nipype/nipype.cfg
+    echo 'profile_runtime = true' >> /root/.nipype/nipype.cfg
+fi
 
 # Run tests
 cd /root/src/nipype/
 make clean
-nosetests -s nipype -c /root/src/nipype/.noserc --xunit-file="/scratch/nosetests.xml" --cover-xml-file="/scratch/coverage.xml"
+nosetests -s nipype -c /root/src/nipype/.noserc --xunit-file="/scratch/nosetests_py${PYTHON_VERSION}.xml" --cover-xml-file="/scratch/coverage_py${PYTHON_VERSION}.xml"
+
+# Workaround: run here the profiler tests in python 3
+if [[ "${PYTHON_VERSION}" -ge "30" ]]; then
+    echo '[execution]' >> /root/.nipype/nipype.cfg
+    echo 'profile_runtime = true' >> /root/.nipype/nipype.cfg
+    nosetests nipype/interfaces/tests/test_runtime_profiler.py --xunit-file="/scratch/nosetests_py${PYTHON_VERSION}_profiler.xml" --cover-xml-file="/scratch/coverage_py${PYTHON_VERSION}_profiler.xml"
+    nosetests nipype/pipeline/plugins/tests/test_multiproc*.py --xunit-file="/scratch/nosetests_py${PYTHON_VERSION}_multiproc.xml" --cover-xml-file="/scratch/coverage_py${PYTHON_VERSION}_multiproc.xml"
+fi
 
 # Copy crashfiles to scratch
 for i in $(find /root/src/nipype/ -name "crash-*" ); do cp $i /scratch/crashfiles/; done

docker/circleci/teardown.sh

@@ -7,8 +7,8 @@ set -u
 set -e
 
 mkdir -p ${CIRCLE_TEST_REPORTS}/nose
-sudo cp ~/scratch/nosetests.xml ${CIRCLE_TEST_REPORTS}/nose/${CIRCLE_PROJECT_REPONAME}.xml
-sudo mv ~/scratch/coverage.xml ~/coverage.xml
+xunitmerge ~/scratch/nosetests*.xml ${CIRCLE_TEST_REPORTS}/nose/${CIRCLE_PROJECT_REPONAME}.xml
+sudo mv ~/scratch/coverage*.xml ~/
 mkdir -p ~/docs
 sudo mv ~/scratch/docs/* ~/docs/
 mkdir -p ~/logs

docker/nipype_test/Dockerfile_base

@@ -91,10 +91,37 @@ RUN curl -sSL http://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/dev/sp
     unzip spm12.zip && \
     rm -rf spm12.zip
 
-ENV MATLABCMD "/opt/mcr/v85/toolbox/matlab" \
-    SPMMCRCMD "/opt/spm12/run_spm12.sh /opt/mcr/v85/ script" \
-    FORCE_SPMMCR 1
+ENV MATLABCMD="/opt/mcr/v85/toolbox/matlab" \
+    SPMMCRCMD="/opt/spm12/run_spm12.sh /opt/mcr/v85/ script" \
+    FORCE_SPMMCR=1
 
+WORKDIR /root
+# Install miniconda
+RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    /bin/bash Miniconda3-latest-Linux-x86_64.sh -b -p /usr/local/miniconda && \
+    rm Miniconda3-latest-Linux-x86_64.sh
 
-CMD ["/bin/bash"]
+ENV PATH /usr/local/miniconda/bin:$PATH
+
+# http://bugs.python.org/issue19846
+# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
+ENV LANG=C.UTF-8 \
+    LC_ALL=C.UTF-8
 
+# Add conda-forge channel in conda
+RUN conda config --add channels conda-forge && \
+    conda install -y lockfile \
+                     nipype \
+                     matplotlib \
+                     sphinx \
+                     boto \
+                     boto3 \
+                     coverage \
+                     mock \
+                     pbr \
+                     nitime \
+                     dipy \
+                     pandas && \
+    pip install nose-cov doctest-ignore-unicode configparser
+
+CMD ["/bin/bash"]

docker/nipype_test/Dockerfile_py27

@@ -26,43 +26,26 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-FROM nipype/nipype_test:base
+FROM nipype/nipype_test:base-0.0.2
 MAINTAINER The nipype developers https://github.com/nipy/nipype
-# Preparations
-RUN ln -snf /bin/bash /bin/sh
-WORKDIR /root
 
-# Install miniconda
-RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh && \
-    /bin/bash Miniconda-latest-Linux-x86_64.sh -b -p /usr/local/miniconda && \
-    rm Miniconda-latest-Linux-x86_64.sh
-
-ENV PATH /usr/local/miniconda/bin:$PATH
-
-# http://bugs.python.org/issue19846
-# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
-ENV LANG C.UTF-8
-ENV LC_ALL C.UTF-8
-
-# Add conda-forge channel in conda
-RUN conda config --add channels conda-forge && \
-    conda install -y lockfile nipype matplotlib sphinx boto boto3 coverage mock pbr nitime dipy && \
+# Downgrade python to 2.7
+RUN conda update -y conda && \
+    conda update --all -y python=2.7 && \
     pip install nose-cov doctest-ignore-unicode configparser
-   
+
 COPY docker/circleci/run_* /usr/bin/
 RUN chmod +x /usr/bin/run_*
 
+# Speed up building
 RUN mkdir -p /root/src/nipype
-COPY . /root/src/nipype
+COPY requirements.txt /root/src/nipype/requirements.txt
+RUN pip install -r /root/src/nipype/requirements.txt
 
+# Re-install nipype
+COPY . /root/src/nipype
 RUN rm -r /usr/local/miniconda/lib/python2.7/site-packages/nipype* && \
     cd /root/src/nipype && \
-    pip install -r requirements.txt && \
-    pip install -e . && \
-    make specs
-
-# Replace imglob with Python3 compatible version
-RUN rm -r ${FSLDIR}/bin/imglob && \
-    ln -s $(which fsl_imglob.py) ${FSLDIR}/bin/imglob
+    pip install -e .
 
 CMD ["/bin/bash"]