Merge remote-tracking branch 'nipy/master'

Author: tsalo

.circle/codecov.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# This script pull all coverage files into the $CIRCLE_TEST_REPORTS folder
+# and sends data to codecov.
+#
+
+# Setting      # $ help set
+set -e         # Exit immediately if a command exits with a non-zero status.
+set -u         # Treat unset variables as an error when substituting.
+set -x         # Print command traces before executing command.
+
+mkdir -p ${CIRCLE_TEST_REPORTS}/
+for report in $( ls ~/scratch/*.xml ); do
+    rname=$( basename $report )
+    cp ${report} ${CIRCLE_TEST_REPORTS}/${rname:: -4}_${CIRCLE_NODE_INDEX}.xml
+done
+
+# Send coverage data to codecov.io
+curl -so codecov.io https://codecov.io/bash
+chmod 755 codecov.io
+
+find "${CIRCLE_TEST_REPORTS}/" -name 'coverage*.xml' -print0 | \
+  xargs -0 -I file ./codecov.io -f file -t "${CODECOV_TOKEN}" -F unittests
+find "${CIRCLE_TEST_REPORTS}/" -name 'smoketests*.xml' -print0 | \
+  xargs -0 -I file ./codecov.io -f file -t "${CODECOV_TOKEN}" -F smoketests

docker/files/tests.sh renamed to .circle/tests.sh

@@ -1,9 +1,12 @@
 #!/bin/bash
+#
+# Balance nipype testing workflows across CircleCI build nodes
+#
 
-set -o nounset
-set -o xtrace
-
-export CODECOV_TOKEN=ac172a50-8e66-42e5-8822-5373fcf54686
+# Setting      # $ help set
+set -e         # Exit immediately if a command exits with a non-zero status.
+set -u         # Treat unset variables as an error when substituting.
+set -x         # Print command traces before executing command.
 
 if [ "${CIRCLE_NODE_TOTAL:-}" != "4" ]; then
   echo "These tests were designed to be run at 4x parallelism."
@@ -14,15 +17,15 @@ fi
 # They may need to be rebalanced in the future.
 case ${CIRCLE_NODE_INDEX} in
   0)
-    docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ level1 && \
-    docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ l2pipeline
-    ;;
-  1)
-    docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow3d && \
-    docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow4d && \
     docker run --rm -it -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /root/src/nipype nipype/nipype_test:py27 /usr/bin/run_pytests.sh py27 && \
     docker run --rm -it -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /root/src/nipype nipype/nipype_test:py35 /usr/bin/run_pytests.sh py35 && \
-    docker run --rm -it -v $SCRATCH:/scratch -w /root/src/nipype/doc nipype/nipype_test:py35 /usr/bin/run_builddocs.sh
+    docker run --rm -it -v $SCRATCH:/scratch -w /root/src/nipype/doc nipype/nipype_test:py35 /usr/bin/run_builddocs.sh && \
+    docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow3d && \
+    docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow4d
+    ;;
+  1)
+    docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ level1 && \
+    docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /root/examples/ l2pipeline
     ;;
   2)
     docker run --rm -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /root/examples/ level1 && \
@@ -34,14 +37,3 @@ case ${CIRCLE_NODE_INDEX} in
     docker run --rm -it -v $HOME/examples:/root/examples:ro -v $SCRATCH:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh fmri_fsl_reuse Linear /root/examples/ level1_workflow
     ;;
 esac
-
-# Put the artifacts in place
-bash docker/files/teardown.sh
-
-# Send coverage data to codecov.io
-curl -so codecov.io https://codecov.io/bash
-chmod 755 codecov.io
-find "${CIRCLE_TEST_REPORTS}/pytest" -name 'coverage*.xml' -print0 | \
-  xargs -0 -I file ./codecov.io -f file -t "${CODECOV_TOKEN}" -F unittests
-find "${CIRCLE_TEST_REPORTS}/pytest" -name 'smoketests*.xml' -print0 | \
-  xargs -0 -I file ./codecov.io -f file -t "${CODECOV_TOKEN}" -F smoketests

.dockerignore

@@ -23,11 +23,13 @@ src/
 # other
 docs/**/*
 docs/
+.circle/**/*
+.circle/
+circle.yml
 .coverage
 .coveragerc
 codecov.yml
 rtd_requirements.txt
-circle.yml
 Vagrantfile
 .travis.yml
 .noserc

CHANGES

@@ -1,13 +1,31 @@
-Upcoming release 0.13
+Upcoming Release
 =====================
 
+* ENH: Added non-steady state detector for EPI data (https://github.com/nipy/nipype/pull/1839)
+* ENH: Enable new BBRegister init options for FSv6+ (https://github.com/nipy/nipype/pull/1811)
+* REF: Splits nipype.interfaces.utility into base, csv, and wrappers (https://github.com/nipy/nipype/pull/1828)
+* FIX: Makespec now runs with nipype in current directory (https://github.com/nipy/nipype/pull/1813)
+* FIX: Flexible nifti opening with mmap if Numpy < 1.12.0 (https://github.com/nipy/nipype/pull/1796 + https://github.com/nipy/nipype/pull/1831)
 * ENH: DVARS includes intensity normalization feature - turned on by default (https://github.com/nipy/nipype/pull/1827)
 * FIX: DVARS is correctly using sum of squares instead of standard deviation (https://github.com/nipy/nipype/pull/1827)
 * ENH: Refactoring of nipype.interfaces.utility (https://github.com/nipy/nipype/pull/1828)
 * FIX: CircleCI were failing silently. Some fixes to tests (https://github.com/nipy/nipype/pull/1833)
 * FIX: Issues in Docker image permissions, and docker documentation (https://github.com/nipy/nipype/pull/1825)
 * ENH: Revised all Dockerfiles and automated deployment to Docker Hub
        from CircleCI (https://github.com/nipy/nipype/pull/1815)
+* ENH: Update ReconAll interface for FreeSurfer v6.0.0 (https://github.com/nipy/nipype/pull/1790)
+* FIX: Cast DVARS float outputs to avoid memmap error (https://github.com/nipy/nipype/pull/1777)
+* FIX: FSL FNIRT intensity mapping files (https://github.com/nipy/nipype/pull/1799)
+* ENH: Additional outputs generated by FSL EDDY (https://github.com/nipy/nipype/pull/1793)
+* TST: Parallelize CircleCI build across 4 containers (https://github.com/nipy/nipype/pull/1769)
+
+
+0.13.0-rc1 (January 4, 2017)
+===============================
+ 
+* FIX: Compatibility with traits 4.6 (https://github.com/nipy/nipype/pull/1770)
+* FIX: Multiproc deadlock (https://github.com/nipy/nipype/pull/1756)
+* TST: Replace nose and unittest with pytest (https://github.com/nipy/nipype/pull/1722, https://github.com/nipy/nipype/pull/1751)
 * FIX: Semaphore capture using MultiProc plugin (https://github.com/nipy/nipype/pull/1689)
 * REF: Refactor AFNI interfaces (https://github.com/nipy/nipype/pull/1678, https://github.com/nipy/nipype/pull/1680)
 * ENH: Move nipype commands to group command using click (https://github.com/nipy/nipype/pull/1608)
@@ -36,7 +54,6 @@ Upcoming release 0.13
 * ENH: Added support for custom job submission check in SLURM (https://github.com/nipy/nipype/pull/1582)
 * ENH: Added ANTs interface CreateJacobianDeterminantImage; replaces deprecated JacobianDeterminant
        (https://github.com/nipy/nipype/pull/1654)
-* ENH: Update ReconAll interface for FreeSurfer v6.0.0 (https://github.com/nipy/nipype/pull/1790)
 
 Release 0.12.1 (August 3, 2016)
 ===============================

README.rst

@@ -78,8 +78,8 @@ Support and Communication
 -------------------------
 
 If you have a problem or would like to ask a question about how to do something in Nipype please open an issue to
-`NeuroStars.org <http://neurostars.org>`_ with a *nipype* tag. `NeuroStars.org <http://neurostars.org>`_  is a 
-platform similar to StackOverflow but dedicated to neuroinformatics. 
+`NeuroStars.org <http://neurostars.org>`_ with a *nipype* tag. `NeuroStars.org <http://neurostars.org>`_  is a
+platform similar to StackOverflow but dedicated to neuroinformatics.
 
 To participate in the Nipype development related discussions please use the following mailing list::
 
@@ -117,16 +117,3 @@ Currently Nipype consists of the following files and directories:
   setup.py
     Script for building and installing NIPYPE.
 
-License information
--------------------
-
-We use the 3-clause BSD license; the full license is in the file ``LICENSE`` in
-the nipype distribution.
-
-There are interfaces to some GNU code but these are entirely optional.
-
-All trademarks referenced herein are property of their respective
-holders.
-
-Copyright (c) 2009-2015, NIPY Developers
-All rights reserved.

circle.yml

@@ -8,6 +8,7 @@ machine:
     DATA_NIPYPE_FSL_COURSE: "${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe"
     DATA_NIPYPE_FSL_FEEDS: "${OSF_NIPYPE_URL}/57f473066c613b01f113e7af"
     SCRATCH: "$HOME/scratch"
+    CODECOV_TOKEN: "ac172a50-8e66-42e5-8822-5373fcf54686"
   services:
     - docker
 
@@ -25,40 +26,46 @@ dependencies:
     - mkdir -p $SCRATCH && sudo setfacl -d -m group:ubuntu:rwx $SCRATCH && sudo setfacl -m group:ubuntu:rwx $SCRATCH
     - mkdir -p $HOME/docker $HOME/examples $SCRATCH/pytest $SCRATCH/logs
   override:
+    - if [[ -e $HOME/docker/cache.tar ]]; then docker load --input $HOME/docker/cache.tar; else echo 'No docker image found in cache'; fi :
+        timeout: 6000
     - if [[ ! -d ~/examples/nipype-tutorial ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-tutorial.tar.bz2 "${DATA_NIPYPE_TUTORIAL_URL}" && tar xjf nipype-tutorial.tar.bz2 -C ~/examples/; fi
     - if [[ ! -d ~/examples/nipype-fsl_course_data ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-fsl_course_data.tar.gz "${DATA_NIPYPE_FSL_COURSE}" && tar xzf nipype-fsl_course_data.tar.gz -C ~/examples/; fi
     - if [[ ! -d ~/examples/feeds ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O fsl-5.0.9-feeds.tar.gz "${DATA_NIPYPE_FSL_FEEDS}" && tar xzf fsl-5.0.9-feeds.tar.gz -C ~/examples/; fi
-    - if [[ -e $HOME/docker/image.tar ]]; then docker load --input $HOME/docker/image.tar; else echo 'No docker image found in cache'; fi
     - docker images
     - sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py
-    - e=1 && for i in {1..5}; do docker build -t nipype/nipype:latest --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
+    - e=1 && for i in {1..5}; do docker build --rm=false -t nipype/nipype:latest --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
         timeout: 21600
-    - e=1 && for i in {1..5}; do docker build -f docker/Dockerfile_py27 -t nipype/nipype_test:py27 . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
+    - e=1 && for i in {1..5}; do docker build --rm=false -f docker/Dockerfile_py27 -t nipype/nipype_test:py27 . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
         timeout: 1600
-    - e=1 && for i in {1..5}; do docker build -f docker/Dockerfile_py35 -t nipype/nipype_test:py35 . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
-        timeout: 1600
-    - docker save -o $HOME/docker/image.tar nipype/nipype:latest nipype/nipype_test:py27 nipype/nipype_test:py35 :
+    - e=1 && for i in {1..5}; do docker build --rm=false -f docker/Dockerfile_py35 -t nipype/nipype_test:py35 . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
         timeout: 1600
+    - docker save -o $HOME/docker/cache.tar nipype/nipype:latest nipype/nipype_test:py27 nipype/nipype_test:py35 :
+        timeout: 6000
 
 test:
   override:
-    - bash docker/files/tests.sh :
+    - bash .circle/tests.sh :
         timeout: 7200
         parallel: true
+  post:
+    # Send coverage data to codecov.io
+    - bash .circle/codecov.sh
 
 general:
   artifacts:
-    - "~/docs"
-    - "~/logs"
+    - "~/scratch/docs"
+    - "~/scratch/logs"
 
 deployment:
   production:
     tag: /.*/
     commands:
+      # Deploy to docker hub      
       - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker push nipype/nipype:latest; fi :
           timeout: 21600
       - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker tag nipype/nipype nipype/nipype:$CIRCLE_TAG && docker push nipype/nipype:$CIRCLE_TAG; fi :
           timeout: 21600
+
 # Automatic deployment to Pypi:
 #       - printf "[distutils]\nindex-servers =\n    pypi\n\n[pypi]\nusername:$PYPI_USER\npassword:$PYPI_PASS\n" > ~/.pypirc
 #       - python setup.py sdist upload -r pypi

docker/files/teardown.sh

@@ -27,7 +27,8 @@
 from ..interfaces.base import (traits, TraitedSpec, BaseInterface,
                                BaseInterfaceInputSpec, File, isdefined,
                                InputMultiPath)
-from nipype.utils import NUMPY_MMAP
+from ..utils import NUMPY_MMAP
+from ..utils.misc import normalize_mc_params
 
 IFLOG = logging.getLogger('interface')
 
@@ -206,7 +207,10 @@ def _list_outputs(self):
 
 
 class FramewiseDisplacementInputSpec(BaseInterfaceInputSpec):
-    in_plots = File(exists=True, mandatory=True, desc='motion parameters as written by FSL MCFLIRT')
+    in_file = File(exists=True, mandatory=True, desc='motion parameters')
+    parameter_source = traits.Enum("FSL", "AFNI", "SPM", "FSFAST",
+                                   desc="Source of movement parameters",
+                                   mandatory=True)
     radius = traits.Float(50, usedefault=True,
                           desc='radius in mm to calculate angular FDs, 50mm is the '
                                'default since it is used in Power et al. 2012')
@@ -260,9 +264,12 @@ class FramewiseDisplacement(BaseInterface):
     }]
 
     def _run_interface(self, runtime):
-        mpars = np.loadtxt(self.inputs.in_plots)  # mpars is N_t x 6
-        diff = mpars[:-1, :] - mpars[1:, :]
-        diff[:, :3] *= self.inputs.radius
+        mpars = np.loadtxt(self.inputs.in_file)  # mpars is N_t x 6
+        mpars = np.apply_along_axis(func1d=normalize_mc_params,
+                                    axis=1, arr=mpars,
+                                    source=self.inputs.parameter_source)
+        diff = mpars[:-1, :6] - mpars[1:, :6]
+        diff[:, 3:6] *= self.inputs.radius
         fd_res = np.abs(diff).sum(axis=1)
 
         self._results = {
@@ -572,6 +579,79 @@ def _list_outputs(self):
             outputs['detrended_file'] = op.abspath(self.inputs.detrended_file)
         return outputs
 
+
+class NonSteadyStateDetectorInputSpec(BaseInterfaceInputSpec):
+    in_file = File(exists=True, mandatory=True, desc='4D NIFTI EPI file')
+
+
+class NonSteadyStateDetectorOutputSpec(TraitedSpec):
+    n_volumes_to_discard = traits.Int(desc='Number of non-steady state volumes'
+                                           'detected in the beginning of the scan.')
+
+
+class NonSteadyStateDetector(BaseInterface):
+    """
+    Returns the number of non-steady state volumes detected at the beginning
+    of the scan.
+    """
+
+    input_spec = NonSteadyStateDetectorInputSpec
+    output_spec = NonSteadyStateDetectorOutputSpec
+
+    def _run_interface(self, runtime):
+        in_nii = nb.load(self.inputs.in_plots)
+        global_signal = in_nii.get_data()[:,:,:,:50].mean(axis=0).mean(axis=0).mean(axis=0)
+
+        self._results = {
+            'out_file': _is_outlier(global_signal)
+        }
+
+        return runtime
+
+    def _list_outputs(self):
+        return self._results
+
+def _is_outlier(points, thresh=3.5):
+    """
+    Returns a boolean array with True if points are outliers and False
+    otherwise.
+
+    Parameters:
+    -----------
+        points : An numobservations by numdimensions array of observations
+        thresh : The modified z-score to use as a threshold. Observations with
+            a modified z-score (based on the median absolute deviation) greater
+            than this value will be classified as outliers.
+
+    Returns:
+    --------
+        mask : A numobservations-length boolean array.
+
+    References:
+    ----------
+        Boris Iglewicz and David Hoaglin (1993), "Volume 16: How to Detect and
+        Handle Outliers", The ASQC Basic References in Quality Control:
+        Statistical Techniques, Edward F. Mykytka, Ph.D., Editor.
+    """
+    if len(points.shape) == 1:
+        points = points[:, None]
+    median = np.median(points, axis=0)
+    diff = np.sum((points - median) ** 2, axis=-1)
+    diff = np.sqrt(diff)
+    med_abs_deviation = np.median(diff)
+
+    modified_z_score = 0.6745 * diff / med_abs_deviation
+
+    timepoints_to_discard = 0
+    for i in range(len(modified_z_score)):
+        if modified_z_score[i] <= thresh:
+            break
+        else:
+            timepoints_to_discard += 1
+
+    return timepoints_to_discard
+
+
 def regress_poly(degree, data, remove_mean=True, axis=-1):
     ''' returns data with degree polynomial regressed out.
     Be default it is calculated along the last axis (usu. time).

nipype/algorithms/confounds.py

@@ -34,7 +34,7 @@
                                OutputMultiPath, TraitedSpec, File,
                                BaseInterfaceInputSpec, isdefined)
 from ..utils.filemanip import filename_to_list, save_json, split_filename
-from ..utils.misc import find_indices
+from ..utils.misc import find_indices, normalize_mc_params
 from .. import logging, config
 iflogger = logging.getLogger('interface')
 
@@ -46,15 +46,12 @@ def _get_affine_matrix(params, source):
     source : the package that generated the parameters
              supports SPM, AFNI, FSFAST, FSL, NIPY
     """
-    if source == 'FSL':
-        params = params[[3, 4, 5, 0, 1, 2]]
-    elif source in ('AFNI', 'FSFAST'):
-        params = params[np.asarray([4, 5, 3, 1, 2, 0]) + (len(params) > 6)]
-        params[3:] = params[3:] * np.pi / 180.
     if source == 'NIPY':
         # nipy does not store typical euler angles, use nipy to convert
         from nipy.algorithms.registration import to_matrix44
         return to_matrix44(params)
+
+    params = normalize_mc_params(params, source)
     # process for FSL, SPM, AFNI and FSFAST
     rotfunc = lambda x: np.array([[np.cos(x), np.sin(x)],
                                   [-np.sin(x), np.cos(x)]])