Merge pull request #17 from FCP-INDI/debug_runtime_prof

Debug runtime prof

Author: pintohutch

.travis.yml

@@ -16,28 +16,48 @@ before_install:
 - if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then export PATH=/home/travis/miniconda2/bin:$PATH; else export PATH=/home/travis/miniconda3/bin:$PATH; fi
 - if $INSTALL_DEB_DEPENDECIES; then sudo rm -rf /dev/shm; fi
 - if $INSTALL_DEB_DEPENDECIES; then sudo ln -s /run/shm /dev/shm; fi
-- if $INSTALL_DEB_DEPENDECIES; then bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh);
-  fi
+- bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh)
+- sudo apt-get update
+- sudo apt-get install xvfb
 - if $INSTALL_DEB_DEPENDECIES; then travis_retry sudo apt-get install -qq --no-install-recommends
   fsl afni elastix; fi
 - if $INSTALL_DEB_DEPENDECIES; then travis_retry sudo apt-get install -qq fsl-atlases;
   fi
 - if $INSTALL_DEB_DEPENDECIES; then source /etc/fsl/fsl.sh; fi
 - if $INSTALL_DEB_DEPENDECIES; then source /etc/afni/afni.sh; fi
 - export FSLOUTPUTTYPE=NIFTI_GZ
+# Install vtk and fix numpy installation problem
+# Fix numpy problem: https://github.com/enthought/enable/issues/34#issuecomment-2029381
+- if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then travis_retry sudo apt-get install -qq libx11-dev swig;
+  echo '[x11]' >> $HOME/.numpy-site.cfg; 
+  echo 'library_dirs = /usr/lib64:/usr/lib:/usr/lib/x86_64-linux-gnu' >> $HOME/.numpy-site.cfg;
+  echo 'include_dirs = /usr/include:/usr/include/X11' >> $HOME/.numpy-site.cfg;
+  fi
 install:
+- sudo apt-get install fusefat
 - conda update --yes conda
 - conda create -n testenv --yes pip python=$TRAVIS_PYTHON_VERSION
 - source activate testenv
 - if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then pip install ordereddict; fi
 - conda install --yes numpy scipy nose networkx dateutil
 - if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then conda install --yes traits; else pip install traits; fi
+- if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then conda install --yes vtk; fi
 - pip install python-coveralls
 - pip install nose-cov
+# Add tvtk (PIL is required by blockcanvas)
+# Install mayavi (see https://github.com/enthought/mayavi/issues/271)
+- if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then
+  pip install http://effbot.org/downloads/Imaging-1.1.7.tar.gz;
+  pip install -e git+https://github.com/enthought/etsdevtools.git#egg=etsdevtools;
+  pip install -e git+https://github.com/enthought/blockcanvas.git#egg=blockcanvas;
+  pip install -e git+https://github.com/enthought/etsproxy.git#egg=etsproxy;
+  pip install https://github.com/dmsurti/mayavi/archive/4d4aaf315a29d6a86707dd95149e27d9ed2225bf.zip;
+  pip install -e git+https://github.com/enthought/ets.git#egg=ets;
+  fi
 - pip install -r requirements.txt  # finish remaining requirements
 - python setup.py install
 script:
-- python -W once:FSL:UserWarning:nipype `which nosetests` --with-doctest --with-cov --cover-package nipype --cov-config .coveragerc --logging-level=INFO
+- python -W once:FSL:UserWarning:nipype `which nosetests` --with-doctest --with-cov --cover-package nipype --cov-config .coveragerc --logging-level=DEBUG --verbosity=3
 after_success:
 - coveralls --config_file .coveragerc
 deploy:

CHANGES

@@ -1,6 +1,16 @@
-Next release
+Release 0.12.0-rc1 (April 20, 2016)
 ============
 
+* ENH: Add nipype_crash_search command (https://github.com/nipy/nipype/pull/1422)
+* ENH: Created interface for BrainSuite Cortical Surface Extraction command line tools (https://github.com/nipy/nipype/pull/1305)
+* FIX: job execution on systems/approaches where locale is undefined (https://github.com/nipy/nipype/pull/1401)
+* FIX: Clean up byte/unicode issues using subprocess (https://github.com/nipy/nipype/pull/1394)
+* FIX: Prevent crash when tvtk is loaded - ETS_TOOLKIT=null (https://github.com/nipy/nipype/pull/973)
+* ENH: New interfaces in dipy: RESTORE, EstimateResponseSH, CSD and StreamlineTractography
+       (https://github.com/nipy/nipype/pull/1090)
+* ENH: Added interfaces of AFNI (https://github.com/nipy/nipype/pull/1360,
+       https://github.com/nipy/nipype/pull/1361, https://github.com/nipy/nipype/pull/1382)
+* ENH: Provides a Nipype wrapper for antsJointFusion (https://github.com/nipy/nipype/pull/1351)
 * ENH: Added support for PETPVC (https://github.com/nipy/nipype/pull/1335)
 * ENH: Merge S3DataSink into DataSink, added AWS documentation (https://github.com/nipy/nipype/pull/1316)
 * TST: Cache APT in CircleCI (https://github.com/nipy/nipype/pull/1333)
@@ -15,6 +25,11 @@ Next release
 * API: Default model level for the bedpostx workflow has been set to "2" following FSL 5.0.9 lead
 * ENH: New interfaces for interacting with AWS S3: S3DataSink and S3DataGrabber (https://github.com/nipy/nipype/pull/1201)
 * ENH: Interfaces for MINC tools (https://github.com/nipy/nipype/pull/1304)
+* FIX: Use realpath to determine hard link source (https://github.com/nipy/nipype/pull/1388)
+* FIX: Correct linking/copying fallback behavior (https://github.com/nipy/nipype/pull/1391)
+* ENH: Nipype workflow and interfaces for FreeSurfer's recon-all (https://github.com/nipy/nipype/pull/1326)
+* FIX: Permit relative path for concatenated_file input to Concatenate() (https://github.com/nipy/nipype/pull/1411)
+* ENH: Makes ReconAll workflow backwards compatible with FreeSurfer 5.3.0 (https://github.com/nipy/nipype/pull/1434)
 
 Release 0.11.0 (September 15, 2015)
 ============

CONTRIBUTING.md

@@ -19,7 +19,7 @@
 
 When opening a new Issue, please take the following steps:
 
-1. Search GitHub and/or [Neurostars](neurostars.org) for your issue to avoid duplicate reports.
+1. Search GitHub and/or [Neurostars](http://neurostars.org) for your issue to avoid duplicate reports.
    Keyword searches for your error messages are most helpful.
 2. If possible, try updating to master and reproducing your issue,
    because we may have already fixed it.

Makefile

@@ -66,7 +66,7 @@ test: clean test-code
 
 html:
 	@echo "building docs"
-	make -C doc clean html
+	make -C doc clean htmlonly
 
 specs:
 	@echo "Checking specs and autogenerating spec tests"

README.rst

@@ -34,6 +34,10 @@ NIPYPE: Neuroimaging in Python: Pipelines and Interfaces
     :target: https://pypi.python.org/pypi/nipype/
     :alt: License
 
+.. image:: https://img.shields.io/badge/gitter-join%20chat%20%E2%86%92-brightgreen.svg?style=flat
+    :target: http://gitter.im/nipy/nipype
+    :alt: Chat
+
 Current neuroimaging software offer users an incredible opportunity to
 analyze data using a variety of different algorithms. However, this has
 resulted in a heterogeneous collection of specialized applications

THANKS.rst

@@ -1,102 +1,22 @@
 .. -*- mode: rst -*-
 
-
 Code contributors
 -----------------
-Contributors to Nipype include but are not limited to:
-
-.. hlist::
 
-  * Aimi Watanabe
-  * Alexander Schaefer
-  * Alexandre Gramfort
-  * Alexandre Savio
-  * Anisha Keshavan
-  * Ariel Rokem
-  * Ben Acland
-  * Ben Cipollini
-  * Basile Pinsard
-  * Brendan Moloney
-  * Brian Cheung
-  * Charl Linssen
-  * Chris Filo Gorgolewski
-  * Chris Steele
-  * Christian Haselgrove
-  * Christopher Burns
-  * Cindee Madison
-  * Claire Tarbert
-  * Colin Buchanan
-  * Daniel Ginsburg
-  * Daniel Haehn
-  * Daniel Margulies
-  * Dav Clark
-  * David Welch
-  * Drew Erickson
-  * Erik Kastman
-  * Félix C. Morency
-  * Gael Varoquaux
-  * Hans Johnson
-  * Janosch Linkersdörfer
-  * Januzz
-  * Jarrod Millman
-  * Jeff Lai
-  * Jessica Forbes
-  * John Salvatore
-  * Lijie Huang
-  * Michael Hallquist
-  * Michael Hanke
-  * Michael Notter
-  * Michael Waskom
-  * Nolan Nichols
-  * Oliver Hinds
-  * Oscar Esteban
-  * Rosalia Tungaraza
-  * Satrajit Ghosh
-  * Sharad Sikka
-  * Stephan Gerhard
-  * Erik Ziegler
-  * Valentin Haenel
-  * Xiangzhen Kong
-  * Xu Wang
-  * Yannick Schwartz
-  * Yaroslav O. Halchenko
+See `Github contributors list <https://github.com/nipy/nipype/graphs/contributors>`__.
 
-For full most up to date list see `Ohloh <https://www.ohloh.net/p/nipype/contributors>`__.
+Funding
+-------
 
-Other contributors
-------------------
+Nipype is currently supported by `1R01EB020740-01A1 Nipype: Dataflows for Reproducible Biomedical Research <https://projectreporter.nih.gov/project_info_description.cfm?aid=9053094>`__.
 
-.. hlist::
+Satrajit Ghosh work on this project was partially funded by NIBIB `R03EB008673 <https://projectreporter.nih.gov/project_info_description.cfm?aid=7501200>`__ and by the `INCF <http://www.incf.org>`__ through a contract with TankThink Labs, LLC.
+Chris Burns was supported by NIMH grant `5R01MH081909-02  Continued Development and Maintenance of the Neuroimaging in Python Project <https://projectreporter.nih.gov/project_info_description.cfm?aid=7288112>`__. Hans Jonson was supported by
+`2 U54 EB005149 - 06	 Core 2b Huntington's Disease - Driving Biological Project <http://projectreporter.nih.gov/project_info_description.cfm?aid=8153616>`__,
+`S10 RR023392	 Enterprise Storage In A Collaborative Neuroimaging Environment	 <http://projectreporter.nih.gov/project_info_description.cfm?aid=7209718>`__,
+`R01 NS040068	 Neurobiological Predictors of Huntington's Disease	 <http://projectreporter.nih.gov/project_info_description.cfm?aid=6266377>`__,
+and `UL1 TR000442	University of Iowa Clinical and Translational Science Program <http://projectreporter.nih.gov/project_info_description.cfm?aid=8467220>`__.
 
-  * Matthew Brett
-  * Michael Castelle
-  * Philippe Ciuciu
-  * Yann Cointepas
-  * Mark D'Esposito
-  * Susan Gabrieli
-  * Brian Hawthorne
-  * Tim Leslie
-  * Fernando Perez
-  * Tyler Perrachione
-  * Jean-Baptiste Poline
-  * Alexis Roche
-  * Denis Riviere
-  * Gretchen Reynolds
-  * Jonathan Taylor
-  * Bertrand Thirion
-  * Bernjamin Thyreau
-  * Mike Trumpis
-  * Karl Young
-  * Tom Waite
 
 We would also like to thank `JetBrains <http://www.jetbrains.com/>`__ for providing `Pycharm <http://www.jetbrains.com/pycharm/>`__ licenses.
 
-Funding
--------
-Satrajit Ghosh work on this project was partially funded by NIBIB R03 EB008673 (PI: Ghosh and Whitfield-Gabrieli) and by the `INCF <http://www.incf.org>`__ through a contract with TankThink Labs, LLC.
-Chris Burns was supported by NIMH grant 5R01MH081909-02 (PI: Desposito). Hans Jonson was supported by
-`2 U54 EB005149 - 06	 Core 2b Huntington's Disease - Driving Biological Project <http://projectreporter.nih.gov/project_info_description.cfm?aid=8153616&icde=16158743&ddparam=&ddvalue=&ddsub=&cr=18&csb=PT&cs=ASC>`__,
-`S10 RR023392	 Enterprise Storage In A Collaborative Neuroimaging Environment	 <http://projectreporter.nih.gov/project_info_description.cfm?aid=7209718&icde=16158552&ddparam=&ddvalue=&ddsub=&cr=1&csb=default&cs=ASC>`__,
-`R01 NS040068	 Neurobiological Predictors of Huntington's Disease	 <http://projectreporter.nih.gov/project_info_description.cfm?aid=6266377&icde=16158103>`__,
-and `UL1 TR000442	University of Iowa Clinical and Translational Science Program <http://projectreporter.nih.gov/project_info_description.cfm?aid=8467220&icde=16159156&ddparam=&ddvalue=&ddsub=&cr=1&csb=default&cs=ASC>`__.
-

Vagrantfile

@@ -20,19 +20,20 @@ $script = <<SCRIPT
 # qconf -aattr queue slots "2, [neuro=3]" main.q
 
 # install anaconda
-wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh
+wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
 chmod +x miniconda.sh
 ./miniconda.sh -b
-echo "export PATH=$HOME/miniconda/bin:\\$PATH" >> .bashrc
+echo "export PATH=$HOME/miniconda3/bin:\\$PATH" >> .bashrc
 
 # install nipype dependencies
-$HOME/miniconda/bin/conda update --yes conda
-$HOME/miniconda/bin/conda install --yes pip numpy scipy nose traits networkx
-$HOME/miniconda/bin/conda install --yes dateutil ipython-notebook matplotlib
-$HOME/miniconda/bin/pip install nibabel --use-mirrors
-$HOME/miniconda/bin/pip install https://github.com/RDFLib/rdflib/archive/master.zip
-$HOME/miniconda/bin/pip install https://github.com/trungdong/prov/archive/rdf.zip
-$HOME/miniconda/bin/pip install https://github.com/nipy/nipype/archive/master.zip
+$HOME/miniconda3/bin/conda update --yes conda
+$HOME/miniconda3/bin/conda install --yes pip scipy nose networkx lxml future simplejson
+$HOME/miniconda3/bin/conda install --yes python-dateutil jupyter matplotlib
+$HOME/miniconda3/bin/pip install nibabel
+$HOME/miniconda3/bin/pip install prov
+$HOME/miniconda3/bin/pip install xvfbwrapper
+$HOME/miniconda3/bin/pip install traits
+$HOME/miniconda3/bin/pip install https://github.com/nipy/nipype/archive/master.zip
 SCRIPT
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|

bin/nipype_crash_search

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+"""Search for tracebacks inside a folder of nipype crash
+log files that match a given regular expression.
+
+Examples:
+nipype_crash_search -d nipype/wd/log -r '.*subject123.*'
+"""
+import re
+import os.path as op
+from glob import glob
+
+from traits.trait_errors import TraitError
+from nipype.utils.filemanip import loadcrash
+
+
+def load_pklz_traceback(crash_filepath):
+    """ Return the traceback message in the given crash file."""
+    try:
+        data = loadcrash(crash_filepath)
+    except TraitError as te:
+        return str(te)
+    except:
+        raise
+    else:
+        return '\n'.join(data['traceback'])
+
+
+def iter_tracebacks(logdir):
+    """ Return an iterator over each file path and
+    traceback field inside `logdir`.
+    Parameters
+    ----------
+    logdir: str
+        Path to the log folder.
+
+    field: str
+        Field name to be read from the crash file.
+
+    Yields
+    ------
+    path_file: str
+
+    traceback: str
+    """
+    crash_files = sorted(glob(op.join(logdir, '*.pkl*')))
+
+    for cf in crash_files:
+        yield cf, load_pklz_traceback(cf)
+
+
+def display_crash_search(logdir, regex):
+    rex = re.compile(regex, re.IGNORECASE)
+    for file, trace in iter_tracebacks(logdir):
+        if rex.search(trace):
+            print("-" * len(file))
+            print(file)
+            print("-" * len(file))
+            print(trace)
+
+
+if __name__ == "__main__":
+    from argparse import ArgumentParser, RawTextHelpFormatter
+    defstr = ' (default %(default)s)'
+    parser = ArgumentParser(prog='nipype_crash_search',
+                            description=__doc__,
+                            formatter_class=RawTextHelpFormatter)
+    parser.add_argument('-l','--logdir', type=str, dest='logdir',
+                        action="store", default=None,
+                        help='The working directory log file.' + defstr)
+    parser.add_argument('-r', '--regex', dest='regex',
+                        default='*',
+                        help='Regular expression to be searched in each traceback.' + defstr)
+
+    args = parser.parse_args()
+
+    display_crash_search(args.logdir, args.regex)

circle.yml

@@ -9,21 +9,28 @@ dependencies:
   pre:
     # Let CircleCI cache the apt archive
     - sudo rm -rf /var/cache/apt/archives && sudo ln -s ~/.apt-cache /var/cache/apt/archives && mkdir -p ~/.apt-cache/partial
-    - wget -O- http://neuro.debian.net/lists/precise.us-ca.full | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list
-    - sudo apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9
-    - sudo apt-get update
+    - bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh)
   override:
     # Install apt packages
-    - sudo apt-get install -y fsl-core fsl-atlases fsl-mni152-templates fsl-feeds afni
-    - echo "source /etc/fsl/fsl.sh" >> $HOME/.profile
-    - echo "source /etc/afni/afni.sh" >> $HOME/.profile
+    - sudo apt-get install -y fsl-core fsl-atlases fsl-mni152-templates fsl-feeds afni swig python-vtk xvfb fusefat
+    - echo 'source /etc/fsl/fsl.sh' >> $HOME/.profile
+    - echo 'source /etc/afni/afni.sh' >> $HOME/.profile
     - mkdir -p ~/examples/ && ln -sf /usr/share/fsl-feeds/ ~/examples/feeds
+    # Enable system-wide vtk
+    - ln -sf /usr/lib/pymodules/python2.7/vtk ~/virtualenvs/venv-system/lib/python2.7/site-packages/
     # Set up python environment
     - pip install --upgrade pip
     - pip install -e .
-    - pip install matplotlib sphinx ipython boto coverage
+    - pip install matplotlib sphinx ipython boto coverage dipy
+    # Add tvtk
+    - pip install http://effbot.org/downloads/Imaging-1.1.7.tar.gz
+    - pip install -e git+https://github.com/enthought/etsdevtools.git#egg=etsdevtools
+    - pip install -e git+https://github.com/enthought/blockcanvas.git#egg=blockcanvas
+    - pip install -e git+https://github.com/enthought/etsproxy.git#egg=etsproxy
+    - pip install -e git+https://github.com/enthought/ets.git#egg=ets
+    - pip install https://github.com/rtfd/readthedocs-sphinx-ext/archive/master.zip
     - gem install fakes3
-    - if [[ ! -d ~/examples/data ]]; then wget "http://tcpdiag.dl.sourceforge.net/project/nipy/nipype/nipype-0.2/nipype-tutorial.tar.bz2" && tar jxvf nipype-tutorial.tar.bz2 && mv nipype-tutorial/* ~/examples/; fi
+    - if [[ ! -d ~/examples/data ]]; then wget "https://dl.dropbox.com/s/jzgq2nupxyz36bp/nipype-tutorial.tar.bz2" && tar jxvf nipype-tutorial.tar.bz2 && mv nipype-tutorial/* ~/examples/; fi
     - if [[ ! -d ~/examples/fsl_course_data ]]; then wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt1.tar.gz" && wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt2.tar.gz" && wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/tbss.tar.gz" && mkdir ~/examples/fsl_course_data && tar zxvf fdt1.tar.gz -C ~/examples/fsl_course_data && tar zxvf fdt2.tar.gz -C ~/examples/fsl_course_data && tar zxvf tbss.tar.gz -C ~/examples/fsl_course_data; fi
     - bash ~/nipype/tools/install_spm_mcr.sh
     - mkdir -p ~/.nipype && echo '[logging]' > ~/.nipype/nipype.cfg && echo 'workflow_level = DEBUG' >> ~/.nipype/nipype.cfg && echo 'interface_level = DEBUG' >> ~/.nipype/nipype.cfg && echo 'filemanip_level = DEBUG' >> ~/.nipype/nipype.cfg

doc/_templates/indexsidebar.html

@@ -2,10 +2,10 @@
 <h3>{{ _('Links') }}</h3>
 
 <ul>
-    <li>Docs: <a href="http://nipy.org/nipype">Stable</a> · <a href="http://www.mit.edu/~satra/nipype-nightly/">Nightly</a></li>
+    <li>Docs: <a href="http://nipy.org/nipype">Stable</a> · <a href="http://nipype.readthedocs.org/en/latest/">Dev</a></li>
     <li>Code: <a href="http://github.com/nipy/nipype">Github</a> · <a href="http://github.com/nipy/nipype/issues">Bugs-Requests</a></li>
     <li>Forum: <a href="http://neurostars.org/t/nipype">User</a> · <a href="http://projects.scipy.org/mailman/listinfo/nipy-devel">Developer</a></li>
-    <li><a href="http://nipy.org/software/license/index.html"><img src="https://img.shields.io/pypi/l/nipype.svg" alt="License"></a> · <a href="http://nipy.org/about/funding.html">Funding</a></li>
+    <li><a href="http://nipy.org/software/license/index.html"><img src="https://img.shields.io/pypi/l/nipype.svg" alt="License"></a> · <a href="about.html#funding">Funding</a></li>
     <li><a href="https://travis-ci.org/nipy/nipype"><img src="https://travis-ci.org/nipy/nipype.png?branch=master" alt="travis"></a> · <a href='https://coveralls.io/r/nipy/nipype'><img src='https://coveralls.io/repos/nipy/nipype/badge.png' alt='Coverage Status' /></a></li>
     <li><a href="https://pypi.python.org/pypi/nipype/"><img src="https://img.shields.io/pypi/dm/nipype.svg" alt="Downloads"></a> · <a href='https://pypi.python.org/pypi/nipype/'><img src='https://img.shields.io/pypi/pyversions/nipype.svg' alt='Python Versions' /></a></li>
 </ul>