Merge branch 'master' of git://github.com/nipy/nipype into fix/art

Author: mgxd

.circle/tests.sh

@@ -17,8 +17,8 @@ fi
 # They may need to be rebalanced in the future.
 case ${CIRCLE_NODE_INDEX} in
   0)
-    docker run --rm=false -it -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_pytests.sh && \
-    docker run --rm=false -it -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_pytests.sh && \
+    docker run --rm=false -it -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_pytests.sh && \
+    docker run --rm=false -it -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_pytests.sh && \
     docker run --rm=false -it -v $WORKDIR:/work -w /src/nipype/doc --entrypoint=/usr/bin/run_builddocs.sh nipype/nipype:py36 /usr/bin/run_builddocs.sh && \
     docker run --rm=false -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d && \
     docker run --rm=false -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d
@@ -30,8 +30,8 @@ case ${CIRCLE_NODE_INDEX} in
     exitcode=$?
     ;;
   2)
-    docker run --rm=false -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 && \
-    docker run --rm=false -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline
+    docker run --rm=false -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 && \
+    docker run --rm=false -it -e NIPYPE_NUMBER_OF_CPUS=4 -e NIPYPE_RESOURCE_MONITOR=1 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline
     exitcode=$?
     ;;
   3)

.travis.yml

@@ -8,9 +8,10 @@ python:
 - 3.5
 - 3.6
 env:
-- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler"
-- INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler"
-- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit"
+- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" CI_SKIP_TEST=1
+- INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler" CI_SKIP_TEST=1
+- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" CI_SKIP_TEST=1
+- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" PIP_FLAGS="--pre" CI_SKIP_TEST=1
 before_install:
 - function apt_inst {
   if $INSTALL_DEB_DEPENDECIES; then sudo rm -rf /dev/shm; fi &&
@@ -36,12 +37,17 @@ before_install:
   conda install python=${TRAVIS_PYTHON_VERSION} &&
   conda config --add channels conda-forge &&
   conda install -y nipype icu &&
-  rm -r ${CONDA_HOME}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype*; }
+  rm -r ${CONDA_HOME}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/nipype*;
+  pushd $HOME;
+  git clone https://github.com/INCF/pybids.git;
+  cd pybids;
+  pip install -e .;
+  popd; }
 #  Add install of vtk and mayavi to test mesh (disabled): conda install -y vtk mayavi
 - travis_retry apt_inst
 - travis_retry conda_inst
 install:
-- travis_retry pip install -e .[$NIPYPE_EXTRAS]
+- travis_retry pip install $PIP_FLAGS -e .[$NIPYPE_EXTRAS]
 script:
 - py.test -v --doctest-modules nipype
 deploy:

.zenodo.json

@@ -523,6 +523,16 @@
       "affiliation": "University of Amsterdam",
       "name": "Lukas Snoek",
       "orcid": "0000-0001-8972-204X"
+    },
+    {
+      "affiliation": "Vrije Universiteit, Amsterdam",
+      "name": "Gilles de Hollander",
+      "orcid": "0000-0003-1988-5091"
+    },
+    {
+      "affiliation": "University of Texas at Austin",
+      "name": "De La Vega, Alejandro",
+      "orcid": "0000-0001-9062-3778"
     }
   ],
   "keywords": [

CHANGES

@@ -1,6 +1,10 @@
 Upcoming release
 ================
 
+* ENH: Improve terminal_output feature (https://github.com/nipy/nipype/pull/2209)
+* ENH: Simple interface to FSL std2imgcoords (https://github.com/nipy/nipype/pull/2209, prev #1398)
+* ENH: Centralize virtual/physical $DISPLAYs (https://github.com/nipy/nipype/pull/#2203)
+* ENH: New ResourceMonitor - replaces resource profiler (https://github.com/nipy/nipype/pull/#2200)
 
 0.13.1 (May 20, 2017)
 =====================

Dockerfile

@@ -87,6 +87,9 @@ COPY requirements.txt requirements.txt
 RUN pip install -r requirements.txt && \
     rm -rf ~/.cache/pip
 
+RUN git clone https://github.com/INCF/pybids.git && \
+    cd pybids && python setup.py develop
+
 # Installing nipype
 COPY . /src/nipype
 RUN cd /src/nipype && \

doc/devel/gitwash/known_projects.inc

@@ -6,7 +6,7 @@
 .. _`PROJECTNAME mailing list`: http://projects.scipy.org/mailman/listinfo/nipy-devel
 
 .. numpy
-.. _numpy: hhttp://numpy.scipy.org
+.. _numpy: http://numpy.scipy.org
 .. _`numpy github`: http://github.com/numpy/numpy
 .. _`numpy mailing list`: http://mail.scipy.org/mailman/listinfo/numpy-discussion
 

doc/devel/interface_specs.rst

@@ -159,6 +159,70 @@ generated depending on inputs, by the tool.  OutputSpecs inherit from
 ``interfaces.base.TraitedSpec`` directly.
 
 
+Controlling outputs to terminal
+-------------------------------
+
+It is very likely that the software wrapped within the interface writes
+to the standard output or the standard error of the terminal.
+Interfaces provide a means to access and retrieve these outputs, by
+using the ``terminal_output`` attribute: ::
+
+  import nipype.interfaces.fsl as fsl
+  mybet = fsl.BET(from_file='bet-settings.json')
+  mybet.terminal_output = 'file_split'
+
+In the example, the ``terminal_output = 'file_split'`` will redirect the
+standard output and the standard error to split files (called
+``stdout.nipype`` and ``stderr.nipype`` respectively).
+The possible values for ``terminal_output`` are:
+
+*file*
+    Redirects both standard output and standard error to the same file
+    called ``output.nipype``.
+    Messages from both streams will be overlapped as they arrive to
+    the file.
+
+*file_split*
+    Redirects the output streams separately, to ``stdout.nipype``
+    and ``stderr.nipype`` respectively, as described in the example.
+
+*file_stdout*
+    Only the standard output will be redirected to ``stdout.nipype``
+    and the standard error will be discarded.
+
+*file_stderr*
+    Only the standard error will be redirected to ``stderr.nipype``
+    and the standard output will be discarded.
+
+*stream*
+    Both output streams are redirected to the current logger printing
+    their messages interleaved and immediately to the terminal.
+
+*allatonce*
+    Both output streams will be forwarded to a buffer and stored
+    separately in the `runtime` object that the `run()` method returns.
+    No files are written nor streams printed out to terminal.
+
+*none*
+    Both outputs are discarded
+
+In all cases, except for the ``'none'`` setting of ``terminal_output``,
+the ``run()`` method will return a "runtime" object that will contain
+the streams in the corresponding properties (``runtime.stdout``
+for the standard output, ``runtime.stderr`` for the standard error, and
+``runtime.merged`` for both when streams are mixed, eg. when using the
+*file* option). ::
+
+  import nipype.interfaces.fsl as fsl
+  mybet = fsl.BET(from_file='bet-settings.json')
+  mybet.terminal_output = 'file_split'
+  ...
+  result = mybet.run()
+  result.runtime.stdout
+  ' ... captured standard output ...'
+
+
+
 Traited Attributes
 ------------------
 

doc/users/config_file.rst

@@ -14,93 +14,100 @@ Logging
 ~~~~~~~
 
 *workflow_level*
-	How detailed the logs regarding workflow should be (possible values:
-	``INFO`` and ``DEBUG``; default value: ``INFO``)
-*filemanip_level*
-	How detailed the logs regarding file operations (for example overwriting
-	warning) should be (possible values: ``INFO`` and ``DEBUG``; default value:
-	``INFO``)
+    How detailed the logs regarding workflow should be (possible values:
+    ``INFO`` and ``DEBUG``; default value: ``INFO``)
+*utils_level*
+    How detailed the logs regarding nipype utils, like file operations
+    (for example overwriting warning) or the resource profiler, should be
+    (possible values: ``INFO`` and ``DEBUG``; default value:
+    ``INFO``)
 *interface_level*
-	How detailed the logs regarding interface execution should be (possible
-	values: ``INFO`` and ``DEBUG``; default value: ``INFO``)
+    How detailed the logs regarding interface execution should be (possible
+    values: ``INFO`` and ``DEBUG``; default value: ``INFO``)
+*filemanip_level* (deprecated as of 1.0)
+    How detailed the logs regarding file operations (for example overwriting
+    warning) should be (possible values: ``INFO`` and ``DEBUG``)
 *log_to_file*
     Indicates whether logging should also send the output to a file (possible
     values: ``true`` and ``false``; default value: ``false``)
 *log_directory*
-	Where to store logs. (string, default value: home directory)
+    Where to store logs. (string, default value: home directory)
 *log_size*
-	Size of a single log file. (integer, default value: 254000)
+    Size of a single log file. (integer, default value: 254000)
 *log_rotate*
-	How many rotation should the log file make. (integer, default value: 4)
+    How many rotation should the log file make. (integer, default value: 4)
 
 Execution
 ~~~~~~~~~
 
 *plugin*
-	This defines which execution plugin to use. (possible values: ``Linear``,
-	``MultiProc``, ``SGE``, ``IPython``; default value: ``Linear``)
+    This defines which execution plugin to use. (possible values: ``Linear``,
+    ``MultiProc``, ``SGE``, ``IPython``; default value: ``Linear``)
 
 *stop_on_first_crash*
-	Should the workflow stop upon first node crashing or try to execute as many
-	nodes as possible? (possible values: ``true`` and ``false``; default value:
-	``false``)
+    Should the workflow stop upon first node crashing or try to execute as many
+    nodes as possible? (possible values: ``true`` and ``false``; default value:
+    ``false``)
 
 *stop_on_first_rerun*
-	Should the workflow stop upon first node trying to recompute (by that we
-	mean rerunning a node that has been run before - this can happen due changed
-	inputs and/or hash_method since the last run). (possible values: ``true``
-	and ``false``; default value: ``false``)
+    Should the workflow stop upon first node trying to recompute (by that we
+    mean rerunning a node that has been run before - this can happen due changed
+    inputs and/or hash_method since the last run). (possible values: ``true``
+    and ``false``; default value: ``false``)
 
 *hash_method*
-	Should the input files be checked for changes using their content (slow, but
-	100% accurate) or just their size and modification date (fast, but
-	potentially prone to errors)? (possible values: ``content`` and
-	``timestamp``; default value: ``timestamp``)
+    Should the input files be checked for changes using their content (slow, but
+    100% accurate) or just their size and modification date (fast, but
+    potentially prone to errors)? (possible values: ``content`` and
+    ``timestamp``; default value: ``timestamp``)
 
 *keep_inputs*
     Ensures that all inputs that are created in the nodes working directory are
     kept after node execution (possible values: ``true`` and ``false``; default
     value: ``false``)
 
 *single_thread_matlab*
-	Should all of the Matlab interfaces (including SPM) use only one thread?
-	This is useful if you are parallelizing your workflow using MultiProc or
-	IPython on a single multicore machine. (possible values: ``true`` and
-	``false``; default value: ``true``)
+    Should all of the Matlab interfaces (including SPM) use only one thread?
+    This is useful if you are parallelizing your workflow using MultiProc or
+    IPython on a single multicore machine. (possible values: ``true`` and
+    ``false``; default value: ``true``)
 
 *display_variable*
-	What ``DISPLAY`` variable should all command line interfaces be
-	run with. This is useful if you are using `xnest
-	<http://www.x.org/archive/X11R7.5/doc/man/man1/Xnest.1.html>`_
-	or `Xvfb <http://www.x.org/archive/X11R6.8.1/doc/Xvfb.1.html>`_
-	and you would like to redirect all spawned windows to
-	it. (possible values: any X server address; default value: not
-	set)
+	Override the ``$DISPLAY`` environment variable for interfaces that require
+    an X server. This option is useful if there is a running X server, but 
+    ``$DISPLAY`` was not defined in nipype's environment. For example, if an X 
+    server is listening on the default port of 6000, set ``display_variable = :0``
+    to enable nipype interfaces to use it. It may also point to displays provided 
+    by VNC, `xnest <http://www.x.org/archive/X11R7.5/doc/man/man1/Xnest.1.html>`_ 
+    or `Xvfb <http://www.x.org/archive/X11R6.8.1/doc/Xvfb.1.html>`_.
+    If neither ``display_variable`` nor the ``$DISPLAY`` environment variable are
+    set, nipype will try to configure a new virtual server using Xvfb.
+    (possible values: any X server address; default value: not set)
 
 *remove_unnecessary_outputs*
-	This will remove any interface outputs not needed by the workflow. If the
-	required outputs from a node changes, rerunning the workflow will rerun the
-	node. Outputs of leaf nodes (nodes whose outputs are not connected to any
-	other nodes) will never be deleted independent of this parameter. (possible
-	values: ``true`` and ``false``; default value: ``true``)
+    This will remove any interface outputs not needed by the workflow. If the
+    required outputs from a node changes, rerunning the workflow will rerun the
+    node. Outputs of leaf nodes (nodes whose outputs are not connected to any
+    other nodes) will never be deleted independent of this parameter. (possible
+    values: ``true`` and ``false``; default value: ``true``)
 
 *try_hard_link_datasink*
-	When the DataSink is used to produce an orginized output file outside
-	of nipypes internal cache structure, a file system hard link will be
-	attempted first. A hard link allow multiple file paths to point to the
-	same physical storage location on disk if the conditions allow. By
-	refering to the same physical file on disk (instead of copying files
-	byte-by-byte) we can avoid unnecessary data duplication.  If hard links
-	are not supported for the source or destination paths specified, then
-	a standard byte-by-byte copy is used.  (possible values: ``true`` and
-	``false``; default value: ``true``)
+    When the DataSink is used to produce an orginized output file outside
+    of nipypes internal cache structure, a file system hard link will be
+    attempted first. A hard link allow multiple file paths to point to the
+    same physical storage location on disk if the conditions allow. By
+    refering to the same physical file on disk (instead of copying files
+    byte-by-byte) we can avoid unnecessary data duplication.  If hard links
+    are not supported for the source or destination paths specified, then
+    a standard byte-by-byte copy is used.  (possible values: ``true`` and
+    ``false``; default value: ``true``)
 
 *use_relative_paths*
-	Should the paths stored in results (and used to look for inputs)
-	be relative or absolute. Relative paths allow moving the whole
-	working directory around but may cause problems with
-	symlinks. (possible values: ``true`` and ``false``; default
-	value: ``false``)
+    Should the paths stored in results (and used to look for inputs)
+    be relative or absolute. Relative paths allow moving the whole
+    working directory around but may cause problems with
+    symlinks. (possible values: ``true`` and ``false``; default
+    value: ``false``)
 
 *local_hash_check*
     Perform the hash check on the job submission machine. This option minimizes
@@ -115,10 +122,10 @@ Execution
     done after a job finish is detected. (float in seconds; default value: 5)
 
 *remove_node_directories (EXPERIMENTAL)*
-	Removes directories whose outputs have already been used
-	up. Doesn't work with IdentiInterface or any node that patches
-	data through (without copying) (possible values: ``true`` and
-	``false``; default value: ``false``)
+    Removes directories whose outputs have already been used
+    up. Doesn't work with IdentiInterface or any node that patches
+    data through (without copying) (possible values: ``true`` and
+    ``false``; default value: ``false``)
 
 *stop_on_unknown_version*
     If this is set to True, an underlying interface will raise an error, when no
@@ -146,18 +153,27 @@ Execution
     crashfiles allow portability across machines and shorter load time.
     (possible values: ``pklz`` and ``txt``; default value: ``pklz``)
 
+*resource_monitor*
+    Enables monitoring the resources occupation (possible values: ``true`` and
+    ``false``; default value: ``false``)
+
+*resource_monitor_frequency*
+    Sampling period (in seconds) between measurements of resources (memory, cpus)
+    being used by an interface. Requires ``resource_monitor`` to be ``true``.
+    (default value: ``1``)
+
 Example
 ~~~~~~~
 
 ::
 
-	[logging]
-	workflow_level = DEBUG
+    [logging]
+    workflow_level = DEBUG
 
-	[execution]
-	stop_on_first_crash = true
-	hash_method = timestamp
-	display_variable = :1
+    [execution]
+    stop_on_first_crash = true
+    hash_method = timestamp
+    display_variable = :1
 
 Workflow.config property has a form of a nested dictionary reflecting the
 structure of the .cfg file.

doc/users/install.rst

@@ -47,7 +47,7 @@ use the following command::
 While `all` installs everything, one can also install select components as
 listed below::
 
-    'doc': ['Sphinx>=1.4', 'matplotlib', 'pydotplus'],
+    'doc': ['Sphinx>=1.4', 'matplotlib', 'pydotplus', 'pydot>=1.2.3'],
     'tests': ['pytest-cov', 'codecov'],
     'nipy': ['nitime', 'nilearn', 'dipy', 'nipy', 'matplotlib'],
     'profiler': ['psutil'],

doc/users/interface_tutorial.rst

@@ -10,7 +10,7 @@ Specifying input settings
 The nipype interface modules provide a Python interface to external
 packages like FSL_ and SPM_.  Within the module are a series of Python
 classes which wrap specific package functionality.  For example, in
-the fsl module, the class :class:`nipype.interfaces.fsl.Bet` wraps the
+the fsl module, the class :class:`nipype.interfaces.fsl.BET` wraps the
 ``bet`` command-line tool.  Using the command-line tool, one would
 specify input settings using flags like ``-o``, ``-m``, ``-f <f>``, etc...
 However, in nipype, options are assigned to Python attributes and can