Merge branch 'master' into fix_tutorials

Author: Mathieu Dubois

CHANGES

@@ -1,6 +1,10 @@
 Next release
 ============
 
+* ENH: Inputs with name_source can be now chained in cascade (https://github.com/nipy/nipype/pull/938)
+* ENH: Improve JSON interfaces: default settings when reading and consistent output creation
+       when writing (https://github.com/nipy/nipype/pull/1047)
+* FIX: AddCSVRow problems when using infields (https://github.com/nipy/nipype/pull/1028)
 * FIX: Removed unused ANTS registration flag (https://github.com/nipy/nipype/pull/999)
 * FIX: Amend create_tbss_non_fa() workflow to match FSL's tbss_non_fa command. (https://github.com/nipy/nipype/pull/1033)
 * FIX: remove unused mandatory flag from spm normalize (https://github.com/nipy/nipype/pull/1048)

doc/devel/interface_specs.rst

@@ -358,6 +358,9 @@ CommandLine
 ``name_source``
     Indicates the list of input fields from which the value of the current File
     output variable will be drawn. This input field must be the name of a File.
+    Chaining is allowed, meaning that an input field can point to another as
+    ``name_source``, which also points as ``name_source`` to a third field.
+    In this situation, the templates for substitutions are also accumulated.
 
 ``name_template``
     By default a ``%s_generated`` template is used to create the output

examples/rsfmri_vol_surface_preprocessing.py

@@ -230,7 +230,7 @@ def build_filter1(motion_params, comp_norm, outliers, detrend_poly=None):
             out_params = np.hstack((out_params, outlier_vector))
         if detrend_poly:
             timepoints = out_params.shape[0]
-            X = np.ones((timepoints, 1))
+            X = np.empty((timepoints, 0))
             for i in range(detrend_poly):
                 X = np.hstack((X, legendre(
                     i + 1)(np.linspace(-1, 1, timepoints))[:, None]))
@@ -542,7 +542,7 @@ def create_reg_workflow(name='registration'):
 
     warpmean = Node(ants.ApplyTransforms(), name='warpmean')
     warpmean.inputs.input_image_type = 3
-    warpmean.inputs.interpolation = 'BSpline'
+    warpmean.inputs.interpolation = 'Linear'
     warpmean.inputs.invert_transform_flags = [False, False]
     warpmean.inputs.terminal_output = 'file'
     warpmean.inputs.args = '--float'
@@ -771,7 +771,7 @@ def merge_files(in1, in2):
     warpall = MapNode(ants.ApplyTransforms(), iterfield=['input_image'],
                       name='warpall')
     warpall.inputs.input_image_type = 3
-    warpall.inputs.interpolation = 'BSpline'
+    warpall.inputs.interpolation = 'Linear'
     warpall.inputs.invert_transform_flags = [False, False]
     warpall.inputs.terminal_output = 'file'
     warpall.inputs.reference_image = target_file
@@ -1003,9 +1003,11 @@ def create_resting_workflow(args, name=None):
     parser.add_argument('--surf_fwhm', default=15., dest='surf_fwhm',
                         type=float, help="Spatial FWHM" + defstr)
     parser.add_argument("-l", "--lowpass_freq", dest="lowpass_freq",
-                        default=0.1, help="Low pass frequency (Hz)" + defstr)
+                        default=0.1, type=float,
+                        help="Low pass frequency (Hz)" + defstr)
     parser.add_argument("-u", "--highpass_freq", dest="highpass_freq",
-                        default=0.01, help="High pass frequency (Hz)" + defstr)
+                        default=0.01, type=float,
+                        help="High pass frequency (Hz)" + defstr)
     parser.add_argument("-o", "--output_dir", dest="sink",
                         help="Output directory base", required=True)
     parser.add_argument("-w", "--work_dir", dest="work_dir",

examples/rsfmri_vol_surface_preprocessing_nipy.py

@@ -226,7 +226,7 @@ def build_filter1(motion_params, comp_norm, outliers, detrend_poly=None):
             out_params = np.hstack((out_params, outlier_vector))
         if detrend_poly:
             timepoints = out_params.shape[0]
-            X = np.ones((timepoints, 1))
+            X = np.empty((timepoints, 0))
             for i in range(detrend_poly):
                 X = np.hstack((X, legendre(
                     i + 1)(np.linspace(-1, 1, timepoints))[:, None]))
@@ -538,7 +538,7 @@ def create_reg_workflow(name='registration'):
 
     warpmean = Node(ants.ApplyTransforms(), name='warpmean')
     warpmean.inputs.input_image_type = 3
-    warpmean.inputs.interpolation = 'BSpline'
+    warpmean.inputs.interpolation = 'Linear'
     warpmean.inputs.invert_transform_flags = [False, False]
     warpmean.inputs.terminal_output = 'file'
     warpmean.inputs.args = '--float'
@@ -759,7 +759,7 @@ def merge_files(in1, in2):
     warpall = MapNode(ants.ApplyTransforms(), iterfield=['input_image'],
                       name='warpall')
     warpall.inputs.input_image_type = 3
-    warpall.inputs.interpolation = 'BSpline'
+    warpall.inputs.interpolation = 'Linear'
     warpall.inputs.invert_transform_flags = [False, False]
     warpall.inputs.terminal_output = 'file'
     warpall.inputs.reference_image = target_file
@@ -1006,9 +1006,11 @@ def create_resting_workflow(args, name=None):
     parser.add_argument('--surf_fwhm', default=15., dest='surf_fwhm',
                         type=float, help="Spatial FWHM" + defstr)
     parser.add_argument("-l", "--lowpass_freq", dest="lowpass_freq",
-                        default=0.1, help="Low pass frequency (Hz)" + defstr)
+                        default=0.1, type=float,
+                        help="Low pass frequency (Hz)" + defstr)
     parser.add_argument("-u", "--highpass_freq", dest="highpass_freq",
-                        default=0.01, help="High pass frequency (Hz)" + defstr)
+                        default=0.01, type=float,
+                        help="High pass frequency (Hz)" + defstr)
     parser.add_argument("-o", "--output_dir", dest="sink",
                         help="Output directory base", required=True)
     parser.add_argument("-w", "--work_dir", dest="work_dir",

nipype/algorithms/misc.py

@@ -34,7 +34,7 @@
 from ..interfaces.base import (BaseInterface, traits, TraitedSpec, File,
                                InputMultiPath, OutputMultiPath,
                                BaseInterfaceInputSpec, isdefined,
-                               DynamicTraitedSpec)
+                               DynamicTraitedSpec, Undefined)
 from nipype.utils.filemanip import fname_presuffix, split_filename
 iflogger = logging.getLogger('interface')
 
@@ -785,7 +785,8 @@ def _list_outputs(self):
 
 
 class AddCSVRowInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec):
-    in_file = traits.File(mandatory=True, desc='Input comma-separated value (CSV) files')
+    in_file = traits.File(mandatory=True,
+                          desc='Input comma-separated value (CSV) files')
     _outputs = traits.Dict(traits.Any, value={}, usedefault=True)
 
     def __setattr__(self, key, value):
@@ -804,13 +805,15 @@ class AddCSVRowOutputSpec(TraitedSpec):
 
 
 class AddCSVRow(BaseInterface):
+
     """Simple interface to add an extra row to a csv file
 
     .. note:: Requires `pandas <http://pandas.pydata.org/>`_
 
     .. warning:: Multi-platform thread-safe execution is possible with
-        `lockfile <https://pythonhosted.org/lockfile/lockfile.html>`_. Please recall that (1)
-        this module is alpha software; and (2) it should be installed for thread-safe writing.
+        `lockfile <https://pythonhosted.org/lockfile/lockfile.html>`_. Please
+        recall that (1) this module is alpha software; and (2) it should be
+        installed for thread-safe writing.
         If lockfile is not installed, then the interface is not thread-safe.
 
 
@@ -822,7 +825,7 @@ class AddCSVRow(BaseInterface):
     >>> addrow.inputs.in_file = 'scores.csv'
     >>> addrow.inputs.si = 0.74
     >>> addrow.inputs.di = 0.93
-    >>> addrow.subject_id = 'S400'
+    >>> addrow.inputs.subject_id = 'S400'
     >>> addrow.inputs.list_of_values = [ 0.4, 0.7, 0.3 ]
     >>> addrow.run() # doctest: +SKIP
     """
@@ -850,22 +853,26 @@ def _run_interface(self, runtime):
         try:
             import pandas as pd
         except ImportError:
-            raise ImportError('This interface requires pandas (http://pandas.pydata.org/) to run.')
+            raise ImportError(('This interface requires pandas '
+                               '(http://pandas.pydata.org/) to run.'))
 
         try:
             import lockfile as pl
             self._have_lock = True
         except ImportError:
-            import warnings
-            warnings.warn(('Python module lockfile was not found: AddCSVRow will not be thread-safe '
-                          'in multi-processor execution'))
+            from warnings import warn
+            warn(('Python module lockfile was not found: AddCSVRow will not be'
+                  ' thread-safe in multi-processor execution'))
 
         input_dict = {}
         for key, val in self.inputs._outputs.items():
             # expand lists to several columns
+            if key == 'trait_added' and val in self.inputs.copyable_trait_names():
+                continue
+
             if isinstance(val, list):
-                for i,v in enumerate(val):
-                    input_dict['%s_%d' % (key,i)]=v
+                for i, v in enumerate(val):
+                    input_dict['%s_%d' % (key, i)] = v
             else:
                 input_dict[key] = val
 
@@ -887,6 +894,13 @@ def _run_interface(self, runtime):
         if self._have_lock:
             self._lock.release()
 
+        # Using nipype.external.portalocker this might be something like:
+        # with pl.Lock(self.inputs.in_file, timeout=1) as fh:
+        #     if op.exists(fh):
+        #         formerdf = pd.read_csv(fh, index_col=0)
+        #         df = pd.concat([formerdf, df], ignore_index=True)
+        #         df.to_csv(fh)
+
         return runtime
 
     def _list_outputs(self):

nipype/interfaces/ants/registration.py

@@ -394,8 +394,15 @@ class Registration(ANTSCommand):
     >>> reg4.inputs.collapse_output_transforms = True
     >>> outputs = reg4._list_outputs()
     >>> print outputs #doctest: +ELLIPSIS
-    {'reverse_invert_flags': [True, False], 'inverse_composite_transform': ['.../nipype/testing/data/output_InverseComposite.h5'], 'warped_image': '.../nipype/testing/data/output_warped_image.nii.gz', 'inverse_warped_image': <undefined>, 'forward_invert_flags': [False, False], 'reverse_transforms': ['.../nipype/testing/data/output_0GenericAffine.mat', '.../nipype/testing/data/output_1InverseWarp.nii.gz'], 'composite_transform': ['.../nipype/testing/data/output_Composite.h5'], 'forward_transforms': ['.../nipype/testing/data/output_0GenericAffine.mat', '.../nipype/testing/data/output_1Warp.nii.gz']}
-    >>> reg4.aggregate_outputs() #doctest: +SKIP
+    {'reverse_invert_flags': [], 'inverse_composite_transform': ['.../nipype/testing/data/output_InverseComposite.h5'], 'warped_image': '.../nipype/testing/data/output_warped_image.nii.gz', 'inverse_warped_image': <undefined>, 'forward_invert_flags': [], 'reverse_transforms': [], 'composite_transform': ['.../nipype/testing/data/output_Composite.h5'], 'forward_transforms': []}
+    
+    >>> # Test collapse transforms flag
+    >>> reg4b = copy.deepcopy(reg4)
+    >>> reg4b.inputs.write_composite_transform = False
+    >>> outputs = reg4b._list_outputs()
+    >>> print outputs #doctest: +ELLIPSIS
+    {'reverse_invert_flags': [True, False], 'inverse_composite_transform': <undefined>, 'warped_image': '.../nipype/testing/data/output_warped_image.nii.gz', 'inverse_warped_image': <undefined>, 'forward_invert_flags': [False, False], 'reverse_transforms': ['.../nipype/testing/data/output_0GenericAffine.mat', '.../nipype/testing/data/output_1InverseWarp.nii.gz'], 'composite_transform': <undefined>, 'forward_transforms': ['.../nipype/testing/data/output_0GenericAffine.mat', '.../nipype/testing/data/output_1Warp.nii.gz']}
+    >>> reg4b.aggregate_outputs() #doctest: +SKIP
 
     >>> # Test multiple metrics per stage
     >>> reg5 = copy.deepcopy(reg)
@@ -709,7 +716,7 @@ def _list_outputs(self):
                 outputs[
                     'reverse_invert_flags'].insert(0, reverseInverseMode)
                 transformCount += 1
-        else:
+        elif not self.inputs.write_composite_transform:
             transformCount = 0
             isLinear = [any(self._linear_transform_names == t)
                         for t in self.inputs.transforms]

nipype/interfaces/base.py

@@ -47,6 +47,12 @@
 
 __docformat__ = 'restructuredtext'
 
+class NipypeInterfaceError(Exception):
+    def __init__(self, value):
+        self.value = value
+    def __str__(self):
+        return repr(self.value)
+
 def _lock_files():
     tmpdir = '/tmp'
     pattern = '.X*-lock'
@@ -1510,9 +1516,13 @@ def _format_arg(self, name, trait_spec, value):
             # Append options using format string.
             return argstr % value
 
-    def _filename_from_source(self, name):
+    def _filename_from_source(self, name, chain=None):
+        if chain is None:
+            chain = []
+
         trait_spec = self.inputs.trait(name)
         retval = getattr(self.inputs, name)
+
         if not isdefined(retval) or "%s" in retval:
             if not trait_spec.name_source:
                 return retval
@@ -1522,26 +1532,42 @@ def _filename_from_source(self, name):
                 name_template = trait_spec.name_template
             if not name_template:
                 name_template = "%s_generated"
-            if isinstance(trait_spec.name_source, list):
-                for ns in trait_spec.name_source:
-                    if isdefined(getattr(self.inputs, ns)):
-                        name_source = ns
-                        break
+
+            ns = trait_spec.name_source
+            while isinstance(ns, list):
+                if len(ns) > 1:
+                    iflogger.warn('Only one name_source per trait is allowed')
+                ns = ns[0]
+
+            if not isinstance(ns, six.string_types):
+                raise ValueError(('name_source of \'%s\' trait sould be an '
+                                 'input trait name') % name)
+
+            if isdefined(getattr(self.inputs, ns)):
+                name_source = ns
+                source = getattr(self.inputs, name_source)
+                while isinstance(source, list):
+                    source = source[0]
+
+                # special treatment for files
+                try:
+                    _, base, _ = split_filename(source)
+                except AttributeError:
+                    base = source
             else:
-                name_source = trait_spec.name_source
-            source = getattr(self.inputs, name_source)
-            while isinstance(source, list):
-                source = source[0]
-            #special treatment for files
-            try:
-                _, base, _ = split_filename(source)
-            except AttributeError:
-                base = source
+                if name in chain:
+                    raise NipypeInterfaceError('Mutually pointing name_sources')
+
+                chain.append(name)
+                base = self._filename_from_source(ns, chain)
+
+            chain = None
             retval = name_template % base
             _, _, ext = split_filename(retval)
             if trait_spec.keep_extension and ext:
                 return retval
             return self._overload_extension(retval, name)
+
         return retval
 
     def _gen_filename(self, name):
@@ -1557,7 +1583,7 @@ def _list_outputs(self):
             outputs = self.output_spec().get()
             for name, trait_spec in traits.iteritems():
                 out_name = name
-                if trait_spec.output_name != None:
+                if trait_spec.output_name is not None:
                     out_name = trait_spec.output_name
                 outputs[out_name] = \
                     os.path.abspath(self._filename_from_source(name))