resolved conflict

Author: satra

nipype/algorithms/misc.py

@@ -562,8 +562,12 @@ class SelectFilesInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec):
         desc="When matching mutliple files, return them in sorted order.")
     raise_on_empty = traits.Bool(True, usedefault=True,
         desc="Raise an exception if a template pattern matches no files.")
-    force_lists = traits.Bool(False, usedefault=True,
-        desc="Return all values as lists even when matching a single file.")
+    force_lists = traits.Either(traits.Bool(), traits.List(traits.Str()),
+        default=False, usedefault=True,
+        desc=("Whether to return outputs as a list even when only one file "
+              "matches the template. Either a boolean that applies to all "
+              "output fields or a list of output field names to coerce to "
+              " a list"))
 
 
 class SelectFiles(IOBase):
@@ -646,6 +650,18 @@ def _list_outputs(self):
         info = dict([(k, v) for k, v in self.inputs.__dict__.items()
                      if k in self._infields])
 
+        force_lists = self.inputs.force_lists
+        if isinstance(force_lists, bool):
+            force_lists = self._outfields if force_lists else []
+        bad_fields = set(force_lists) - set(self._outfields)
+        if bad_fields:
+            bad_fields = ", ".join(list(bad_fields))
+            plural = "s" if len(bad_fields) > 1 else ""
+            verb = "were" if len(bad_fields) > 1 else "was"
+            msg = ("The field%s '%s' %s set in 'force_lists' and not in "
+                   "'templates'.") % (plural, bad_fields, verb)
+            raise ValueError(msg)
+
         for field, template in self._templates.iteritems():
 
             # Build the full template path
@@ -673,7 +689,7 @@ def _list_outputs(self):
                 filelist.sort()
 
             # Handle whether this must be a list or not
-            if not self.inputs.force_lists:
+            if field not in force_lists:
                 filelist = list_to_filename(filelist)
 
             outputs[field] = filelist

nipype/interfaces/fsl/preprocess.py

@@ -20,7 +20,7 @@
     warnings.warn('nipy not installed')
 else:
     import nipy.modalities.fmri.design_matrix as dm
-    import nipy.labs.glm as GLM
+    import nipy.labs.glm.glm as GLM
 
 if have_nipy:
     try:

nipype/interfaces/io.py

@@ -6,6 +6,7 @@
 import os.path as op
 from tempfile import mkstemp, mkdtemp
 
+from nose.tools import assert_raises
 import nipype
 from nipype.testing import assert_equal, assert_true, assert_false
 import nipype.interfaces.io as nio
@@ -43,6 +44,16 @@ def test_selectfiles():
                      "interfaces/spm/preprocess.py")
     yield assert_equal, res.outputs.preprocess, [wanted]
 
+    dg.inputs.package = "fsl"
+    dg.inputs.force_lists = ["model"]
+    res = dg.run()
+    preproc = op.join(op.dirname(nipype.__file__),
+                      "interfaces/fsl/preprocess.py")
+    model = [op.join(op.dirname(nipype.__file__),
+                     "interfaces/fsl/model.py")]
+    yield assert_equal, res.outputs.preprocess, preproc
+    yield assert_equal, res.outputs.model, model
+
     templates = {"converter": "interfaces/dcm{to!s}nii.py"}
     dg = nio.SelectFiles(templates, base_directory=base_dir)
     dg.inputs.to = 2
@@ -51,6 +62,17 @@ def test_selectfiles():
     yield assert_equal, res.outputs.converter, wanted
 
 
+def test_selectfiles_valueerror():
+    """Test ValueError when force_lists has field that isn't in template."""
+    base_dir = op.dirname(nipype.__file__)
+    templates = {"model": "interfaces/{package}/model.py",
+                 "preprocess": "interfaces/{package}/pre*.py"}
+    force_lists = ["model", "preprocess", "registration"]
+    sf = nio.SelectFiles(templates, base_directory=base_dir,
+                         force_lists=force_lists)
+    yield assert_raises, ValueError, sf.run
+
+
 def test_datagrabber_order():
     tempdir = mkdtemp()
     file1 = mkstemp(prefix='sub002_L1_R1.q', dir=tempdir)

nipype/interfaces/nipy/model.py

@@ -1209,6 +1209,11 @@ def hash_exists(self, updatehash=False):
         # of the dictionary itself.
         hashed_inputs, hashvalue = self._get_hashval()
         outdir = self.output_dir()
+        hashfiles = glob(os.path.join(outdir, '_0x*.json'))
+        if len(hashfiles) > 1:
+            warn('Removing multiple hashfiles and forcing node to rerun')
+            for hashfile in hashfiles:
+                os.unlink(hashfile)
         hashfile = os.path.join(outdir, '_0x%s.json' % hashvalue)
         if updatehash and os.path.exists(outdir):
             logger.debug("Updating hash: %s" % hashvalue)
@@ -1302,6 +1307,10 @@ def run(self, updatehash=False):
                 logger.debug(("%s found and can_resume is True or Node is a "
                               "MapNode - resuming execution") %
                              hashfile_unfinished)
+                if isinstance(self, MapNode):
+                    # remove old json files
+                    for filename in glob(os.path.join(outdir, '_0x*.json')):
+                        os.unlink(filename)
             outdir = make_output_dir(outdir)
             self._save_hashfile(hashfile_unfinished, hashed_inputs)
             self.write_report(report_type='preexec', cwd=outdir)

nipype/interfaces/tests/test_io.py

@@ -3,6 +3,7 @@
 """Tests for the engine module
 """
 from copy import deepcopy
+from glob import glob
 import os
 from shutil import rmtree
 from tempfile import mkdtemp
@@ -13,6 +14,7 @@
                             assert_false)
 import nipype.interfaces.base as nib
 import nipype.pipeline.engine as pe
+from nipype import logging
 
 class InputSpec(nib.TraitedSpec):
     input1 = nib.traits.Int(desc='a random int')
@@ -439,3 +441,47 @@ def func2(a):
     yield assert_false, error_raised
     os.chdir(cwd)
     rmtree(wd)
+
+
+def test_mapnode_json():
+    """Tests that mapnodes don't generate excess jsons
+    """
+    cwd = os.getcwd()
+    wd = mkdtemp()
+    os.chdir(wd)
+    from nipype import MapNode, Function, Workflow
+    def func1(in1):
+        return in1 + 1
+    n1 = MapNode(Function(input_names=['in1'],
+                          output_names=['out'],
+                          function=func1),
+                 iterfield=['in1'],
+                 name='n1')
+    n1.inputs.in1 = [1]
+    w1 = Workflow(name='test')
+    w1.base_dir = wd
+    w1.config = {'crashdump_dir': wd}
+    w1.add_nodes([n1])
+    w1.run()
+    n1.inputs.in1 = [2]
+    w1.run()
+    # should rerun
+    n1.inputs.in1 = [1]
+    eg = w1.run()
+
+    node = eg.nodes()[0]
+    outjson = glob(os.path.join(node.output_dir(), '_0x*.json'))
+    yield assert_equal, len(outjson), 1
+
+    # check that multiple json's don't trigger rerun
+    with open(os.path.join(node.output_dir(), 'test.json'), 'wt') as fp:
+        fp.write('dummy file')
+    w1.config['execution'].update(**{'stop_on_first_rerun': True})
+    error_raised = False
+    try:
+        w1.run()
+    except:
+        error_raised = True
+    yield assert_false, error_raised
+    os.chdir(cwd)
+    rmtree(wd)