Merge branch 'master' into enh/MRTrix3

Conflicts:
	CHANGES

Author: oesteban

CHANGES

@@ -2,7 +2,10 @@ Next release
 ============
 
 * ENH: New interfaces for MRTrix3 (https://github.com/nipy/nipype/pull/1126)
+* ENH: Added interface to simulate DWIs using the multi-tensor model
+       (https://github.com/nipy/nipype/pull/1085)
 * ENH: New interface for FSL fslcpgeom utility (https://github.com/nipy/nipype/pull/1152)
+* ENH: Added SLURMGraph plugin for submitting jobs to SLURM with dependencies (https://github.com/nipy/nipype/pull/1136)
 * FIX: Enable absolute path definitions in DCMStack (https://github.com/nipy/nipype/pull/1089,
        replaced by https://github.com/nipy/nipype/pull/1093)
 * ENH: New mesh.MeshWarpMaths to operate on surface-defined warpings

doc/users/plugins.rst

@@ -139,9 +139,11 @@ particular node might use more resources than other nodes in a workflow.
 	 
 SGEGraph
 ~~~~~~~~
-SGEGraph_ is a exuction plugin working with Sun Grid Engine that allows for
+SGEGraph_ is an execution plugin working with Sun Grid Engine that allows for
 submitting entire graph of dependent jobs at once. This way Nipype does not 
-need to run a monitoring process - SGE takes care of this.
+need to run a monitoring process - SGE takes care of this. The use of SGEGraph_
+is preferred over SGE_ since the latter adds unnecessary load on the submit 
+machine.
 
 .. note::
 
@@ -175,6 +177,26 @@ Optional arguments::
 
   template: custom template file to use
   sbatch_args: any other command line args to be passed to bsub.
+  
+  
+SLURMGraph
+~~~~~~~~~~
+SLURMGraph_ is an execution plugin working with SLURM that allows for
+submitting entire graph of dependent jobs at once. This way Nipype does not 
+need to run a monitoring process - SLURM takes care of this. The use of SLURMGraph_
+plugin is preferred over the vanilla SLURM_ plugin since the latter adds 
+unnecessary load on the submit machine. 
+
+
+.. note::
+
+  When rerunning unfinished workflows using SLURMGraph you may decide not to 
+  submit jobs for Nodes that previously finished running. This can speed up
+  execution, but new or modified inputs that would previously trigger a Node
+  to rerun will be ignored. The following option turns on this functionality::
+
+     workflow.run(plugin='SLURMGraph', plugin_args = {'dont_resubmit_completed_jobs': True})
+
 
 HTCondor
 --------
@@ -183,12 +205,12 @@ DAGMan
 ~~~~~~
 
 With its DAGMan_ component HTCondor_ (previously Condor) allows for submitting
-entire graphs of dependent jobs at once (similar to SGEGraph_). With the ``CondorDAGMan`` plug-in
-Nipype can utilize this functionality to submit complete workflows directly and
-in a single step.  Consequently, and in contrast to other plug-ins, workflow
-execution returns almost instantaneously -- Nipype is only used to generate the
-workflow graph, while job scheduling and dependency resolution are entirely
-managed by HTCondor_.
+entire graphs of dependent jobs at once (similar to SGEGraph_ and SLURMGaaoh_). 
+With the ``CondorDAGMan`` plug-in Nipype can utilize this functionality to 
+submit complete workflows directly and in a single step.  Consequently, and 
+in contrast to other plug-ins, workflow execution returns almost 
+instantaneously -- Nipype is only used to generate the workflow graph, 
+while job scheduling and dependency resolution are entirely managed by HTCondor_.
 
 Please note that although DAGMan_ supports specification of data dependencies
 as well as data provisioning on compute nodes this functionality is currently

nipype/interfaces/ants/registration.py

@@ -574,6 +574,7 @@ def _list_outputs(self):
                                                'brainvols.csv')
         return outputs
 
+
 class JointFusionInputSpec(ANTSCommandInputSpec):
     dimension = traits.Enum(3, 2, 4, argstr='%d', position=0, usedefault=True, mandatory=True,
                             desc='image dimension (2, 3, or 4)')
@@ -588,21 +589,23 @@ class JointFusionInputSpec(ANTSCommandInputSpec):
     patch_radius = traits.ListInt(minlen=3, maxlen=3, argstr='-rp %s', desc='Patch radius for similarity measures, scalar or vector. Default: 2x2x2')
     search_radius = traits.ListInt(minlen=3, maxlen=3, argstr='-rs %s', desc='Local search radius. Default: 3x3x3')
     exclusion_region = File(exists=True, argstr='-x %s', desc='Specify an exclusion region for the given label.')
-    output_posteriors_name_template = traits.Str('POSTERIOR_%02d.nii.gz', argstr='-p %s',
-                                                 desc="Save the posterior maps (probability that each voxel belongs to each " +\
-                                                 "label) as images. The number of images saved equals the number of labels. " +\
-                                                 "The filename pattern must be in C printf format, e.g. posterior%04d.nii.gz")
-    output_voting_weights_name_template = traits.Str('WEIGHTED_%04d.nii.gz', argstr='-w %s', desc="Save the voting weights as " +\
-                                                     "images. The number of images saved equals the number of atlases. The " +\
-                                                     "filename pattern must be in C printf format, e.g. weight%04d.nii.gz")
+    # TODO: These are almost never needed except for debugging
+    # output_posteriors_name_template = traits.Str('POSTERIOR_%02d.nii.gz', argstr='-p %s',
+    #                                              desc="Save the posterior maps (probability that each voxel belongs to each " +\
+    #                                              "label) as images. The number of images saved equals the number of labels. " +\
+    #                                              "The filename pattern must be in C printf format, e.g. posterior%04d.nii.gz")
+    # output_voting_weights_name_template = traits.Str('WEIGHTED_%04d.nii.gz', argstr='-w %s', desc="Save the voting weights as " +\
+    #                                                  "images. The number of images saved equals the number of atlases. The " +\
+    #                                                  "filename pattern must be in C printf format, e.g. weight%04d.nii.gz")
     atlas_group_id = traits.ListInt(argstr='-gp %d...', desc='Assign a group ID for each atlas')
     atlas_group_weights = traits.ListInt(argstr='-gpw %d...', desc='Assign the voting weights to each atlas group')
 
 
 class JointFusionOutputSpec(TraitedSpec):
     output_label_image = File(exists=True)
-    # TODO: optional outputs - output_posteriors, output_voting_weights
-
+    # TODO: These are almost never needed except for debugging,
+    #       so delay complicated implementation until a need arises
+    #       optional outputs - output_posteriors, output_voting_weights
 
 class JointFusion(ANTSCommand):
     """
@@ -616,22 +619,24 @@ class JointFusion(ANTSCommand):
     >>> at.inputs.method = 'Joint[0.1,2]'
     >>> at.inputs.output_label_image ='fusion_labelimage_output.nii'
     >>> at.inputs.warped_intensity_images = ['im1.nii',
-    ...                                      'im2.nii']
+    ...                                      'im2.nii',
+    ...                                      'im3.nii']
     >>> at.inputs.warped_label_images = ['segmentation0.nii.gz',
+    ...                                  'segmentation1.nii.gz',
     ...                                  'segmentation1.nii.gz']
     >>> at.inputs.target_image = 'T1.nii'
     >>> at.inputs.patch_radius = [3,2,1]
     >>> at.inputs.search_radius = [1,2,3]
     >>> at.cmdline
-    'jointfusion 3 1 -m Joint[0.1,2] -rp 3x2x1 -rs 1x2x3 -tg T1.nii -g im1.nii -g im2.nii -l segmentation0.nii.gz -l segmentation1.nii.gz fusion_labelimage_output.nii'
+    'jointfusion 3 1 -m Joint[0.1,2] -rp 3x2x1 -rs 1x2x3 -tg T1.nii -g im1.nii -g im2.nii -g im3.nii -l segmentation0.nii.gz -l segmentation1.nii.gz -l segmentation1.nii.gz fusion_labelimage_output.nii'
 
     Alternately, you can specify the voting method and parameters more 'Pythonically':
 
     >>> at.inputs.method = 'Joint'
     >>> at.inputs.alpha = 0.5
     >>> at.inputs.beta = 1
     >>> at.cmdline
-    'jointfusion 3 1 -m Joint[0.5,1] -rp 3x2x1 -rs 1x2x3 -tg T1.nii -g im1.nii -g im2.nii -l segmentation0.nii.gz -l segmentation1.nii.gz fusion_labelimage_output.nii'
+    'jointfusion 3 1 -m Joint[0.5,1] -rp 3x2x1 -rs 1x2x3 -tg T1.nii -g im1.nii -g im2.nii -g im3.nii -l segmentation0.nii.gz -l segmentation1.nii.gz -l segmentation1.nii.gz fusion_labelimage_output.nii'
     """
     input_spec = JointFusionInputSpec
     output_spec = JointFusionOutputSpec

nipype/interfaces/ants/segmentation.py

@@ -33,6 +33,9 @@ def test_Registration_inputs():
     initial_moving_transform_com=dict(argstr='%s',
     xor=['initial_moving_transform'],
     ),
+    initialize_transforms_per_stage=dict(argstr='--initialize-transforms-per-stage %d',
+    usedefault=True,
+    ),
     interpolation=dict(argstr='%s',
     usedefault=True,
     ),
@@ -70,6 +73,8 @@ def test_Registration_inputs():
     radius_or_number_of_bins=dict(requires=['metric_weight'],
     usedefault=True,
     ),
+    restore_state=dict(argstr='--restore-state %s',
+    ),
     sampling_percentage=dict(requires=['sampling_strategy'],
     ),
     sampling_percentage_item_trait=dict(),
@@ -78,6 +83,8 @@ def test_Registration_inputs():
     ),
     sampling_strategy_item_trait=dict(),
     sampling_strategy_stage_trait=dict(),
+    save_state=dict(argstr='--save-state %s',
+    ),
     shrink_factors=dict(mandatory=True,
     ),
     sigma_units=dict(requires=['smoothing_sigmas'],
@@ -117,6 +124,7 @@ def test_Registration_outputs():
     inverse_warped_image=dict(),
     reverse_invert_flags=dict(),
     reverse_transforms=dict(),
+    save_state=dict(),
     warped_image=dict(),
     )
     outputs = Registration.output_spec()

nipype/interfaces/ants/tests/test_auto_Registration.py

@@ -67,6 +67,7 @@ class BRAINSFitInputSpec(CommandLineInputSpec):
     NEVER_USE_THIS_FLAG_IT_IS_OUTDATED_02 = traits.Bool(desc="DO NOT USE THIS FLAG", argstr="--NEVER_USE_THIS_FLAG_IT_IS_OUTDATED_02 ")
     permitParameterVariation = InputMultiPath(traits.Int, desc="A bit vector to permit linear transform parameters to vary under optimization.  The vector order corresponds with transform parameters, and beyond the end ones fill in as a default.  For instance, you can choose to rotate only in x (pitch) with 1,0,0;  this is mostly for expert use in turning on and off individual degrees of freedom in rotation, translation or scaling without multiplying the number of transform representations; this trick is probably meaningless when tried with the general affine transform.", sep=",", argstr="--permitParameterVariation %s")
     costMetric = traits.Enum("MMI", "MSE", "NC", "MC", desc="The cost metric to be used during fitting. Defaults to MMI. Options are MMI (Mattes Mutual Information), MSE (Mean Square Error), NC (Normalized Correlation), MC (Match Cardinality for binary images)", argstr="--costMetric %s")
+    writeOutputTransformInFloat = traits.Bool(desc="By default, the output registration transforms (either the output composite transform or each transform component) are written to the disk in double precision. If this flag is ON, the output transforms will be written in single (float) precision. It is especially important if the output transform is a displacement field transform, or it is a composite transform that includes several displacement fields.", argstr="--writeOutputTransformInFloat ")
 
 
 class BRAINSFitOutputSpec(TraitedSpec):

nipype/interfaces/slicer/registration/brainsfit.py

@@ -146,6 +146,8 @@ def test_BRAINSFit_inputs():
     ),
     writeTransformOnFailure=dict(argstr='--writeTransformOnFailure ',
     ),
+    writeOutputTransformInFloat=dict(argstr='--writeOutputTransformInFloat ',
+    ),
     )
     inputs = BRAINSFit.input_spec()
 

nipype/interfaces/slicer/registration/tests/test_auto_BRAINSFit.py

@@ -15,3 +15,4 @@
 from .sgegraph import SGEGraphPlugin
 from .lsf import LSFPlugin
 from .slurm import SLURMPlugin
+from .slurmgraph import SLURMGraphPlugin

nipype/pipeline/plugins/__init__.py

@@ -80,14 +80,6 @@ def make_job_name(jobnumber, nodeslist):
             for idx, pyscript in enumerate(pyfiles):
                 node = nodes[idx]
                 node_status_done = node_completed_status(node)
-                ## If a node has no dependencies, and it is requested to run_without_submitting
-                ## then run this node in place
-                if (not node_status_done) and (len(dependencies[idx]) == 0 ) and (node.run_without_submitting == True):
-                    try:
-                        node.run()
-                    except Exception:
-                        node._clean_queue(idx, nodes)
-                    node_status_done = True # if successfully run locally, then claim true
 
                 #if the node itself claims done, then check to ensure all
                 #dependancies are also done
@@ -130,7 +122,7 @@ def make_job_name(jobnumber, nodeslist):
                         values = ' '
                         for jobid in dependencies[idx]:
                             ## Avoid dependancies of done jobs
-                            if cache_doneness_per_node[jobid] == False:
+                            if not self._dont_resubmit_completed_jobs or cache_doneness_per_node[jobid] == False:
                                 values += "${{{0}}},".format(make_job_name(jobid, nodes))
                         if values != ' ': # i.e. if some jobs were added to dependency list
                             values = values.rstrip(',')