Merge branch 'fix/csv'

* fix/csv: (47 commits)
  added group_id to pipeline names
  Fixed TesselationSmooth outputs and error reporting.
  Fixed an unintended comment
  CSVFibers fix
  Added more global metrics
  Fixes for global CSV merging
  Number/percentage of tracts are now output from CreateMatrix as a .mat file
  Typo fix
  Group mrtrix tutorial runs properly
  Fixed os.path.exists call
  Spacing fix
  Uncommented subject lists
  Fix for std::badalloc, lmax float to int, parcellation subject dir must exist
  Fix for creatematrix outputs
  ConcatCSV interface and CreateMatrix output fixes
  Changed CSV concatenation to use Function interfaces
  Fix for single subject global network csvs'
  Improved CSV merging. Single subj and group level fiber csvs
  Number/percentage of tracts are now output from CreateMatrix as a .mat file
  Typo fix
  ...

Author: chrisgorgo

examples/dmri_connectivity_advanced.py

@@ -56,12 +56,13 @@
 import nipype.interfaces.cmtk as cmtk
 import nipype.interfaces.dipy as dipy
 import inspect
-import os.path as op                      # system functions
+import os, os.path as op                      # system functions
 from nipype.workflows.dmri.fsl.dti import create_eddy_correct_pipeline
 from nipype.workflows.dmri.camino.connectivity_mapping import select_aparc_annot
 from nipype.utils.misc import package_check
 import warnings
 from nipype.workflows.dmri.connectivity.nx import create_networkx_pipeline, create_cmats_to_csv_pipeline
+from nipype.workflows.smri.freesurfer import create_tessellation_flow
 
 try:
     package_check('cmp')
@@ -82,6 +83,9 @@
 fs.FSCommand.set_default_subjects_dir(subjects_dir)
 fsl.FSLCommand.set_default_output_type('NIFTI')
 
+fs_dir = os.environ['FREESURFER_HOME']
+lookup_file = op.join(fs_dir,'FreeSurferColorLUT.txt')
+
 """
 This needs to point to the fdt folder you can find after extracting
 
@@ -328,7 +332,7 @@
 
 CFFConverter = pe.Node(interface=cmtk.CFFConverter(), name="CFFConverter")
 CFFConverter.inputs.script_files = op.abspath(inspect.getfile(inspect.currentframe()))
-giftiSurfaces = pe.Node(interface=util.Merge(8), name="GiftiSurfaces")
+giftiSurfaces = pe.Node(interface=util.Merge(9), name="GiftiSurfaces")
 giftiLabels = pe.Node(interface=util.Merge(2), name="GiftiLabels")
 niftiVolumes = pe.Node(interface=util.Merge(3), name="NiftiVolumes")
 fiberDataArrays = pe.Node(interface=util.Merge(4), name="FiberDataArrays")
@@ -344,6 +348,9 @@
 NxStatsCFFConverter = pe.Node(interface=cmtk.CFFConverter(), name="NxStatsCFFConverter")
 NxStatsCFFConverter.inputs.script_files = op.abspath(inspect.getfile(inspect.currentframe()))
 
+tessflow = create_tessellation_flow(name='tessflow', out_format='gii')
+tessflow.inputs.inputspec.lookup_file = lookup_file
+
 """
 Connecting the workflow
 =======================
@@ -371,6 +378,9 @@
 mapping.connect([(inputnode, FreeSurferSourceRH,[("subjects_dir","subjects_dir")])])
 mapping.connect([(inputnode, FreeSurferSourceRH,[("subject_id","subject_id")])])
 
+mapping.connect([(inputnode, tessflow,[("subjects_dir","inputspec.subjects_dir")])])
+mapping.connect([(inputnode, tessflow,[("subject_id","inputspec.subject_id")])])
+
 mapping.connect([(inputnode, parcellate,[("subjects_dir","subjects_dir")])])
 mapping.connect([(inputnode, parcellate,[("subject_id","subject_id")])])
 mapping.connect([(parcellate, mri_convert_ROI_scale500,[('roi_file','in_file')])])
@@ -516,6 +526,7 @@
 mapping.connect([(mris_convertRHinflated, giftiSurfaces,[("converted","in6")])])
 mapping.connect([(mris_convertLHsphere, giftiSurfaces,[("converted","in7")])])
 mapping.connect([(mris_convertRHsphere, giftiSurfaces,[("converted","in8")])])
+mapping.connect([(tessflow, giftiSurfaces,[("outputspec.meshes","in9")])])
 
 mapping.connect([(mris_convertLHlabels, giftiLabels,[("converted","in1")])])
 mapping.connect([(mris_convertRHlabels, giftiLabels,[("converted","in2")])])

examples/dmri_group_connectivity_mrtrix.py

@@ -140,12 +140,6 @@
 
     l1pipeline = create_group_connectivity_pipeline(group_list, group_id, data_dir, subjects_dir, output_dir, info)
 
-    # This is used to demonstrate the ease through which different parameters can be set for each group.
-    if group_id == 'parkinsons':
-        l1pipeline.inputs.connectivity.mapping.threshold_FA.absolute_threshold_value = 0.5
-    else:
-        l1pipeline.inputs.connectivity.mapping.threshold_FA.absolute_threshold_value = 0.7
-
     # Here with invert the b-vectors in the Y direction and set the maximum harmonic order of the
     # spherical deconvolution step
     l1pipeline.inputs.connectivity.mapping.fsl2mrtrix.invert_y = True

nipype/algorithms/misc.py

@@ -649,32 +649,33 @@ def _list_outputs(self):
         return outputs
 
 def merge_csvs(in_list):
-	for idx, in_file in enumerate(in_list):
-		try:
-			in_array = np.loadtxt(in_file, delimiter=',')
-		except ValueError, ex:
-			try:
-				in_array = np.loadtxt(in_file, delimiter=',', skiprows=1)
-			except ValueError, ex:
-				first = open(in_file, 'r')
-				header_line = first.readline()
-				header_list = header_line.split(',')
-				n_cols = len(header_list)
-				try:
-					in_array = np.loadtxt(in_file, delimiter=',', skiprows=1, usecols=range(1,n_cols))
-				except ValueError, ex:
-					in_array = np.loadtxt(in_file, delimiter=',', skiprows=1, usecols=range(1,n_cols-1))
-		if idx == 0:
-			out_array = in_array
-		else:
-			out_array = np.dstack((out_array, in_array))
-	out_array = np.squeeze(out_array)
-	iflogger.info('Final output array shape:')
-	iflogger.info(np.shape(out_array))
-	return out_array
+    for idx, in_file in enumerate(in_list):
+        try:
+            in_array = np.loadtxt(in_file, delimiter=',')
+        except ValueError, ex:
+            try:
+                in_array = np.loadtxt(in_file, delimiter=',', skiprows=1)
+            except ValueError, ex:
+                first = open(in_file, 'r')
+                header_line = first.readline()
+                header_list = header_line.split(',')
+                n_cols = len(header_list)
+                try:
+                    in_array = np.loadtxt(in_file, delimiter=',', skiprows=1, usecols=range(1,n_cols))
+                except ValueError, ex:
+                    in_array = np.loadtxt(in_file, delimiter=',', skiprows=1, usecols=range(1,n_cols-1))
+        if idx == 0:
+            out_array = in_array
+        else:
+            out_array = np.dstack((out_array, in_array))
+    out_array = np.squeeze(out_array)
+    iflogger.info('Final output array shape:')
+    iflogger.info(np.shape(out_array))
+    return out_array
 
 def remove_identical_paths(in_files):
     import os.path as op
+    from nipype.utils.filemanip import split_filename
     if len(in_files) > 1:
         out_names = list()
         commonprefix = op.commonprefix(in_files)
@@ -699,24 +700,27 @@ def maketypelist(rowheadings, shape, extraheadingBool, extraheading):
         for idx in range(1,(min(shape)+1)):
             typelist.append((str(idx), float))
     else:
-        typelist.append((str(1), float))
+        for idx in range(1,(shape[0]+1)):
+            typelist.append((str(idx), float))
     if extraheadingBool:
         typelist.append((extraheading, 'a40'))
     iflogger.info(typelist)
     return typelist
 
 def makefmtlist(output_array, typelist, rowheadingsBool, shape, extraheadingBool):
-    output = np.zeros(max(shape), typelist)
     fmtlist = []
     if rowheadingsBool:
         fmtlist.append('%s')
     if len(shape) > 1:
+        output = np.zeros(max(shape), typelist)
         for idx in range(1,min(shape)+1):
             output[str(idx)] = output_array[:,idx-1]
             fmtlist.append('%f')
     else:
-        output[str(1)] = output_array
-        fmtlist.append('%f')
+        output = np.zeros(1, typelist)
+        for idx in range(1,len(output_array)+1):
+            output[str(idx)] = output_array[idx-1]
+            fmtlist.append('%f')
     if extraheadingBool:
         fmtlist.append('%s')
     fmt = ','.join(fmtlist)
@@ -727,6 +731,7 @@ class MergeCSVFilesInputSpec(TraitedSpec):
     out_file = File('merged.csv', usedefault=True, desc='Output filename for merged CSV file')
     column_headings = traits.List(traits.Str, desc='List of column headings to save in merged CSV file (must be equal to number of input files). If left undefined, these will be pulled from the input filenames.')
     row_headings = traits.List(traits.Str, desc='List of row headings to save in merged CSV file (must be equal to number of rows in the input files).')
+    row_heading_title = traits.Str('label', usedefault=True, desc='Column heading for the row headings added')
     extra_column_heading = traits.Str(desc='New heading to add for the added field.')
     extra_field = traits.Str(desc='New field to add to each row. This is useful for saving the group or subject ID in the file.')
 
@@ -756,6 +761,7 @@ class MergeCSVFiles(BaseInterface):
 
     def _run_interface(self, runtime):
         extraheadingBool = False
+        extraheading = ''
         rowheadingsBool = False
         """
         This block defines the column headings.
@@ -775,14 +781,15 @@ def _run_interface(self, runtime):
                 extraheading = 'type'
                 iflogger.info('Extra column heading was not defined. Using "type"')
             headings.append(extraheading)
-            extraheadingBool = True
+            extraheadingBool = True            
 
         if len(self.inputs.in_files) == 1:
             iflogger.warn('Only one file input!')
 
         if isdefined(self.inputs.row_headings):
             iflogger.info('Row headings have been provided. Adding "labels" column header.')
-            csv_headings = '"labels","' + '","'.join(itertools.chain(headings)) + '"\n'
+            prefix = '"{p}","'.format(p=self.inputs.row_heading_title)
+            csv_headings = prefix + '","'.join(itertools.chain(headings)) + '"\n'
             rowheadingsBool = True
         else:
             iflogger.info('Row headings have not been provided.')
@@ -814,12 +821,16 @@ def _run_interface(self, runtime):
             for row_heading in row_heading_list:
                 row_heading_with_quotes = '"' + row_heading + '"'
                 row_heading_list_with_quotes.append(row_heading_with_quotes)
-            row_headings = np.array(row_heading_list_with_quotes)
+            row_headings = np.array(row_heading_list_with_quotes, dtype='|S40')
             output['heading'] = row_headings
 
         if isdefined(self.inputs.extra_field):
             extrafieldlist = []
-            for idx in range(0,max(shape)):
+            if len(shape) > 1:
+                mx = shape[0]
+            else:
+                mx = 1
+            for idx in range(0,mx):
                 extrafieldlist.append(self.inputs.extra_field)
             iflogger.info(len(extrafieldlist))
             output[extraheading] = extrafieldlist