Merge pull request #1496 from oesteban/rf/niworkflows-pr-299

[MAINT] Update to keep up with nipreps/niworkflows#299

Author: oesteban

docs/workflows.rst

@@ -19,7 +19,10 @@ is presented below:
     :simple_form: yes
 
     from fmriprep.workflows.base import init_single_subject_wf
+    from collections import namedtuple
+    BIDSLayout = namedtuple('BIDSLayout', ['root'], defaults='.')
     wf = init_single_subject_wf(
+        layout=BIDSLayout(),
         subject_id='test',
         name='single_subject_wf',
         task_id='',
@@ -30,7 +33,6 @@ is presented below:
         freesurfer=True,
         reportlets_dir='.',
         output_dir='.',
-        bids_dir='.',
         skull_strip_template='OASIS30ANTs',
         skull_strip_fixed_seed=False,
         template='MNI152NLin2009cAsym',

fmriprep/__about__.py

@@ -84,20 +84,19 @@
 ]
 
 REQUIRES = [
-    'grabbit==0.2.3',
     'indexed_gzip>=0.8.8',
     'nibabel>=2.2.1',
     'nilearn',
     'nipype>=1.1.6',
     'nitime',
-    'niworkflows>=0.6.1,<0.7.0a0',
+    'niworkflows>=0.7.0,<0.8.0a0',
     'numpy',
     'pandas',
     'psutil>=5.4',
-    'pybids==0.6.5',
+    'pybids>=0.7.0,<0.8.0a0',
     'pyyaml',
     'scikit-image',
-    'smriprep>=0.0.4,<0.1.0a0',
+    'smriprep>=0.0.5,<0.1.0a0',
     'statsmodels',
     'tedana>=0.0.5',
     'templateflow>=0.0.4',

fmriprep/cli/run.py

@@ -552,6 +552,7 @@ def build_workflow(opts, retval):
     from subprocess import check_call, CalledProcessError, TimeoutExpired
     from pkg_resources import resource_filename as pkgrf
     from shutil import copyfile
+    from bids import BIDSLayout
 
     from nipype import logging, config as ncfg
     from niworkflows.utils.bids import collect_participants
@@ -606,8 +607,9 @@ def build_workflow(opts, retval):
 
     # First check that bids_dir looks like a BIDS folder
     bids_dir = os.path.abspath(opts.bids_dir)
+    layout = BIDSLayout(bids_dir, validate=False)
     subject_list = collect_participants(
-        bids_dir, participant_label=opts.participant_label)
+        layout, participant_label=opts.participant_label)
 
     # Load base plugin_settings from file if --use-plugin
     if opts.use_plugin is not None:
@@ -715,6 +717,7 @@ def build_workflow(opts, retval):
         logger.warning('Option --debug is deprecated and has no effect')
 
     retval['workflow'] = init_fmriprep_wf(
+        layout=layout,
         subject_list=subject_list,
         task_id=opts.task_id,
         echo_idx=opts.echo_idx,
@@ -730,7 +733,6 @@ def build_workflow(opts, retval):
         skull_strip_fixed_seed=opts.skull_strip_fixed_seed,
         work_dir=work_dir,
         output_dir=output_dir,
-        bids_dir=bids_dir,
         freesurfer=opts.run_reconall,
         output_spaces=output_spaces,
         template=opts.template,

fmriprep/workflows/base.py

@@ -33,7 +33,7 @@
 from .bold import init_func_preproc_wf
 
 
-def init_fmriprep_wf(subject_list, task_id, echo_idx, run_uuid, work_dir, output_dir, bids_dir,
+def init_fmriprep_wf(layout, subject_list, task_id, echo_idx, run_uuid, work_dir, output_dir,
                      ignore, debug, low_mem, anat_only, longitudinal, t2s_coreg,
                      omp_nthreads, skull_strip_template, skull_strip_fixed_seed,
                      freesurfer, output_spaces, template, medial_surface_nan, cifti_output, hires,
@@ -51,15 +51,17 @@ def init_fmriprep_wf(subject_list, task_id, echo_idx, run_uuid, work_dir, output
         :simple_form: yes
 
         import os
-        os.environ['FREESURFER_HOME'] = os.getcwd()
+        from collections import namedtuple
+        BIDSLayout = namedtuple('BIDSLayout', ['root'], defaults='.')
         from fmriprep.workflows.base import init_fmriprep_wf
-        wf = init_fmriprep_wf(subject_list=['fmripreptest'],
+        os.environ['FREESURFER_HOME'] = os.getcwd()
+        wf = init_fmriprep_wf(layout=BIDSLayout(),
+                              subject_list=['fmripreptest'],
                               task_id='',
                               echo_idx=None,
                               run_uuid='X',
                               work_dir='.',
                               output_dir='.',
-                              bids_dir='.',
                               ignore=[],
                               debug=False,
                               low_mem=False,
@@ -90,6 +92,8 @@ def init_fmriprep_wf(subject_list, task_id, echo_idx, run_uuid, work_dir, output
 
     Parameters
 
+        layout : BIDSLayout object
+            BIDS dataset layout
         subject_list : list
             List of subject labels
         task_id : str or None
@@ -103,8 +107,6 @@ def init_fmriprep_wf(subject_list, task_id, echo_idx, run_uuid, work_dir, output
             Directory in which to store workflow execution state and temporary files
         output_dir : str
             Directory in which to save derivatives
-        bids_dir : str
-            Root directory of BIDS dataset
         ignore : list
             Preprocessing steps to skip (may include "slicetiming", "fieldmaps")
         debug : bool
@@ -182,13 +184,13 @@ def init_fmriprep_wf(subject_list, task_id, echo_idx, run_uuid, work_dir, output
     reportlets_dir = os.path.join(work_dir, 'reportlets')
     for subject_id in subject_list:
         single_subject_wf = init_single_subject_wf(
+            layout=layout,
             subject_id=subject_id,
             task_id=task_id,
             echo_idx=echo_idx,
             name="single_subject_" + subject_id + "_wf",
             reportlets_dir=reportlets_dir,
             output_dir=output_dir,
-            bids_dir=bids_dir,
             ignore=ignore,
             debug=debug,
             low_mem=low_mem,
@@ -230,8 +232,8 @@ def init_fmriprep_wf(subject_list, task_id, echo_idx, run_uuid, work_dir, output
     return fmriprep_wf
 
 
-def init_single_subject_wf(subject_id, task_id, echo_idx, name, reportlets_dir, output_dir,
-                           bids_dir, ignore, debug, low_mem, anat_only, longitudinal, t2s_coreg,
+def init_single_subject_wf(layout, subject_id, task_id, echo_idx, name, reportlets_dir,
+                           output_dir, ignore, debug, low_mem, anat_only, longitudinal, t2s_coreg,
                            omp_nthreads, skull_strip_template, skull_strip_fixed_seed,
                            freesurfer, output_spaces, template, medial_surface_nan,
                            cifti_output, hires, use_bbr, bold2t1w_dof, fmap_bspline, fmap_demean,
@@ -252,13 +254,15 @@ def init_single_subject_wf(subject_id, task_id, echo_idx, name, reportlets_dir,
         :simple_form: yes
 
         from fmriprep.workflows.base import init_single_subject_wf
-        wf = init_single_subject_wf(subject_id='test',
+        from collections import namedtuple
+        BIDSLayout = namedtuple('BIDSLayout', ['root'], defaults='.')
+        wf = init_single_subject_wf(layout=BIDSLayout(),
+                                    subject_id='test',
                                     task_id='',
                                     echo_idx=None,
                                     name='single_subject_wf',
                                     reportlets_dir='.',
                                     output_dir='.',
-                                    bids_dir='.',
                                     ignore=[],
                                     debug=False,
                                     low_mem=False,
@@ -288,6 +292,8 @@ def init_single_subject_wf(subject_id, task_id, echo_idx, name, reportlets_dir,
 
     Parameters
 
+        layout : BIDSLayout object
+            BIDS dataset layout
         subject_id : str
             List of subject labels
         task_id : str or None
@@ -321,8 +327,6 @@ def init_single_subject_wf(subject_id, task_id, echo_idx, name, reportlets_dir,
             Directory in which to save reportlets
         output_dir : str
             Directory in which to save derivatives
-        bids_dir : str
-            Root directory of BIDS dataset
         freesurfer : bool
             Enable FreeSurfer surface reconstruction (may increase runtime)
         output_spaces : list
@@ -377,9 +381,8 @@ def init_single_subject_wf(subject_id, task_id, echo_idx, name, reportlets_dir,
             't1w': ['/completely/made/up/path/sub-01_T1w.nii.gz'],
             'bold': ['/completely/made/up/path/sub-01_task-nback_bold.nii.gz']
         }
-        layout = None
     else:
-        subject_data, layout = collect_data(bids_dir, subject_id, task_id, echo_idx)
+        subject_data = collect_data(layout, subject_id, task_id, echo_idx)[0]
 
     # Make sure we always go through these two checks
     if not anat_only and subject_data['bold'] == []:
@@ -421,7 +424,8 @@ def init_single_subject_wf(subject_id, task_id, echo_idx, name, reportlets_dir,
     bidssrc = pe.Node(BIDSDataGrabber(subject_data=subject_data, anat_only=anat_only),
                       name='bidssrc')
 
-    bids_info = pe.Node(BIDSInfo(), name='bids_info', run_without_submitting=True)
+    bids_info = pe.Node(BIDSInfo(
+        bids_dir=layout.root, bids_validate=False), name='bids_info')
 
     summary = pe.Node(SubjectSummary(output_spaces=output_spaces, template=template),
                       name='summary', run_without_submitting=True)
@@ -464,12 +468,12 @@ def init_single_subject_wf(subject_id, task_id, echo_idx, name, reportlets_dir,
         (bidssrc, summary, [('t1w', 't1w'),
                             ('t2w', 't2w'),
                             ('bold', 'bold')]),
-        (bids_info, summary, [('subject_id', 'subject_id')]),
+        (bids_info, summary, [('subject', 'subject_id')]),
+        (bids_info, anat_preproc_wf, [(('subject', _prefix), 'inputnode.subject_id')]),
         (bidssrc, anat_preproc_wf, [('t1w', 'inputnode.t1w'),
                                     ('t2w', 'inputnode.t2w'),
                                     ('roi', 'inputnode.roi'),
                                     ('flair', 'inputnode.flair')]),
-        (summary, anat_preproc_wf, [('subject_id', 'inputnode.subject_id')]),
         (bidssrc, ds_report_summary, [(('t1w', fix_multi_T1w_source_name), 'source_file')]),
         (summary, ds_report_summary, [('out_report', 'in_file')]),
         (bidssrc, ds_report_about, [(('t1w', fix_multi_T1w_source_name), 'source_file')]),
@@ -532,3 +536,9 @@ def init_single_subject_wf(subject_id, task_id, echo_idx, name, reportlets_dir,
         ])
 
     return workflow
+
+
+def _prefix(subid):
+    if subid.startswith('sub-'):
+        return subid
+    return '-'.join(('sub', subid))

fmriprep/workflows/bold/base.py

@@ -246,15 +246,15 @@ def init_func_preproc_wf(bold_file, ignore, freesurfer,
 
     sbref_file = None
     # For doc building purposes
-    if layout is None or bold_file == 'bold_preprocesing':
+    if not hasattr(layout, 'parse_file_entities'):
         LOGGER.log(25, 'No valid layout: building empty workflow.')
         metadata = {
             'RepetitionTime': 2.0,
             'SliceTiming': [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],
             'PhaseEncodingDirection': 'j',
         }
         fmaps = [{
-            'type': 'phasediff',
+            'suffix': 'phasediff',
             'phasediff': 'sub-03/ses-2/fmap/sub-03_ses-2_run-1_phasediff.nii.gz',
             'magnitude1': 'sub-03/ses-2/fmap/sub-03_ses-2_run-1_magnitude1.nii.gz',
             'magnitude2': 'sub-03/ses-2/fmap/sub-03_ses-2_run-1_magnitude2.nii.gz',
@@ -264,16 +264,16 @@ def init_func_preproc_wf(bold_file, ignore, freesurfer,
     else:
         # Find associated sbref, if possible
         entities = layout.parse_file_entities(ref_file)
-        entities['type'] = 'sbref'
-        files = layout.get(**entities, extensions=['nii', 'nii.gz'])
+        entities['suffix'] = 'sbref'
+        files = layout.get(return_type='file', extensions=['nii', 'nii.gz'], **entities)
         refbase = os.path.basename(ref_file)
         if 'sbref' in ignore:
             LOGGER.info("Single-band reference files ignored.")
         elif files and multiecho:
             LOGGER.warning("Single-band reference found, but not supported in "
                            "multi-echo workflows at this time. Ignoring.")
         elif files:
-            sbref_file = files[0].filename
+            sbref_file = files[0]
             sbbase = os.path.basename(sbref_file)
             if len(files) > 1:
                 LOGGER.warning(
@@ -291,11 +291,11 @@ def init_func_preproc_wf(bold_file, ignore, freesurfer,
         if 'fieldmaps' not in ignore:
             fmaps = layout.get_fieldmap(ref_file, return_list=True)
             for fmap in fmaps:
-                fmap['metadata'] = layout.get_metadata(fmap[fmap['type']])
+                fmap['metadata'] = layout.get_metadata(fmap[fmap['suffix']])
 
         # Run SyN if forced or in the absence of fieldmap correction
         if force_syn or (use_syn and not fmaps):
-            fmaps.append({'type': 'syn'})
+            fmaps.append({'suffix': 'syn'})
 
         # Short circuits: (True and True and (False or 'TooShort')) == 'TooShort'
         run_stc = ("SliceTiming" in metadata and
@@ -476,14 +476,14 @@ def init_func_preproc_wf(bold_file, ignore, freesurfer,
     if not fmaps:
         LOGGER.warning('SDC: no fieldmaps found or they were ignored (%s).',
                        ref_file)
-    elif fmaps[0]['type'] == 'syn':
+    elif fmaps[0]['suffix'] == 'syn':
         LOGGER.warning(
             'SDC: no fieldmaps found or they were ignored. '
             'Using EXPERIMENTAL "fieldmap-less SyN" correction '
             'for dataset %s.', ref_file)
     else:
         LOGGER.log(25, 'SDC: fieldmap estimation of type "%s" intended for %s found.',
-                   fmaps[0]['type'], ref_file)
+                   fmaps[0]['suffix'], ref_file)
 
     # MULTI-ECHO EPI DATA #############################################
     if multiecho:
@@ -613,7 +613,7 @@ def init_func_preproc_wf(bold_file, ignore, freesurfer,
 
     if fmaps:
         from ..fieldmap.unwarp import init_fmap_unwarp_report_wf
-        sdc_type = fmaps[0]['type']
+        sdc_type = fmaps[0]['suffix']
 
         # Report on BOLD correction
         fmap_unwarp_report_wf = init_fmap_unwarp_report_wf(

fmriprep/workflows/fieldmap/base.py

@@ -79,7 +79,7 @@ def init_sdc_wf(fmaps, bold_meta, omp_nthreads=1,
         from fmriprep.workflows.fieldmap import init_sdc_wf
         wf = init_sdc_wf(
             fmaps=[{
-                'type': 'phasediff',
+                'suffix': 'phasediff',
                 'phasediff': 'sub-03/ses-2/fmap/sub-03_ses-2_run-1_phasediff.nii.gz',
                 'magnitude1': 'sub-03/ses-2/fmap/sub-03_ses-2_run-1_magnitude1.nii.gz',
                 'magnitude2': 'sub-03/ses-2/fmap/sub-03_ses-2_run-1_magnitude2.nii.gz',
@@ -140,7 +140,7 @@ def init_sdc_wf(fmaps, bold_meta, omp_nthreads=1,
     """
 
     # TODO: To be removed (filter out unsupported fieldmaps):
-    fmaps = [fmap for fmap in fmaps if fmap['type'] in FMAP_PRIORITY]
+    fmaps = [fmap for fmap in fmaps if fmap['suffix'] in FMAP_PRIORITY]
 
     workflow = Workflow(name='sdc_wf' if fmaps else 'sdc_bypass_wf')
     inputnode = pe.Node(niu.IdentityInterface(
@@ -170,15 +170,15 @@ def init_sdc_wf(fmaps, bold_meta, omp_nthreads=1,
 """
 
     # In case there are multiple fieldmaps prefer EPI
-    fmaps.sort(key=lambda fmap: FMAP_PRIORITY[fmap['type']])
+    fmaps.sort(key=lambda fmap: FMAP_PRIORITY[fmap['suffix']])
     fmap = fmaps[0]
 
     # PEPOLAR path
-    if fmap['type'] == 'epi':
+    if fmap['suffix'] == 'epi':
         outputnode.inputs.method = 'PEB/PEPOLAR (phase-encoding based / PE-POLARity)'
         # Get EPI polarities and their metadata
         epi_fmaps = [(fmap_['epi'], fmap_['metadata']["PhaseEncodingDirection"])
-                     for fmap_ in fmaps if fmap_['type'] == 'epi']
+                     for fmap_ in fmaps if fmap_['suffix'] == 'epi']
         sdc_unwarp_wf = init_pepolar_unwarp_wf(
             bold_meta=bold_meta,
             epi_fmaps=epi_fmaps,
@@ -193,11 +193,11 @@ def init_sdc_wf(fmaps, bold_meta, omp_nthreads=1,
         ])
 
     # FIELDMAP path
-    if fmap['type'] in ['fieldmap', 'phasediff']:
-        outputnode.inputs.method = 'FMB (%s-based)' % fmap['type']
+    if fmap['suffix'] in ['fieldmap', 'phasediff']:
+        outputnode.inputs.method = 'FMB (%s-based)' % fmap['suffix']
         # Import specific workflows here, so we don't break everything with one
         # unused workflow.
-        if fmap['type'] == 'fieldmap':
+        if fmap['suffix'] == 'fieldmap':
             from .fmap import init_fmap_wf
             fmap_estimator_wf = init_fmap_wf(
                 omp_nthreads=omp_nthreads,
@@ -206,7 +206,7 @@ def init_sdc_wf(fmaps, bold_meta, omp_nthreads=1,
             fmap_estimator_wf.inputs.inputnode.fieldmap = fmap['fieldmap']
             fmap_estimator_wf.inputs.inputnode.magnitude = fmap['magnitude']
 
-        if fmap['type'] == 'phasediff':
+        if fmap['suffix'] == 'phasediff':
             from .phdiff import init_phdiff_wf
             fmap_estimator_wf = init_phdiff_wf(omp_nthreads=omp_nthreads)
             # set inputs
@@ -235,7 +235,7 @@ def init_sdc_wf(fmaps, bold_meta, omp_nthreads=1,
         ])
 
     # FIELDMAP-less path
-    if any(fm['type'] == 'syn' for fm in fmaps):
+    if any(fm['suffix'] == 'syn' for fm in fmaps):
         syn_sdc_wf = init_syn_sdc_wf(
             bold_pe=bold_meta.get('PhaseEncodingDirection', None),
             omp_nthreads=omp_nthreads)
@@ -250,7 +250,7 @@ def init_sdc_wf(fmaps, bold_meta, omp_nthreads=1,
         ])
 
         # XXX Eliminate branch when forcing isn't an option
-        if fmap['type'] == 'syn':  # No fieldmaps, but --use-syn
+        if fmap['suffix'] == 'syn':  # No fieldmaps, but --use-syn
             outputnode.inputs.method = 'FLB ("fieldmap-less", SyN-based)'
             sdc_unwarp_wf = syn_sdc_wf
         else:  # --force-syn was called when other fieldmap was present

fmriprep/workflows/fieldmap/phdiff.py

@@ -73,7 +73,7 @@ def _pick1st(inlist):
         return inlist[0]
 
     # Read phasediff echo times
-    meta = pe.Node(ReadSidecarJSON(), name='meta', mem_gb=0.01, run_without_submitting=True)
+    meta = pe.Node(ReadSidecarJSON(bids_validate=False), name='meta', mem_gb=0.01)
 
     # Merge input magnitude images
     magmrg = pe.Node(IntraModalMerge(), name='magmrg')