Merge remote-tracking branch 'upstream/master' into no_error_aroma

Author: oesteban

CHANGES.rst

@@ -1,3 +1,21 @@
+1.3.0 (February 7, 2019)
+========================
+
+We start the 1.3.x series including a few bugfixes, housekeeping duty and a refactors
+to leverage `sMRIPrep <https://github.com/poldracklab/smriprep>`__ (which is a fork of
+fMRIPrep's anatomical workflow), pybids>=0.7 for querying dataset, and 
+`TemplateFlow <https://github.com/templateflow>`__ for handling standard spaces.
+
+* [FIX] Bad ``fsnative`` replacement in CIfTI workflow (#1476) @oesteban
+* [FIX] Avoid warning when generating boilerplate (#1464) @oesteban
+* [MAINT] resolves #1485 : patch fmriprep-docker automount for use with Python 3.7 (#1486) @rciric
+* [RF] Use anatomical workflows from sMRIPrep (#1482) @oesteban
+* [MAINT] Update sentry-sdk (#1490) @chrisfilo
+* [ENH] Remaining TemplateFlow integrations (#1494) @oesteban
+* [MAINT] Update to keep up with poldracklab/niworkflows#299 (#1496) @oesteban
+* [FIX] Updating bids-validator to 1.1.3 (#1498) @chrisfilo
+
+
 1.2.6-1 (January 24, 2019)
 ==========================
 

Dockerfile

@@ -104,7 +104,7 @@ RUN apt-get install -y nodejs
 RUN npm install -g svgo
 
 # Installing bids-validator
-RUN npm install -g [email protected].0
+RUN npm install -g [email protected].3
 
 # Installing and setting up ICA_AROMA
 RUN mkdir -p /opt/ICA-AROMA && \
@@ -161,10 +161,13 @@ RUN pip install "datalad==0.10.0" && \
     rm -rf ~/.cache/pip
 
 RUN git config --global user.name "First Last" && \
-    git config --global user.email "[email protected]" && \
-    datalad install -r https://github.com/templateflow/templateflow.git
-RUN datalad get $TEMPLATEFLOW_HOME/tpl-MNI152NLin2009cAsym/*_T1w.nii.gz \
+    git config --global user.email "[email protected]"
+
+RUN datalad install -r https://github.com/templateflow/templateflow.git && \
+    datalad get $TEMPLATEFLOW_HOME/tpl-MNI152NLin2009cAsym/*_T1w.nii.gz \
                 $TEMPLATEFLOW_HOME/tpl-MNI152NLin2009cAsym/*_desc-brain_mask.nii.gz \
+                $TEMPLATEFLOW_HOME/tpl-MNI152NLin2009cAsym/tpl-MNI152NLin2009cAsym_res-02_desc-fMRIPrep_boldref.nii.gz \
+                $TEMPLATEFLOW_HOME/tpl-MNI152NLin2009cAsym/tpl-MNI152NLin2009cAsym_res-02_desc-DKT31_dseg.nii.gz \
                 $TEMPLATEFLOW_HOME/tpl-MNI152Lin/*_T1w.nii.gz \
                 $TEMPLATEFLOW_HOME/tpl-MNI152Lin/*_desc-brain_mask.nii.gz \
                 $TEMPLATEFLOW_HOME/tpl-OASIS30ANTs/*_T1w.nii.gz \

docs/workflows.rst

@@ -19,7 +19,10 @@ is presented below:
     :simple_form: yes
 
     from fmriprep.workflows.base import init_single_subject_wf
+    from collections import namedtuple
+    BIDSLayout = namedtuple('BIDSLayout', ['root'], defaults='.')
     wf = init_single_subject_wf(
+        layout=BIDSLayout(),
         subject_id='test',
         name='single_subject_wf',
         task_id='',
@@ -30,7 +33,6 @@ is presented below:
         freesurfer=True,
         reportlets_dir='.',
         output_dir='.',
-        bids_dir='.',
         skull_strip_template='OASIS30ANTs',
         skull_strip_fixed_seed=False,
         template='MNI152NLin2009cAsym',

fmriprep/__about__.py

@@ -84,27 +84,28 @@
 ]
 
 REQUIRES = [
-    'grabbit==0.2.3',
     'indexed_gzip>=0.8.8',
     'nibabel>=2.2.1',
     'nilearn',
     'nipype>=1.1.6',
     'nitime',
-    'niworkflows>=0.6.1,<0.7.0a0',
+    'niworkflows',
     'numpy',
     'pandas',
     'psutil>=5.4',
-    'pybids==0.6.5',
+    'pybids>=0.7.0,<0.8.0a0',
     'pyyaml',
     'scikit-image',
-    'smriprep>=0.0.4,<0.1.0a0',
+    'smriprep>=0.0.5,<0.1.0a0',
     'statsmodels',
     'tedana>=0.0.5',
     'templateflow>=0.0.4',
 ]
 
 
 LINKS_REQUIRES = [
+    'git+https://github.com/poldracklab/niworkflows.git@'
+    'b97787ee2f1b93cafb104664b60fa0c4fd81253a#egg=niworkflows-0.7.1',
 ]
 
 TESTS_REQUIRES = [
@@ -125,7 +126,7 @@
     ],
     'duecredit': ['duecredit'],
     'resmon': [],
-    'sentry': ['sentry-sdk>=0.5.3'],
+    'sentry': ['sentry-sdk>=0.6.9'],
     'tests': TESTS_REQUIRES,
 }
 EXTRA_REQUIRES['docs'] = EXTRA_REQUIRES['doc']

fmriprep/cli/run.py

@@ -556,6 +556,7 @@ def build_workflow(opts, retval):
     from subprocess import check_call, CalledProcessError, TimeoutExpired
     from pkg_resources import resource_filename as pkgrf
     from shutil import copyfile
+    from bids import BIDSLayout
 
     from nipype import logging, config as ncfg
     from niworkflows.utils.bids import collect_participants
@@ -610,8 +611,9 @@ def build_workflow(opts, retval):
 
     # First check that bids_dir looks like a BIDS folder
     bids_dir = os.path.abspath(opts.bids_dir)
+    layout = BIDSLayout(bids_dir, validate=False)
     subject_list = collect_participants(
-        bids_dir, participant_label=opts.participant_label)
+        layout, participant_label=opts.participant_label)
 
     # Load base plugin_settings from file if --use-plugin
     if opts.use_plugin is not None:
@@ -719,6 +721,7 @@ def build_workflow(opts, retval):
         logger.warning('Option --debug is deprecated and has no effect')
 
     retval['workflow'] = init_fmriprep_wf(
+        layout=layout,
         subject_list=subject_list,
         task_id=opts.task_id,
         echo_idx=opts.echo_idx,
@@ -734,7 +737,6 @@ def build_workflow(opts, retval):
         skull_strip_fixed_seed=opts.skull_strip_fixed_seed,
         work_dir=work_dir,
         output_dir=output_dir,
-        bids_dir=bids_dir,
         freesurfer=opts.run_reconall,
         output_spaces=output_spaces,
         template=opts.template,

fmriprep/utils/bids.py

@@ -6,13 +6,6 @@
 Utilities to handle BIDS inputs
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Fetch some test data
-
-    >>> import os
-    >>> from niworkflows import data
-    >>> data_root = data.get_bids_examples(variant='BIDS-examples-1-enh-ds054')
-    >>> os.chdir(data_root)
-
 """
 import os
 import json

fmriprep/workflows/base.py

@@ -33,7 +33,7 @@
 from .bold import init_func_preproc_wf
 
 
-def init_fmriprep_wf(subject_list, task_id, echo_idx, run_uuid, work_dir, output_dir, bids_dir,
+def init_fmriprep_wf(layout, subject_list, task_id, echo_idx, run_uuid, work_dir, output_dir,
                      ignore, debug, low_mem, anat_only, longitudinal, t2s_coreg,
                      omp_nthreads, skull_strip_template, skull_strip_fixed_seed,
                      freesurfer, output_spaces, template, medial_surface_nan, cifti_output, hires,
@@ -51,15 +51,17 @@ def init_fmriprep_wf(subject_list, task_id, echo_idx, run_uuid, work_dir, output
         :simple_form: yes
 
         import os
-        os.environ['FREESURFER_HOME'] = os.getcwd()
+        from collections import namedtuple
+        BIDSLayout = namedtuple('BIDSLayout', ['root'], defaults='.')
         from fmriprep.workflows.base import init_fmriprep_wf
-        wf = init_fmriprep_wf(subject_list=['fmripreptest'],
+        os.environ['FREESURFER_HOME'] = os.getcwd()
+        wf = init_fmriprep_wf(layout=BIDSLayout(),
+                              subject_list=['fmripreptest'],
                               task_id='',
                               echo_idx=None,
                               run_uuid='X',
                               work_dir='.',
                               output_dir='.',
-                              bids_dir='.',
                               ignore=[],
                               debug=False,
                               low_mem=False,
@@ -90,6 +92,8 @@ def init_fmriprep_wf(subject_list, task_id, echo_idx, run_uuid, work_dir, output
 
     Parameters
 
+        layout : BIDSLayout object
+            BIDS dataset layout
         subject_list : list
             List of subject labels
         task_id : str or None
@@ -103,8 +107,6 @@ def init_fmriprep_wf(subject_list, task_id, echo_idx, run_uuid, work_dir, output
             Directory in which to store workflow execution state and temporary files
         output_dir : str
             Directory in which to save derivatives
-        bids_dir : str
-            Root directory of BIDS dataset
         ignore : list
             Preprocessing steps to skip (may include "slicetiming", "fieldmaps")
         debug : bool
@@ -182,13 +184,13 @@ def init_fmriprep_wf(subject_list, task_id, echo_idx, run_uuid, work_dir, output
     reportlets_dir = os.path.join(work_dir, 'reportlets')
     for subject_id in subject_list:
         single_subject_wf = init_single_subject_wf(
+            layout=layout,
             subject_id=subject_id,
             task_id=task_id,
             echo_idx=echo_idx,
             name="single_subject_" + subject_id + "_wf",
             reportlets_dir=reportlets_dir,
             output_dir=output_dir,
-            bids_dir=bids_dir,
             ignore=ignore,
             debug=debug,
             low_mem=low_mem,
@@ -230,8 +232,8 @@ def init_fmriprep_wf(subject_list, task_id, echo_idx, run_uuid, work_dir, output
     return fmriprep_wf
 
 
-def init_single_subject_wf(subject_id, task_id, echo_idx, name, reportlets_dir, output_dir,
-                           bids_dir, ignore, debug, low_mem, anat_only, longitudinal, t2s_coreg,
+def init_single_subject_wf(layout, subject_id, task_id, echo_idx, name, reportlets_dir,
+                           output_dir, ignore, debug, low_mem, anat_only, longitudinal, t2s_coreg,
                            omp_nthreads, skull_strip_template, skull_strip_fixed_seed,
                            freesurfer, output_spaces, template, medial_surface_nan,
                            cifti_output, hires, use_bbr, bold2t1w_dof, fmap_bspline, fmap_demean,
@@ -252,13 +254,15 @@ def init_single_subject_wf(subject_id, task_id, echo_idx, name, reportlets_dir,
         :simple_form: yes
 
         from fmriprep.workflows.base import init_single_subject_wf
-        wf = init_single_subject_wf(subject_id='test',
+        from collections import namedtuple
+        BIDSLayout = namedtuple('BIDSLayout', ['root'], defaults='.')
+        wf = init_single_subject_wf(layout=BIDSLayout(),
+                                    subject_id='test',
                                     task_id='',
                                     echo_idx=None,
                                     name='single_subject_wf',
                                     reportlets_dir='.',
                                     output_dir='.',
-                                    bids_dir='.',
                                     ignore=[],
                                     debug=False,
                                     low_mem=False,
@@ -288,6 +292,8 @@ def init_single_subject_wf(subject_id, task_id, echo_idx, name, reportlets_dir,
 
     Parameters
 
+        layout : BIDSLayout object
+            BIDS dataset layout
         subject_id : str
             List of subject labels
         task_id : str or None
@@ -321,8 +327,6 @@ def init_single_subject_wf(subject_id, task_id, echo_idx, name, reportlets_dir,
             Directory in which to save reportlets
         output_dir : str
             Directory in which to save derivatives
-        bids_dir : str
-            Root directory of BIDS dataset
         freesurfer : bool
             Enable FreeSurfer surface reconstruction (may increase runtime)
         output_spaces : list
@@ -377,9 +381,8 @@ def init_single_subject_wf(subject_id, task_id, echo_idx, name, reportlets_dir,
             't1w': ['/completely/made/up/path/sub-01_T1w.nii.gz'],
             'bold': ['/completely/made/up/path/sub-01_task-nback_bold.nii.gz']
         }
-        layout = None
     else:
-        subject_data, layout = collect_data(bids_dir, subject_id, task_id, echo_idx)
+        subject_data = collect_data(layout, subject_id, task_id, echo_idx)[0]
 
     # Make sure we always go through these two checks
     if not anat_only and subject_data['bold'] == []:
@@ -421,7 +424,8 @@ def init_single_subject_wf(subject_id, task_id, echo_idx, name, reportlets_dir,
     bidssrc = pe.Node(BIDSDataGrabber(subject_data=subject_data, anat_only=anat_only),
                       name='bidssrc')
 
-    bids_info = pe.Node(BIDSInfo(), name='bids_info', run_without_submitting=True)
+    bids_info = pe.Node(BIDSInfo(
+        bids_dir=layout.root, bids_validate=False), name='bids_info')
 
     summary = pe.Node(SubjectSummary(output_spaces=output_spaces, template=template),
                       name='summary', run_without_submitting=True)
@@ -464,12 +468,12 @@ def init_single_subject_wf(subject_id, task_id, echo_idx, name, reportlets_dir,
         (bidssrc, summary, [('t1w', 't1w'),
                             ('t2w', 't2w'),
                             ('bold', 'bold')]),
-        (bids_info, summary, [('subject_id', 'subject_id')]),
+        (bids_info, summary, [('subject', 'subject_id')]),
+        (bids_info, anat_preproc_wf, [(('subject', _prefix), 'inputnode.subject_id')]),
         (bidssrc, anat_preproc_wf, [('t1w', 'inputnode.t1w'),
                                     ('t2w', 'inputnode.t2w'),
                                     ('roi', 'inputnode.roi'),
                                     ('flair', 'inputnode.flair')]),
-        (summary, anat_preproc_wf, [('subject_id', 'inputnode.subject_id')]),
         (bidssrc, ds_report_summary, [(('t1w', fix_multi_T1w_source_name), 'source_file')]),
         (summary, ds_report_summary, [('out_report', 'in_file')]),
         (bidssrc, ds_report_about, [(('t1w', fix_multi_T1w_source_name), 'source_file')]),
@@ -532,3 +536,9 @@ def init_single_subject_wf(subject_id, task_id, echo_idx, name, reportlets_dir,
         ])
 
     return workflow
+
+
+def _prefix(subid):
+    if subid.startswith('sub-'):
+        return subid
+    return '-'.join(('sub', subid))

fmriprep/workflows/bold/base.py

@@ -246,15 +246,15 @@ def init_func_preproc_wf(bold_file, ignore, freesurfer,
 
     sbref_file = None
     # For doc building purposes
-    if layout is None or bold_file == 'bold_preprocesing':
+    if not hasattr(layout, 'parse_file_entities'):
         LOGGER.log(25, 'No valid layout: building empty workflow.')
         metadata = {
             'RepetitionTime': 2.0,
             'SliceTiming': [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],
             'PhaseEncodingDirection': 'j',
         }
         fmaps = [{
-            'type': 'phasediff',
+            'suffix': 'phasediff',
             'phasediff': 'sub-03/ses-2/fmap/sub-03_ses-2_run-1_phasediff.nii.gz',
             'magnitude1': 'sub-03/ses-2/fmap/sub-03_ses-2_run-1_magnitude1.nii.gz',
             'magnitude2': 'sub-03/ses-2/fmap/sub-03_ses-2_run-1_magnitude2.nii.gz',
@@ -264,16 +264,16 @@ def init_func_preproc_wf(bold_file, ignore, freesurfer,
     else:
         # Find associated sbref, if possible
         entities = layout.parse_file_entities(ref_file)
-        entities['type'] = 'sbref'
-        files = layout.get(**entities, extensions=['nii', 'nii.gz'])
+        entities['suffix'] = 'sbref'
+        files = layout.get(return_type='file', extensions=['nii', 'nii.gz'], **entities)
         refbase = os.path.basename(ref_file)
         if 'sbref' in ignore:
             LOGGER.info("Single-band reference files ignored.")
         elif files and multiecho:
             LOGGER.warning("Single-band reference found, but not supported in "
                            "multi-echo workflows at this time. Ignoring.")
         elif files:
-            sbref_file = files[0].filename
+            sbref_file = files[0]
             sbbase = os.path.basename(sbref_file)
             if len(files) > 1:
                 LOGGER.warning(
@@ -291,11 +291,11 @@ def init_func_preproc_wf(bold_file, ignore, freesurfer,
         if 'fieldmaps' not in ignore:
             fmaps = layout.get_fieldmap(ref_file, return_list=True)
             for fmap in fmaps:
-                fmap['metadata'] = layout.get_metadata(fmap[fmap['type']])
+                fmap['metadata'] = layout.get_metadata(fmap[fmap['suffix']])
 
         # Run SyN if forced or in the absence of fieldmap correction
         if force_syn or (use_syn and not fmaps):
-            fmaps.append({'type': 'syn'})
+            fmaps.append({'suffix': 'syn'})
 
         # Short circuits: (True and True and (False or 'TooShort')) == 'TooShort'
         run_stc = ("SliceTiming" in metadata and
@@ -476,14 +476,14 @@ def init_func_preproc_wf(bold_file, ignore, freesurfer,
     if not fmaps:
         LOGGER.warning('SDC: no fieldmaps found or they were ignored (%s).',
                        ref_file)
-    elif fmaps[0]['type'] == 'syn':
+    elif fmaps[0]['suffix'] == 'syn':
         LOGGER.warning(
             'SDC: no fieldmaps found or they were ignored. '
             'Using EXPERIMENTAL "fieldmap-less SyN" correction '
             'for dataset %s.', ref_file)
     else:
         LOGGER.log(25, 'SDC: fieldmap estimation of type "%s" intended for %s found.',
-                   fmaps[0]['type'], ref_file)
+                   fmaps[0]['suffix'], ref_file)
 
     # MULTI-ECHO EPI DATA #############################################
     if multiecho:
@@ -613,7 +613,7 @@ def init_func_preproc_wf(bold_file, ignore, freesurfer,
 
     if fmaps:
         from ..fieldmap.unwarp import init_fmap_unwarp_report_wf
-        sdc_type = fmaps[0]['type']
+        sdc_type = fmaps[0]['suffix']
 
         # Report on BOLD correction
         fmap_unwarp_report_wf = init_fmap_unwarp_report_wf(