Merge remote-tracking branch 'upstream/master' into enh/bids_filter

bpinsard · bpinsard · commit a6252de52035 · 2019-09-30T13:55:06.000-04:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -260,7 +260,7 @@ jobs:
 
   build_docs:
     docker:
-        - image: python:latest
+      - image: python:latest
     working_directory: /tmp/src/fmriprep
     environment:
       - FSLOUTPUTTYPE: 'NIFTI'
@@ -771,16 +771,15 @@ jobs:
             fi
 
   test_deploy_pypi:
-    machine:
-      image: circleci/classic:201711-01
+    docker:
+      - image: circleci/python:3.7.4
     working_directory: /tmp/src/fmriprep
     steps:
       - checkout
       - run:
           name: Build fMRIPrep
           command: |
-            pyenv local 3.5.2
-            pip install twine  # For use in checking distributions
+            pip install --user twine  # For use in checking distributions
             THISVERSION=$( python get_version.py )
             THISVERSION=${THISVERSION%.dirty*}
             THISVERSION=${CIRCLE_TAG:-$THISVERSION}
@@ -795,7 +794,6 @@ jobs:
       - run:
           name: Check sdist distribution
           command: |
-            pyenv local 3.5.2
             THISVERSION=$( python get_version.py )
             THISVERSION=${THISVERSION%.dirty*}
             THISVERSION=${CIRCLE_TAG:-$THISVERSION}
@@ -814,7 +812,6 @@ jobs:
       - run:
           name: Check wheel distribution
           command: |
-            pyenv local 3.5.2
             THISVERSION=$( python get_version.py )
             THISVERSION=${THISVERSION%.dirty*}
             THISVERSION=${CIRCLE_TAG:-$THISVERSION}
@@ -832,7 +829,6 @@ jobs:
       - run:
           name: Build fmriprep-docker
           command: |
-            pyenv local 3.5.2
             THISVERSION=$( python get_version.py )
             THISVERSION=${THISVERSION%.dirty*}
             cd wrapper
@@ -846,15 +842,14 @@ jobs:
           path: /tmp/src/fmriprep/wrapper/dist
 
   deploy_pypi:
-    machine:
-      image: circleci/classic:201711-01
+    docker:
+      - image: circleci/python:3.7.4
     working_directory: /tmp/src/fmriprep
     steps:
       - checkout
       - run:
           name: Build fMRIPrep
           command: |
-            pyenv local 3.5.2
             THISVERSION=$( python get_version.py )
             virtualenv --python=python build
             source build/bin/activate
@@ -866,7 +861,6 @@ jobs:
       - run:
           name: Build fmriprep-docker
           command: |
-            pyenv local 3.5.2
             THISVERSION=$( python get_version.py )
             cd wrapper
             virtualenv --python=python build
@@ -878,8 +872,7 @@ jobs:
       - run:
           name: Upload packages to PyPI
           command: |
-            pyenv local 3.5.2
-            pip install twine
+            pip install --user twine
             twine upload dist/fmriprep* wrapper/dist/fmriprep*
 
   deployable:
diff --git a/Dockerfile b/Dockerfile
@@ -112,7 +112,8 @@ RUN mkdir -p /opt/ICA-AROMA && \
   curl -sSL "https://github.com/maartenmennes/ICA-AROMA/archive/v0.4.4-beta.tar.gz" \
   | tar -xzC /opt/ICA-AROMA --strip-components 1 && \
   chmod +x /opt/ICA-AROMA/ICA_AROMA.py
-ENV PATH=/opt/ICA-AROMA:$PATH
+ENV PATH="/opt/ICA-AROMA:$PATH" \
+    AROMA_VERSION="0.4.4-beta"
 
 # Installing and setting up miniconda
 RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \
diff --git a/docs/contributors.rst b/docs/contributors.rst
@@ -122,7 +122,12 @@ If it is necessary to rebuild the Docker image, a local image named
 ``fmriprep`` may be built from within the working fmriprep
 repository, located in ``~/projects/fmriprep``: ::
 
-    ~/projects/fmriprep$ docker build -t fmriprep .
+    ~/projects/fmriprep$ VERSION=$( python get_version.py )
+    ~/projects/fmriprep$ docker build -t fmriprep --build-arg VERSION=$VERSION .
+
+The ``VERSION`` build argument is necessary to ensure that help text
+can be reliably generated. The ``get_version.py`` tool constructs the
+version string from the current repository state.
 
 To work in this image, replace ``poldracklab/fmriprep:latest`` with
 ``fmriprep`` in any of the above commands.
@@ -232,4 +237,4 @@ An example of how this works is shown here: ::
     (transformation matrices, and six corresponding rotation and translation
     parameters) are estimated before any spatiotemporal filtering using
     `mcflirt` [FSL {fsl_ver}, @mcflirt].
-    """.format(fsl_ver=fsl.Info().version() or '<ver>')
+    """.format(fsl_ver=fsl.Info().version() or '<ver>')
diff --git a/fmriprep/cli/run.py b/fmriprep/cli/run.py
@@ -7,6 +7,7 @@
 """
 
 import os
+import re
 from pathlib import Path
 import logging
 import sys
@@ -549,7 +550,9 @@ def build_workflow(opts, retval):
     retval['run_uuid'] = run_uuid
 
     # First check that bids_dir looks like a BIDS folder
-    layout = BIDSLayout(str(bids_dir), validate=False)
+    layout = BIDSLayout(str(bids_dir), validate=False,
+                        ignore=("code", "stimuli", "sourcedata", "models",
+                                "derivatives", re.compile(r'^\.')))
     subject_list = collect_participants(
         layout, participant_label=opts.participant_label)
     retval['subject_list'] = subject_list
diff --git a/fmriprep/interfaces/confounds.py b/fmriprep/interfaces/confounds.py
@@ -107,6 +107,7 @@ class ICAConfoundsOutputSpec(TraitedSpec):
         File(exists=True, desc='output confounds file extracted from ICA-AROMA'))
     aroma_noise_ics = File(exists=True, desc='ICA-AROMA noise components')
     melodic_mix = File(exists=True, desc='melodic mix file')
+    aroma_metadata = File(exists=True, desc='tabulated ICA-AROMA metadata')
 
 
 class ICAConfounds(SimpleInterface):
@@ -116,8 +117,12 @@ class ICAConfounds(SimpleInterface):
     output_spec = ICAConfoundsOutputSpec
 
     def _run_interface(self, runtime):
-        aroma_confounds, motion_ics_out, melodic_mix_out = _get_ica_confounds(
-            self.inputs.in_directory, self.inputs.skip_vols, newpath=runtime.cwd)
+        (aroma_confounds,
+         motion_ics_out,
+         melodic_mix_out,
+         aroma_metadata) = _get_ica_confounds(self.inputs.in_directory,
+                                              self.inputs.skip_vols,
+                                              newpath=runtime.cwd)
 
         if self.inputs.err_on_aroma_warn and aroma_confounds is None:
             raise RuntimeError('ICA-AROMA failed')
@@ -126,6 +131,7 @@ def _run_interface(self, runtime):
 
         self._results['aroma_noise_ics'] = motion_ics_out
         self._results['melodic_mix'] = melodic_mix_out
+        self._results['aroma_metadata'] = aroma_metadata
         return runtime
 
 
@@ -218,10 +224,12 @@ def _get_ica_confounds(ica_out_dir, skip_vols, newpath=None):
     # load the txt files from ICA-AROMA
     melodic_mix = os.path.join(ica_out_dir, 'melodic.ica/melodic_mix')
     motion_ics = os.path.join(ica_out_dir, 'classified_motion_ICs.txt')
+    aroma_metadata = os.path.join(ica_out_dir, 'classification_overview.txt')
 
     # Change names of motion_ics and melodic_mix for output
     melodic_mix_out = os.path.join(newpath, 'MELODICmix.tsv')
     motion_ics_out = os.path.join(newpath, 'AROMAnoiseICs.csv')
+    aroma_metadata_out = os.path.join(newpath, 'classification_overview.tsv')
 
     # copy metion_ics file to derivatives name
     shutil.copyfile(motion_ics, motion_ics_out)
@@ -238,18 +246,27 @@ def _get_ica_confounds(ica_out_dir, skip_vols, newpath=None):
     # save melodic_mix_arr
     np.savetxt(melodic_mix_out, melodic_mix_arr, delimiter='\t')
 
+    # process the metadata so that the IC column entries match the BIDS name of
+    # the regressor
+    aroma_metadata = pd.read_csv(aroma_metadata, sep='\t')
+    aroma_metadata['IC'] = [
+        'aroma_motion_{}'.format(name) for name in aroma_metadata['IC']]
+    aroma_metadata.columns = [
+        re.sub('[ |\-|\/]', '_', c) for c in aroma_metadata.columns]
+    aroma_metadata.to_csv(aroma_metadata_out, sep='\t', index=False)
+
     # Return dummy list of ones if no noise compnents were found
     if motion_ic_indices.size == 0:
         LOGGER.warning('No noise components were classified')
-        return None, motion_ics_out, melodic_mix_out
+        return None, motion_ics_out, melodic_mix_out, aroma_metadata_out
 
     # the "good" ics, (e.g., not motion related)
     good_ic_arr = np.delete(melodic_mix_arr, motion_ic_indices, 1).T
 
     # return dummy lists of zeros if no signal components were found
     if good_ic_arr.size == 0:
         LOGGER.warning('No signal components were classified')
-        return None, motion_ics_out, melodic_mix_out
+        return None, motion_ics_out, melodic_mix_out, aroma_metadata_out
 
     # transpose melodic_mix_arr so x refers to the correct dimension
     aggr_confounds = np.asarray([melodic_mix_arr.T[x] for x in motion_ic_indices])
@@ -260,7 +277,7 @@ def _get_ica_confounds(ica_out_dir, skip_vols, newpath=None):
                  columns=['aroma_motion_%02d' % (x + 1) for x in motion_ic_indices]).to_csv(
         aroma_confounds, sep="\t", index=None)
 
-    return aroma_confounds, motion_ics_out, melodic_mix_out
+    return aroma_confounds, motion_ics_out, melodic_mix_out, aroma_metadata_out
 
 
 class FMRISummaryInputSpec(BaseInterfaceInputSpec):
diff --git a/fmriprep/workflows/bold/base.py b/fmriprep/workflows/bold/base.py
@@ -21,6 +21,7 @@
 
 from niworkflows.engine.workflows import LiterateWorkflow as Workflow
 from niworkflows.interfaces.cifti import GenerateCifti
+from niworkflows.interfaces.utils import DictMerge
 
 from ...utils.meepi import combine_meepi_source
 
@@ -406,7 +407,7 @@ def init_func_preproc_wf(
 
     outputnode = pe.Node(niu.IdentityInterface(
         fields=['bold_t1', 'bold_t1_ref', 'bold_mask_t1', 'bold_aseg_t1', 'bold_aparc_t1',
-                'bold_std', 'bold_std_ref' 'bold_mask_std', 'bold_aseg_std', 'bold_aparc_std',
+                'bold_std', 'bold_std_ref', 'bold_mask_std', 'bold_aseg_std', 'bold_aparc_std',
                 'bold_native', 'bold_cifti', 'cifti_variant', 'cifti_variant_key', 'surfaces',
                 'confounds', 'aroma_noise_ics', 'melodic_mix', 'nonaggr_denoised_file',
                 'confounds_metadata']),
@@ -875,10 +876,17 @@ def init_func_preproc_wf(
                                         function=_to_join),
                            name='aroma_confounds')
 
+            mrg_conf_metadata = pe.Node(niu.Merge(2), name='merge_confound_metadata',
+                                        run_without_submitting=True)
+            mrg_conf_metadata2 = pe.Node(DictMerge(), name='merge_confound_metadata2',
+                                         run_without_submitting=True)
             workflow.disconnect([
                 (bold_confounds_wf, outputnode, [
                     ('outputnode.confounds_file', 'confounds'),
                 ]),
+                (bold_confounds_wf, outputnode, [
+                    ('outputnode.confounds_metadata', 'confounds_metadata'),
+                ]),
             ])
             workflow.connect([
                 (bold_std_trans_wf, ica_aroma_wf, [
@@ -893,13 +901,19 @@ def init_func_preproc_wf(
                     ('outputnode.skip_vols', 'inputnode.skip_vols')]),
                 (bold_confounds_wf, join, [
                     ('outputnode.confounds_file', 'in_file')]),
+                (bold_confounds_wf, mrg_conf_metadata,
+                    [('outputnode.confounds_metadata', 'in1')]),
                 (ica_aroma_wf, join,
                     [('outputnode.aroma_confounds', 'join_file')]),
+                (ica_aroma_wf, mrg_conf_metadata,
+                    [('outputnode.aroma_metadata', 'in2')]),
+                (mrg_conf_metadata, mrg_conf_metadata2, [('out', 'in_dicts')]),
                 (ica_aroma_wf, outputnode,
                     [('outputnode.aroma_noise_ics', 'aroma_noise_ics'),
                      ('outputnode.melodic_mix', 'melodic_mix'),
                      ('outputnode.nonaggr_denoised_file', 'nonaggr_denoised_file')]),
                 (join, outputnode, [('out_file', 'confounds')]),
+                (mrg_conf_metadata2, outputnode, [('out_dict', 'confounds_metadata')]),
             ])
 
     # SURFACES ##################################################################################
diff --git a/fmriprep/workflows/bold/confounds.py b/fmriprep/workflows/bold/confounds.py
@@ -9,6 +9,7 @@
 .. autofunction:: init_ica_aroma_wf
 
 """
+from os import getenv
 from nipype.pipeline import engine as pe
 from nipype.interfaces import utility as niu, fsl
 from nipype.algorithms import confounds as nac
@@ -245,8 +246,9 @@ def init_bold_confs_wf(
         acompcor.inputs.repetition_time = metadata['RepetitionTime']
 
     # Global and segment regressors
+    signals_class_labels = ["csf", "white_matter", "global_signal"]
     mrg_lbl = pe.Node(niu.Merge(3), name='merge_rois', run_without_submitting=True)
-    signals = pe.Node(SignalExtraction(class_labels=["csf", "white_matter", "global_signal"]),
+    signals = pe.Node(SignalExtraction(class_labels=signals_class_labels),
                       name="signals", mem_gb=mem_gb)
 
     # Arrange confounds
@@ -270,8 +272,10 @@ def init_bold_confs_wf(
         TSV2JSON(index_column='component', output=None,
                  additional_metadata={'Method': 'aCompCor'}, enforce_case=True),
         name='acc_metadata_fmt')
-    mrg_conf_metadata = pe.Node(niu.Merge(2), name='merge_confound_metadata',
+    mrg_conf_metadata = pe.Node(niu.Merge(3), name='merge_confound_metadata',
                                 run_without_submitting=True)
+    mrg_conf_metadata.inputs.in3 = {label: {'Method': 'Mean'}
+                                    for label in signals_class_labels}
     mrg_conf_metadata2 = pe.Node(DictMerge(), name='merge_confound_metadata2',
                                  run_without_submitting=True)
 
@@ -648,7 +652,7 @@ def init_ica_aroma_wf(metadata, mem_gb, omp_nthreads,
 
     outputnode = pe.Node(niu.IdentityInterface(
         fields=['aroma_confounds', 'aroma_noise_ics', 'melodic_mix',
-                'nonaggr_denoised_file']), name='outputnode')
+                'nonaggr_denoised_file', 'aroma_metadata']), name='outputnode')
 
     select_std = pe.Node(KeySelect(
         fields=['bold_mask_std', 'bold_std']),
@@ -687,6 +691,13 @@ def _getusans_func(image, thresh):
     ica_aroma_confound_extraction = pe.Node(ICAConfounds(err_on_aroma_warn=err_on_aroma_warn),
                                             name='ica_aroma_confound_extraction')
 
+    ica_aroma_metadata_fmt = pe.Node(
+        TSV2JSON(index_column='IC', output=None, enforce_case=True,
+                 additional_metadata={'Method': {
+                                      'Name': 'ICA-AROMA',
+                                      'Version': getenv('AROMA_VERSION', 'n/a')}}),
+        name='ica_aroma_metadata_fmt')
+
     ds_report_ica_aroma = pe.Node(
         DerivativesDataSink(desc='aroma', keep_dtype=True),
         name='ds_report_ica_aroma', run_without_submitting=True,
@@ -732,10 +743,13 @@ def _getbtthresh(medianval):
         (ica_aroma, ica_aroma_confound_extraction, [('out_dir', 'in_directory')]),
         (inputnode, ica_aroma_confound_extraction, [
             ('skip_vols', 'skip_vols')]),
+        (ica_aroma_confound_extraction, ica_aroma_metadata_fmt, [
+            ('aroma_metadata', 'in_file')]),
         # output for processing and reporting
         (ica_aroma_confound_extraction, outputnode, [('aroma_confounds', 'aroma_confounds'),
                                                      ('aroma_noise_ics', 'aroma_noise_ics'),
                                                      ('melodic_mix', 'melodic_mix')]),
+        (ica_aroma_metadata_fmt, outputnode, [('output', 'aroma_metadata')]),
         (ica_aroma, add_non_steady_state, [
             ('nonaggr_denoised_file', 'bold_cut_file')]),
         (select_std, add_non_steady_state, [
diff --git a/setup.cfg b/setup.cfg
@@ -20,15 +20,15 @@ classifiers =
 [options]
 python_requires = >=3.5
 install_requires =
-    niworkflows ~= 0.10.3
+    niworkflows @ git+https://github.com/poldracklab/niworkflows.git@6118af2d8ef59abfa79f665da7caad646c40bc5c
     smriprep ~= 0.3.2
     templateflow ~= 0.4.1
     nibabel >=2.2.1
     indexed_gzip >=0.8.8
     nilearn !=0.5.0, !=0.5.1
     nipype >=1.2.2
     psutil >=5.4
-    pybids ~= 0.9.3
+    pybids ~= 0.9.4
     tedana >=0.0.5
     nitime
     numpy
