Merge remote-tracking branch 'upstream/master'

Author: dPys

.circleci/config.yml

@@ -251,6 +251,24 @@ jobs:
          key: THP002-anat-v00-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
          paths:
             - /tmp/THP002/work
+      - run:
+          name: Run full diffusion workflow on THP002
+          no_output_timeout: 2h
+          command: |
+            mkdir -p /tmp/THP002/work /tmp/THP002/derivatives
+            sudo setfacl -d -m group:$(id -gn):rwx /tmp/THP002/derivatives && \
+                sudo setfacl -m group:$(id -gn):rwx /tmp/THP002/derivatives
+            sudo setfacl -d -m group:$(id -gn):rwx /tmp/THP002/work && \
+                sudo setfacl -m group:$(id -gn):rwx /tmp/THP002/work
+            docker run -e FS_LICENSE=$FS_LICENSE --rm \
+                -v /tmp/data/THP002:/data \
+                -v /tmp/THP002/derivatives:/out \
+                -v /tmp/fslicense/license.txt:/tmp/fslicense/license.txt:ro \
+                -v /tmp/config/nipype.cfg:/home/dmriprep/.nipype/nipype.cfg \
+                -v /tmp/THP002/work:/work \
+                --user $(id -u):$(id -g) \
+                nipreps/dmriprep:latest /data /out participant -vv --sloppy \
+                --notrack --skip-bids-validation -w /work --omp-nthreads 2 --nprocs 2
       - store_artifacts:
           path: /tmp/THP002/derivatives/dmriprep
       - run:

CHANGES.rst

@@ -1,3 +1,15 @@
+0.2.2 (January 15, 2020)
+========================
+A release to show the deployment process on the Sprint.
+
+  * ENH: b0 reference and skullstrip workflow (#50)
+  * FIX: Version on docker target of ``Makefile`` (#54)
+  * FIX/ENH: Remove sentry tracking and add popylar tracking. (#51)
+  * MAINT: Some small changes to the Dockerfile. (#53)
+  * ENH: Set up customized report specification (#34)
+  * MAINT: Use a local docker registry instead of load/save (#46)
+
+
 0.2.1 (December 12, 2019)
 =========================
 A bugfix release to test that versioned documentation is working.

Dockerfile

@@ -154,8 +154,8 @@ RUN python -c "from matplotlib import font_manager" && \
     sed -i 's/\(backend *: \).*$/\1Agg/g' $( python -c "import matplotlib; print(matplotlib.matplotlib_fname())" )
 
 # Precaching atlases
-COPY setup.cfg fmriprep-setup.cfg
-RUN pip install --no-cache-dir "$( grep templateflow fmriprep-setup.cfg | xargs )" && \
+COPY setup.cfg dmriprep-setup.cfg
+RUN pip install --no-cache-dir "$( grep templateflow dmriprep-setup.cfg | xargs )" && \
     python -c "from templateflow import api as tfapi; \
                tfapi.get('MNI152NLin6Asym', atlas=None, resolution=[1, 2], extension=['.nii', '.nii.gz']); \
                tfapi.get('MNI152NLin2009cAsym', atlas=None, extension=['.nii', '.nii.gz']); \
@@ -169,13 +169,12 @@ ARG VERSION
 # Force static versioning within container
 RUN echo "${VERSION}" > /src/dmriprep/dmriprep/VERSION && \
     echo "include dmriprep/VERSION" >> /src/dmriprep/MANIFEST.in && \
-    pip install --no-cache-dir "/src/dmriprep[all]"
+    cd /src/dmriprep && \
+    pip install --no-cache-dir .[all]
 
 RUN find $HOME -type d -exec chmod go=u {} + && \
     find $HOME -type f -exec chmod go=u {} +
 
-ENV IS_DOCKER_8395080871=1
-
 RUN ldconfig
 WORKDIR /tmp/
 ENTRYPOINT ["/usr/local/miniconda/bin/dmriprep"]

Makefile

@@ -87,8 +87,8 @@ dist: clean ## builds source and wheel package
 install: clean ## install the package to the active Python's site-packages
 	python setup.py install
 
-docker: docker
+docker:
 	docker build --rm -t nipreps/dmriprep:latest \
 		--build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \
 		--build-arg VCS_REF=`git rev-parse --short HEAD` \
-		--build-arg VERSION=$( python get_version.py ) .
+		--build-arg VERSION=`python get_version.py` .

dmriprep/__about__.py

@@ -14,3 +14,5 @@
 DOWNLOAD_URL = (
     'https://github.com/nipreps/{name}/archive/{ver}.tar.gz'.format(
         name=__packagename__, ver=__version__))
+
+__ga_id__ = "UA-156165436-1"

dmriprep/cli/run.py

@@ -214,6 +214,7 @@ def main():
     from nipype import logging as nlogging
     from multiprocessing import set_start_method, Process, Manager
     from ..utils.bids import write_derivative_description, validate_input_dir
+    from ..__about__ import __ga_id__
     set_start_method('forkserver')
     warnings.showwarning = _warn_redirect
     opts = get_parser().parse_args()
@@ -229,11 +230,9 @@ def main():
             if os.getenv('DOCKER_VERSION_8395080871'):
                 exec_env = 'dmriprep-docker'
 
-    sentry_sdk = None
     if not opts.notrack:
-        import sentry_sdk
-        from ..utils.sentry import sentry_setup
-        sentry_setup(opts, exec_env)
+        import popylar
+        popylar.track_event(__ga_id__, 'run', 'cli_run')
 
     # Validate inputs
     if not opts.skip_bids_validation:
@@ -301,27 +300,14 @@ def main():
     # Clean up master process before running workflow, which may create forks
     gc.collect()
 
-    # Sentry tracking
-    if not opts.notrack:
-        from ..utils.sentry import start_ping
-        start_ping(run_uuid, len(subject_list))
-
     errno = 1  # Default is error exit unless otherwise set
     try:
         dmriprep_wf.run(**plugin_settings)
     except Exception as e:
         if not opts.notrack:
-            from ..utils.sentry import process_crashfile
-            crashfolders = [output_dir / 'dmriprep' / 'sub-{}'.format(s) / 'log' / run_uuid
-                            for s in subject_list]
-            for crashfolder in crashfolders:
-                for crashfile in crashfolder.glob('crash*.*'):
-                    process_crashfile(crashfile)
-
-            if "Workflow did not execute cleanly" not in str(e):
-                sentry_sdk.capture_exception(e)
+            popylar.track_event(__ga_id__, 'run', 'cli_error')
         logger.critical('dMRIPrep failed: %s', e)
-        raise
+        raise e
     else:
         if opts.run_reconall:
             from templateflow import api
@@ -334,8 +320,8 @@ def main():
         errno = 0
         logger.log(25, 'dMRIPrep finished without errors')
         if not opts.notrack:
-            sentry_sdk.capture_message('dMRIPrep finished without errors',
-                                       level='info')
+            popylar.track_event(__ga_id__, 'run', 'cli_finished')
+
     finally:
         from niworkflows.reports import generate_reports
         from subprocess import check_call, CalledProcessError, TimeoutExpired
@@ -389,10 +375,6 @@ def main():
             packagename='dmriprep')
         write_derivative_description(bids_dir, output_dir / 'dmriprep')
 
-        if failed_reports and not opts.notrack:
-            sentry_sdk.capture_message(
-                'Report generation failed for %d subjects' % failed_reports,
-                level='error')
         sys.exit(int((errno + failed_reports) > 0))
 
 

dmriprep/config/__init__.py

@@ -2,4 +2,5 @@
 # vi: set ft=python sts=4 ts=4 sw=4 et:
 """Settings."""
 
+DEFAULT_MEMORY_MIN_GB = 0.01
 NONSTANDARD_REFERENCES = ['anat', 'T1w', 'dwi', 'fsnative']

dmriprep/config/reports-spec.yml

@@ -26,6 +26,16 @@ sections:
     caption: Surfaces (white and pial) reconstructed with FreeSurfer (<code>recon-all</code>)
       overlaid on the participant's T1w template.
     subtitle: Surface reconstruction
+- name: Diffusion
+  ordering: session,acquisition,run
+  reportlets:
+  - bids: {datatype: dwi, desc: validation, suffix: dwi}
+  - bids: {datatype: dwi, desc: brain, suffix: mask}
+    caption: Average b=0 that serves for reference in early preprocessing steps.
+    descriptions: The reference b=0 is obtained as the voxel-wise median across
+      all b=0 found in the dataset, after accounting for signal drift.
+      The red contour shows the brain mask calculated using this reference b=0.
+    subtitle: Reference b=0 and brain mask
 - name: About
   reportlets:
   - bids: {datatype: anat, desc: about, suffix: T1w}

dmriprep/data/tests/dwi_mask.nii.gz

@@ -0,0 +1,145 @@
+"""Image tools interfaces."""
+import numpy as np
+import nibabel as nb
+from nipype.utils.filemanip import fname_presuffix
+from nipype import logging
+from nipype.interfaces.base import (
+    traits, TraitedSpec, BaseInterfaceInputSpec, SimpleInterface, File
+)
+
+LOGGER = logging.getLogger('nipype.interface')
+
+
+class _ExtractB0InputSpec(BaseInterfaceInputSpec):
+    in_file = File(exists=True, mandatory=True, desc='dwi file')
+    b0_ixs = traits.List(traits.Int, mandatory=True,
+                         desc='Index of b0s')
+
+
+class _ExtractB0OutputSpec(TraitedSpec):
+    out_file = File(exists=True, desc='b0 file')
+
+
+class ExtractB0(SimpleInterface):
+    """
+    Extract all b=0 volumes from a dwi series.
+
+    Example
+    -------
+    >>> os.chdir(tmpdir)
+    >>> extract_b0 = ExtractB0()
+    >>> extract_b0.inputs.in_file = str(data_dir / 'dwi.nii.gz')
+    >>> extract_b0.inputs.b0_ixs = [0, 1, 2]
+    >>> res = extract_b0.run()  # doctest: +SKIP
+
+    """
+
+    input_spec = _ExtractB0InputSpec
+    output_spec = _ExtractB0OutputSpec
+
+    def _run_interface(self, runtime):
+        self._results['out_file'] = extract_b0(
+            self.inputs.in_file,
+            self.inputs.b0_ixs,
+            newpath=runtime.cwd)
+        return runtime
+
+
+def extract_b0(in_file, b0_ixs, newpath=None):
+    """Extract the *b0* volumes from a DWI dataset."""
+    out_file = fname_presuffix(
+        in_file, suffix='_b0', newpath=newpath)
+
+    img = nb.load(in_file)
+    data = img.get_fdata(dtype='float32')
+
+    b0 = data[..., b0_ixs]
+
+    hdr = img.header.copy()
+    hdr.set_data_shape(b0.shape)
+    hdr.set_xyzt_units('mm')
+    hdr.set_data_dtype(np.float32)
+    nb.Nifti1Image(b0, img.affine, hdr).to_filename(out_file)
+    return out_file
+
+
+class _RescaleB0InputSpec(BaseInterfaceInputSpec):
+    in_file = File(exists=True, mandatory=True, desc='b0s file')
+    mask_file = File(exists=True, mandatory=True, desc='mask file')
+
+
+class _RescaleB0OutputSpec(TraitedSpec):
+    out_ref = File(exists=True, desc='One average b0 file')
+    out_b0s = File(exists=True, desc='series of rescaled b0 volumes')
+
+
+class RescaleB0(SimpleInterface):
+    """
+    Rescale the b0 volumes to deal with average signal decay over time.
+
+    Example
+    -------
+    >>> os.chdir(tmpdir)
+    >>> rescale_b0 = RescaleB0()
+    >>> rescale_b0.inputs.in_file = str(data_dir / 'dwi.nii.gz')
+    >>> rescale_b0.inputs.mask_file = str(data_dir / 'dwi_mask.nii.gz')
+    >>> res = rescale_b0.run()  # doctest: +SKIP
+
+    """
+
+    input_spec = _RescaleB0InputSpec
+    output_spec = _RescaleB0OutputSpec
+
+    def _run_interface(self, runtime):
+        self._results['out_b0s'] = rescale_b0(
+            self.inputs.in_file,
+            self.inputs.mask_file,
+            newpath=runtime.cwd
+        )
+        self._results['out_ref'] = median(
+            self._results['out_b0s'],
+            newpath=runtime.cwd
+        )
+        return runtime
+
+
+def rescale_b0(in_file, mask_file, newpath=None):
+    """Rescale the input volumes using the median signal intensity."""
+    out_file = fname_presuffix(
+        in_file, suffix='_rescaled_b0', newpath=newpath)
+
+    img = nb.load(in_file)
+    if img.dataobj.ndim == 3:
+        return in_file
+
+    data = img.get_fdata(dtype='float32')
+    mask_img = nb.load(mask_file)
+    mask_data = mask_img.get_fdata(dtype='float32')
+
+    median_signal = np.median(data[mask_data > 0, ...], axis=0)
+    rescaled_data = 1000 * data / median_signal
+    hdr = img.header.copy()
+    nb.Nifti1Image(rescaled_data, img.affine, hdr).to_filename(out_file)
+    return out_file
+
+
+def median(in_file, newpath=None):
+    """Average a 4D dataset across the last dimension using median."""
+    out_file = fname_presuffix(
+        in_file, suffix='_b0ref', newpath=newpath)
+
+    img = nb.load(in_file)
+    if img.dataobj.ndim == 3:
+        return in_file
+    if img.shape[-1] == 1:
+        nb.squeeze_image(img).to_filename(out_file)
+        return out_file
+
+    median_data = np.median(img.get_fdata(dtype='float32'),
+                            axis=-1)
+
+    hdr = img.header.copy()
+    hdr.set_xyzt_units('mm')
+    hdr.set_data_dtype(np.float32)
+    nb.Nifti1Image(median_data, img.affine, hdr).to_filename(out_file)
+    return out_file