From 75702b6df1a0361a4ac1e3561961ae162984ffcf Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 20 Mar 2020 19:33:27 -0700 Subject: [PATCH 01/10] enh: replace the binary mask of MNI152NLin2009cAsym with the new probseg Resolves: #476 --- niworkflows/func/util.py | 19 +++++++++++++++++-- setup.cfg | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/niworkflows/func/util.py b/niworkflows/func/util.py index e8b38249e6e..51f54dfd767 100644 --- a/niworkflows/func/util.py +++ b/niworkflows/func/util.py @@ -345,6 +345,8 @@ def init_enhance_and_skullstrip_bold_wf( apply_mask = pe.Node(fsl.ApplyMask(), name="apply_mask") if not pre_mask: + from ..interfaces.nibabel import Binarize + bold_template = get_template( "MNI152NLin2009cAsym", resolution=2, desc="fMRIPrep", suffix="boldref" ) @@ -383,10 +385,22 @@ def init_enhance_and_skullstrip_bold_wf( norm.inputs.fixed_image = str(bold_template) map_brainmask = pe.Node( ApplyTransforms( - interpolation="MultiLabel", input_image=str(brain_mask) + interpolation="BSpline", + float=True, + # Use the higher resolution and probseg for numerical stability in rounding + input_image=str( + get_template( + "MNI152NLin2009cAsym", + resolution=1, + desc="brain", + suffix="probseg", + ) + ), ), name="map_brainmask", ) + binarize_mask = pe.Node(Binarize(thresh_low=0.75), name="binarize_mask") + # fmt: off workflow.connect([ (inputnode, init_aff, [("in_file", "moving_image")]), @@ -397,7 +411,8 @@ def init_enhance_and_skullstrip_bold_wf( ("reverse_invert_flags", "invert_transform_flags"), ("reverse_transforms", "transforms"), ]), - (map_brainmask, pre_dilate, [("output_image", "in_file")]), + (map_brainmask, binarize_mask, [("output_image", "in_file")]), + (binarize_mask, pre_dilate, [("out_mask", "in_file")]), ]) # fmt: on else: diff --git a/setup.cfg b/setup.cfg index 6ca1bf5a1cb..7f76bfb5067 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,7 +40,7 @@ install_requires = seaborn svgutils transforms3d - templateflow >= 0.4.2 + templateflow >= 0.5.2 test_requires = coverage < 5 pytest >= 4.4 From 62c878f71663b64cc4063ac0ccb983ff928c8ed4 Mon Sep 17 00:00:00 2001 From: oesteban Date: Fri, 20 Mar 2020 22:20:51 -0700 Subject: [PATCH 02/10] maint(ci): update circle and travis make sure that the new masks are uploaded to the artifacts --- .circleci/config.yml | 40 +++++++++++++++++------------ niworkflows/func/tests/test_util.py | 2 +- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 33e7527233a..7e35b15d240 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -285,10 +285,10 @@ jobs: - regression-v2-{{ .Revision }} - restore_cache: keys: - - masks-workdir-v1-{{ .Branch }}-{{epoch}} - - masks-workdir-v1-{{ .Branch }}- - - masks-workdir-v1-master- - - masks-workdir-v1- + - masks-workdir-v2-{{ .Branch }}-{{epoch}} + - masks-workdir-v2-{{ .Branch }}- + - masks-workdir-v2-master- + - masks-workdir-v2- - run: name: Run regression tests on EPI masks no_output_timeout: 2h @@ -305,14 +305,29 @@ jobs: coverage run -p --rcfile=setup.cfg \ -m pytest --junit-xml=/tmp/masks/reports/regression.xml \ niworkflows/func/tests/ - - save_cache: - key: masks-workdir-v1-{{ .Branch }}-{{ epoch }} - paths: - - /tmp/masks/workdir + - run: + name: Package new masks + when: always + no_output_timeout: 10m + command: | + cd reports/ + tar cvfz /tmp/masks/fmriprep_bold_mask.tar.gz fmriprep_bold_mask/*/*.nii.gz + - store_artifacts: + path: /tmp/masks/fmriprep_bold_mask.tar.gz + + - run: + name: Clear reports folder & delete plot generator cache + command: | + rm -rf /tmp/masks/reports/fmriprep_bold_mask/ + find workdir/ -name "mask_diff_plot" -exec rm -rf {} + - store_artifacts: path: /tmp/masks/reports - store_test_results: path: /tmp/masks/reports + - save_cache: + key: masks-workdir-v2-{{ .Branch }}-{{ epoch }} + paths: + - /tmp/masks/workdir - run: name: Coverage preparation @@ -337,15 +352,6 @@ jobs: cp /tmp/masks/reports/coverage.xml . sed -i "s+/src/niworkflows+/tmp/src/niworkflows+g" coverage.xml python -m codecov --file coverage.xml --flags masks -e CIRCLE_JOB - - run: - name: Package new masks - when: always - no_output_timeout: 10m - working_directory: /tmp/data - command: | - tar cfz /tmp/masks/fmriprep_bold_mask.tar.gz fmriprep_bold_mask/*/*.nii.gz - - store_artifacts: - path: /tmp/masks/fmriprep_bold_mask.tar.gz test_package: machine: diff --git a/niworkflows/func/tests/test_util.py b/niworkflows/func/tests/test_util.py index 68351be3191..ef59d3f517e 100755 --- a/niworkflows/func/tests/test_util.py +++ b/niworkflows/func/tests/test_util.py @@ -85,7 +85,7 @@ def test_masking(input_fname, expected_fname): bold_reference_wf.base_dir = str(base_dir) out_fname = fname_presuffix( - basename, suffix="_masks.svg", use_ext=False, newpath=str(newpath) + basename, suffix="_mask.svg", use_ext=False, newpath=str(newpath) ) newpath.mkdir(parents=True, exist_ok=True) From b027ea6c0ad14466eddee39feda4ab5108a6ad4f Mon Sep 17 00:00:00 2001 From: oesteban Date: Sat, 21 Mar 2020 16:31:28 -0700 Subject: [PATCH 03/10] maint: update circleci regression masks --- .circleci/config.yml | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7e35b15d240..262f47022c0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -151,8 +151,8 @@ jobs: steps: - restore_cache: keys: - - regression-v2-{{ .Revision }} - - regression-v2- + - regression-v3-{{ .Revision }} + - regression-v3- - run: name: Get truncated BOLD series command: | @@ -175,7 +175,7 @@ jobs: echo "Pre-computed masks were cached" fi - save_cache: - key: regression-v2-{{ .Revision }}-{{ epoch }} + key: regression-v3-{{ .Revision }}-{{ epoch }} paths: - /tmp/data @@ -282,7 +282,7 @@ jobs: - restore_cache: keys: - - regression-v2-{{ .Revision }} + - regression-v3-{{ .Revision }} - restore_cache: keys: - masks-workdir-v2-{{ .Branch }}-{{epoch}} @@ -305,20 +305,13 @@ jobs: coverage run -p --rcfile=setup.cfg \ -m pytest --junit-xml=/tmp/masks/reports/regression.xml \ niworkflows/func/tests/ - - run: - name: Package new masks - when: always - no_output_timeout: 10m - command: | - cd reports/ - tar cvfz /tmp/masks/fmriprep_bold_mask.tar.gz fmriprep_bold_mask/*/*.nii.gz - - store_artifacts: - path: /tmp/masks/fmriprep_bold_mask.tar.gz - - run: name: Clear reports folder & delete plot generator cache command: | + pushd reports/ + tar cvfz fmriprep_bold_mask.tar.gz fmriprep_bold_mask/*/*.nii.gz rm -rf /tmp/masks/reports/fmriprep_bold_mask/ + popd find workdir/ -name "mask_diff_plot" -exec rm -rf {} + - store_artifacts: path: /tmp/masks/reports From 9068d19a067ce0dfa733a7a3e72c6d021e1be50d Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Wed, 25 Mar 2020 09:13:14 -0700 Subject: [PATCH 04/10] Update niworkflows/func/util.py Co-Authored-By: Chris Markiewicz --- niworkflows/func/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/niworkflows/func/util.py b/niworkflows/func/util.py index 51f54dfd767..1a006522789 100644 --- a/niworkflows/func/util.py +++ b/niworkflows/func/util.py @@ -399,7 +399,7 @@ def init_enhance_and_skullstrip_bold_wf( ), name="map_brainmask", ) - binarize_mask = pe.Node(Binarize(thresh_low=0.75), name="binarize_mask") + binarize_mask = pe.Node(Binarize(thresh_low=0.5), name="binarize_mask") # fmt: off workflow.connect([ From 6fee9256cbe3b199031231d272adc9f56af9e99f Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 7 Apr 2020 18:27:44 -0700 Subject: [PATCH 05/10] fix(templateflow): fix a bad API name accessing a probseg resource --- docs/requirements.txt | 2 +- niworkflows/func/util.py | 2 +- setup.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 5e52cc136db..3c5b4594812 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -10,4 +10,4 @@ sphinx-argparse sphinx>=2.1.2,<3.0 sphinx_rtd_theme sphinxcontrib-apidoc ~= 0.3.0 -templateflow +templateflow >= 0.6.0rc1 diff --git a/niworkflows/func/util.py b/niworkflows/func/util.py index 1a006522789..5ba2df1ac4e 100644 --- a/niworkflows/func/util.py +++ b/niworkflows/func/util.py @@ -392,7 +392,7 @@ def init_enhance_and_skullstrip_bold_wf( get_template( "MNI152NLin2009cAsym", resolution=1, - desc="brain", + label="brain", suffix="probseg", ) ), diff --git a/setup.cfg b/setup.cfg index 7f76bfb5067..3966e5d3ac0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,7 +40,7 @@ install_requires = seaborn svgutils transforms3d - templateflow >= 0.5.2 + templateflow >= 0.6.0rc1 test_requires = coverage < 5 pytest >= 4.4 From 960c872cb96a189f3eb89112eabb2d41d02adbd7 Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 8 Apr 2020 12:09:09 -0700 Subject: [PATCH 06/10] tests: update masks for regression tests --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 262f47022c0..3e06ad2ee4f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -151,8 +151,8 @@ jobs: steps: - restore_cache: keys: - - regression-v3-{{ .Revision }} - - regression-v3- + - regression-v4-{{ .Revision }} + - regression-v4- - run: name: Get truncated BOLD series command: | @@ -175,7 +175,7 @@ jobs: echo "Pre-computed masks were cached" fi - save_cache: - key: regression-v3-{{ .Revision }}-{{ epoch }} + key: regression-v4-{{ .Revision }}-{{ epoch }} paths: - /tmp/data @@ -282,7 +282,7 @@ jobs: - restore_cache: keys: - - regression-v3-{{ .Revision }} + - regression-v4-{{ .Revision }} - restore_cache: keys: - masks-workdir-v2-{{ .Branch }}-{{epoch}} From 66c37d47a95ca0c1aad2042d50deede7c092ac5b Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 6 May 2020 19:00:42 -0700 Subject: [PATCH 07/10] chore(pin): templateflow ~= 0.6 (and remove requirements.txt) Taking advantage of the new auto-update. This commit also removes the requirements.txt file that was used exclusively by the Docker image distribution: pyntcloud now is in Pypi and we can safely pin templateflow to >=0.6.0 since the autoupdate was implemented. --- docs/requirements.txt | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 3c5b4594812..5e52cc136db 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -10,4 +10,4 @@ sphinx-argparse sphinx>=2.1.2,<3.0 sphinx_rtd_theme sphinxcontrib-apidoc ~= 0.3.0 -templateflow >= 0.6.0rc1 +templateflow diff --git a/setup.cfg b/setup.cfg index 3966e5d3ac0..382f97a3075 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,7 +40,7 @@ install_requires = seaborn svgutils transforms3d - templateflow >= 0.6.0rc1 + templateflow >= 0.6 test_requires = coverage < 5 pytest >= 4.4 From 2af5e715f5cbdff5157abfd0158f4850cf455569 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 2 Jun 2020 08:49:34 -0700 Subject: [PATCH 08/10] fix: try a less generous threshold (0.15) for binarization --- niworkflows/func/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/niworkflows/func/util.py b/niworkflows/func/util.py index 5ba2df1ac4e..330c5c659b9 100644 --- a/niworkflows/func/util.py +++ b/niworkflows/func/util.py @@ -399,7 +399,7 @@ def init_enhance_and_skullstrip_bold_wf( ), name="map_brainmask", ) - binarize_mask = pe.Node(Binarize(thresh_low=0.5), name="binarize_mask") + binarize_mask = pe.Node(Binarize(thresh_low=0.15), name="binarize_mask") # fmt: off workflow.connect([ From 294a8679d0a1eb3a7536429b7c794fa902f01703 Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 2 Jun 2020 17:06:03 -0700 Subject: [PATCH 09/10] enh: add ``brainmask_thresh`` as a parameter --- niworkflows/func/util.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/niworkflows/func/util.py b/niworkflows/func/util.py index 330c5c659b9..995c76223bd 100644 --- a/niworkflows/func/util.py +++ b/niworkflows/func/util.py @@ -29,6 +29,7 @@ def init_bold_reference_wf( omp_nthreads, bold_file=None, + brainmask_thresh=0.5, pre_mask=False, name="bold_reference_wf", gen_report=False, @@ -50,10 +51,16 @@ def init_bold_reference_wf( Parameters ---------- - bold_file : str - BOLD series NIfTI file omp_nthreads : int Maximum number of threads an individual process may use + bold_file : str + BOLD series NIfTI file + brainmask_thresh: :obj:`float` + Lower threshold for the probabilistic brainmask to obtain + the final binary mask (default: 0.5). + pre_mask : bool + Indicates whether the ``pre_mask`` input will be set (and thus, step 1 + should be skipped). name : str Name of workflow (default: ``bold_reference_wf``) gen_report : bool @@ -133,7 +140,9 @@ def init_bold_reference_wf( EstimateReferenceImage(), name="gen_ref", mem_gb=1 ) # OE: 128x128x128x50 * 64 / 8 ~ 900MB. enhance_and_skullstrip_bold_wf = init_enhance_and_skullstrip_bold_wf( - omp_nthreads=omp_nthreads, pre_mask=pre_mask + brainmask_thresh=brainmask_thresh, + omp_nthreads=omp_nthreads, + pre_mask=pre_mask, ) calc_dummy_scans = pe.Node( @@ -188,7 +197,10 @@ def init_bold_reference_wf( def init_enhance_and_skullstrip_bold_wf( - name="enhance_and_skullstrip_bold_wf", pre_mask=False, omp_nthreads=1 + brainmask_thresh=0.5, + name="enhance_and_skullstrip_bold_wf", + omp_nthreads=1, + pre_mask=False, ): """ Enhance and run brain extraction on a BOLD EPI image. @@ -238,13 +250,16 @@ def init_enhance_and_skullstrip_bold_wf( Parameters ---------- + brainmask_thresh: :obj:`float` + Lower threshold for the probabilistic brainmask to obtain + the final binary mask (default: 0.5). name : str Name of workflow (default: ``enhance_and_skullstrip_bold_wf``) + omp_nthreads : int + number of threads available to parallel nodes pre_mask : bool Indicates whether the ``pre_mask`` input will be set (and thus, step 1 should be skipped). - omp_nthreads : int - number of threads available to parallel nodes Inputs ------ @@ -399,7 +414,7 @@ def init_enhance_and_skullstrip_bold_wf( ), name="map_brainmask", ) - binarize_mask = pe.Node(Binarize(thresh_low=0.15), name="binarize_mask") + binarize_mask = pe.Node(Binarize(thresh_low=brainmask_thresh), name="binarize_mask") # fmt: off workflow.connect([ From ef680deb69ea83aa60e30d5140395fc821568edf Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 5 Jun 2020 09:23:13 -0700 Subject: [PATCH 10/10] Update util.py --- niworkflows/func/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/niworkflows/func/util.py b/niworkflows/func/util.py index 995c76223bd..c2c4496268b 100644 --- a/niworkflows/func/util.py +++ b/niworkflows/func/util.py @@ -29,7 +29,7 @@ def init_bold_reference_wf( omp_nthreads, bold_file=None, - brainmask_thresh=0.5, + brainmask_thresh=0.85, pre_mask=False, name="bold_reference_wf", gen_report=False, @@ -57,7 +57,7 @@ def init_bold_reference_wf( BOLD series NIfTI file brainmask_thresh: :obj:`float` Lower threshold for the probabilistic brainmask to obtain - the final binary mask (default: 0.5). + the final binary mask (default: 0.85). pre_mask : bool Indicates whether the ``pre_mask`` input will be set (and thus, step 1 should be skipped).