Merge remote-tracking branch 'upstream/next' into enh/fs_long

Author: bpinsard

.circleci/config.yml

@@ -12,6 +12,9 @@ _machine_defaults: &machine_defaults
 _python_defaults: &python_defaults
   docker:
     - image: cimg/python:3.10.9
+      auth:
+        username: $DOCKER_USER
+        password: $DOCKER_PAT
   working_directory: /tmp/src/smriprep
 
 _docker_auth: &docker_auth
@@ -43,6 +46,9 @@ _pull_from_registry: &pull_from_registry
 docs_deploy: &docs
   docker:
     - image: node:8.10.0
+      auth:
+        username: $DOCKER_USER
+        password: $DOCKER_PAT
   working_directory: /tmp/gh-pages
   steps:
     - run:
@@ -282,8 +288,10 @@ jobs:
           paths:
             - /home/circleci/.local
 
-  test_wrapper:
+  test:
     <<: *machine_defaults
+    environment:
+      - FS_LICENSE: /tmp/fslicense/license.txt
     steps:
       - attach_workspace:
           at: /tmp
@@ -323,6 +331,24 @@ jobs:
             which smriprep-docker
             smriprep-docker -i nipreps/smriprep:latest --help
             smriprep-docker -i nipreps/smriprep:latest --version
+      - restore_cache:
+          keys:
+            - testdata-v2-{{ .Branch }}-{{ epoch }}
+            - testdata-v2-{{ .Branch }}
+            - testdata-v2-
+      - restore_cache:
+          keys:
+            - templateflow-v1-{{ .Branch }}-{{ epoch }}
+            - templateflow-v1-{{ .Branch }}
+            - templateflow-v1-
+      - run:
+          name: Run Pytest
+          no_output_timeout: 2h
+          command: |
+            bash /tmp/src/smriprep/.circleci/pytest.sh
+      - codecov/upload:
+          file: /tmp/data/pytest_cov.xml
+          flags: pytest
 
   test_deploy_pypi:
     <<: *python_defaults
@@ -414,9 +440,9 @@ jobs:
       - run: *pull_from_registry
       - restore_cache:
           keys:
-            - ds005-anat-v9-{{ .Branch }}-{{ epoch }}
-            - ds005-anat-v9-{{ .Branch }}
-            - ds005-anat-v9-master
+            - ds005-anat-v0-{{ .Branch }}-{{ epoch }}
+            - ds005-anat-v0-{{ .Branch }}
+            - ds005-anat-v0-next
       - restore_cache:
           keys:
             - testdata-v2-{{ .Branch }}-{{ epoch }}
@@ -470,7 +496,7 @@ jobs:
             rm -rf /tmp/ds005/work/reportlets
             rm -rf /tmp/ds005/work/smriprep_wf/fsdir_run_*/
       - save_cache:
-          key: ds005-anat-v9-{{ .Branch }}-{{ epoch }}
+          key: ds005-anat-v0-{{ .Branch }}-{{ epoch }}
           paths:
             - /tmp/ds005/work
 
@@ -772,8 +798,6 @@ workflows:
               only: /.*/
 
       - get_data:
-          context:
-            - fs-license
           filters:
             branches:
               ignore:
@@ -793,10 +817,11 @@ workflows:
             tags:
               only: /.*/
 
-      - test_wrapper:
+      - test:
           context:
             - nipreps-common
           requires:
+            - get_data
             - build
           filters:
             branches:
@@ -852,7 +877,7 @@ workflows:
             - build_docs
             - ds005
             - ds054
-            - test_wrapper
+            - test
           filters:
             branches:
               only: /master/
@@ -865,7 +890,7 @@ workflows:
             - ds005
             - ds054
             - test_deploy_pypi
-            - test_wrapper
+            - test
           filters:
             branches:
               ignore: /.*/

.circleci/ds005_outputs.txt

@@ -16,6 +16,7 @@ smriprep/sub-01/anat/sub-01_desc-brain_mask.json
 smriprep/sub-01/anat/sub-01_desc-brain_mask.nii.gz
 smriprep/sub-01/anat/sub-01_desc-preproc_T1w.json
 smriprep/sub-01/anat/sub-01_desc-preproc_T1w.nii.gz
+smriprep/sub-01/anat/sub-01_desc-ribbon_mask.json
 smriprep/sub-01/anat/sub-01_desc-ribbon_mask.nii.gz
 smriprep/sub-01/anat/sub-01_dseg.nii.gz
 smriprep/sub-01/anat/sub-01_from-fsnative_to-T1w_mode-image_xfm.txt
@@ -25,7 +26,9 @@ smriprep/sub-01/anat/sub-01_hemi-L_desc-reg_sphere.surf.gii
 smriprep/sub-01/anat/sub-01_hemi-L_inflated.surf.gii
 smriprep/sub-01/anat/sub-01_hemi-L_midthickness.surf.gii
 smriprep/sub-01/anat/sub-01_hemi-L_pial.surf.gii
+smriprep/sub-01/anat/sub-01_hemi-L_space-fsLR_desc-msmsulc_sphere.surf.gii
 smriprep/sub-01/anat/sub-01_hemi-L_space-fsLR_desc-reg_sphere.surf.gii
+smriprep/sub-01/anat/sub-01_hemi-L_sphere.surf.gii
 smriprep/sub-01/anat/sub-01_hemi-L_sulc.shape.gii
 smriprep/sub-01/anat/sub-01_hemi-L_thickness.shape.gii
 smriprep/sub-01/anat/sub-01_hemi-L_white.surf.gii
@@ -34,12 +37,20 @@ smriprep/sub-01/anat/sub-01_hemi-R_desc-reg_sphere.surf.gii
 smriprep/sub-01/anat/sub-01_hemi-R_inflated.surf.gii
 smriprep/sub-01/anat/sub-01_hemi-R_midthickness.surf.gii
 smriprep/sub-01/anat/sub-01_hemi-R_pial.surf.gii
+smriprep/sub-01/anat/sub-01_hemi-R_space-fsLR_desc-msmsulc_sphere.surf.gii
 smriprep/sub-01/anat/sub-01_hemi-R_space-fsLR_desc-reg_sphere.surf.gii
+smriprep/sub-01/anat/sub-01_hemi-R_sphere.surf.gii
 smriprep/sub-01/anat/sub-01_hemi-R_sulc.shape.gii
 smriprep/sub-01/anat/sub-01_hemi-R_thickness.shape.gii
 smriprep/sub-01/anat/sub-01_hemi-R_white.surf.gii
 smriprep/sub-01/anat/sub-01_label-CSF_probseg.nii.gz
 smriprep/sub-01/anat/sub-01_label-GM_probseg.nii.gz
 smriprep/sub-01/anat/sub-01_label-WM_probseg.nii.gz
+smriprep/sub-01/anat/sub-01_space-fsLR_den-91k_curv.dscalar.nii
+smriprep/sub-01/anat/sub-01_space-fsLR_den-91k_curv.json
+smriprep/sub-01/anat/sub-01_space-fsLR_den-91k_sulc.dscalar.nii
+smriprep/sub-01/anat/sub-01_space-fsLR_den-91k_sulc.json
+smriprep/sub-01/anat/sub-01_space-fsLR_den-91k_thickness.dscalar.nii
+smriprep/sub-01/anat/sub-01_space-fsLR_den-91k_thickness.json
 smriprep/sub-01.html
 /tmp/ds005/derivatives

.circleci/ds005_run.sh

@@ -19,4 +19,5 @@ docker run -it -e FMRIPREP_DEV=1 -u $(id -u) \
     --ncpus 2 --omp-nthreads 2 -vv \
     --fs-license-file /tmp/fslicense/license.txt \
     --fs-subjects-dir /tmp/ds005/freesurfer \
+    --cifti-output \
     ${@:1}

.circleci/pytest.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+docker run --rm -it \
+    -v /tmp/data:/tmp/data:rw \
+    -v /tmp/fslicense:/tmp/fslicense:ro \
+    -v /tmp/templateflow:/home/smriprep/.cache/templateflow \
+    -v /tmp/src/smriprep/docker/multiproc.coveragerc:/tmp/multiproc.coveragerc:ro \
+    -v /tmp/src/smriprep/.circleci/nipype.cfg:/home/smriprep/.nipype/nipype.cfg \
+    -e FS_LICENSE=/tmp/fslicense/license.txt \
+    --entrypoint=pytest \
+    nipreps/smriprep:latest \
+    -v --doctest-modules --pyargs smriprep \
+    --cov smriprep --cov-report=xml:/tmp/data/pytest_cov.xml \
+    ${@:1}

.github/workflows/pythonpackage.yml

@@ -5,7 +5,7 @@ on:
     branches: [ '*' ]
     tags: [ '*' ]
   pull_request:
-    branches: [ master, 'maint/*' ]
+    branches: [ master, 'maint/*', 'next' ]
   schedule:
     - cron: '0 0 * * *'
 
@@ -65,7 +65,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8, 3.9, "3.10", "3.11"]
+        python-version: ["3.10", "3.11"]
         install: [repo]
         include:
           - python-version: "3.11"

.gitignore

@@ -311,3 +311,4 @@ Icon
 Network Trash Folder
 Temporary Items
 .apdisk
+>>>>>>> upstream/master

docs/conf.py

@@ -64,7 +64,6 @@
     "seaborn",
     "skimage",
     "svgutils",
-    "templateflow",
     "transforms3d",
 ]
 

pyproject.toml

@@ -17,7 +17,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
 ]
 license = {file = "LICENSE"}
-requires-python = ">=3.8"
+requires-python = ">=3.10"
 dependencies = [
     "importlib_resources >= 1.3; python_version < '3.9'",
     "indexed_gzip >= 0.8.8",

smriprep/cli/run.py

@@ -72,7 +72,7 @@ def get_parser():
         "output_dir",
         action="store",
         type=Path,
-        help="the output path for the outcomes of preprocessing and visual " "reports",
+        help="the output path for the outcomes of preprocessing and visual reports",
     )
     parser.add_argument(
         "analysis_level",
@@ -93,6 +93,15 @@ def get_parser():
         help="a space delimited list of participant identifiers or a single "
         "identifier (the sub- prefix can be removed)",
     )
+    g_bids.add_argument(
+        "-d",
+        "--derivatives",
+        action="store",
+        metavar="PATH",
+        type=Path,
+        nargs="*",
+        help="Search PATH(s) for pre-computed derivatives.",
+    )
     g_bids.add_argument(
         "--bids-filter-file",
         action="store",
@@ -132,7 +141,7 @@ def get_parser():
     g_perfm.add_argument(
         "--low-mem",
         action="store_true",
-        help="attempt to reduce memory usage (will increase disk usage " "in working directory)",
+        help="attempt to reduce memory usage (will increase disk usage in working directory)",
     )
     g_perfm.add_argument(
         "--use-plugin",
@@ -212,6 +221,18 @@ def get_parser():
         help="Reuse freesurfer base template"
         "(from longitudinal preprocessing)",
     )
+    g_fs.add_argument(
+        "--cifti-output",
+        nargs="?",
+        const="91k",
+        default=False,
+        choices=("91k", "170k"),
+        type=str,
+        help="Output morphometry as CIFTI dense scalars. "
+        "Optionally, the number of grayordinate can be specified "
+        "(default is 91k, which equates to 2mm resolution)",
+    )
+
     # Surface generation xor
     g_surfs = parser.add_argument_group("Surface preprocessing options")
     g_surfs.add_argument(
@@ -248,7 +269,8 @@ def get_parser():
         "--fast-track",
         action="store_true",
         default=False,
-        help="fast-track the workflow by searching for existing derivatives.",
+        help="fast-track the workflow by searching for existing derivatives. "
+        "(DEPRECATED for --derivatives).",
     )
     g_other.add_argument(
         "--resource-monitor",
@@ -280,7 +302,7 @@ def get_parser():
         "--stop-on-first-crash",
         action="store_true",
         default=False,
-        help="Force stopping on first crash, even if a work directory" " was specified.",
+        help="Force stopping on first crash, even if a work directory was specified.",
     )
     g_other.add_argument(
         "--notrack",
@@ -427,6 +449,7 @@ def build_workflow(opts, retval):
     from shutil import copyfile
     from os import cpu_count
     import uuid
+    import warnings
     from time import strftime
     from subprocess import check_call, CalledProcessError, TimeoutExpired
     from pkg_resources import resource_filename as pkgrf
@@ -517,9 +540,11 @@ def build_workflow(opts, retval):
             "logging": {"log_directory": str(log_dir), "log_to_file": True},
             "execution": {
                 "crashdump_dir": str(log_dir),
-                "crashfile_format": "txt",
+                "crashfile_format": "pklz",
                 "get_linked_libs": False,
                 "stop_on_first_crash": opts.stop_on_first_crash,
+                "poll_sleep_duration": 0.1,
+                "keep_unnecessary_outputs": True,
             },
             "monitoring": {
                 "enabled": opts.resource_monitor,
@@ -568,11 +593,20 @@ def build_workflow(opts, retval):
         ),
     )
 
+    derivatives = opts.derivatives or []
+    if opts.fast_track:
+        # XXX Makes strong assumption of legacy layout
+        smriprep_dir = str(output_dir / "smriprep")
+        warnings.warn(
+            f"Received DEPRECATED --fast-track flag. Adding {smriprep_dir} to --derivatives list."
+        )
+        derivatives.append(smriprep_dir)
+
     # Build main workflow
     retval["workflow"] = init_smriprep_wf(
         sloppy=opts.sloppy,
         debug=False,
-        fast_track=opts.fast_track,
+        derivatives=derivatives,
         freesurfer=opts.run_reconall,
         fs_subjects_dir=opts.fs_subjects_dir,
         hires=opts.hires,
@@ -591,6 +625,7 @@ def build_workflow(opts, retval):
         subject_list=subject_list,
         work_dir=str(work_dir),
         bids_filters=bids_filters,
+        cifti_output=opts.cifti_output,
     )
     retval["return_code"] = 0