Skip to content

Commit 742d327

Browse files
effigieseffigies
authored
fMRIPrep 23.2.0alpha (#3108)
This is an omnibus PR merging the `next` branch that we've been working on for the last few months. This is ready for public comment, but not merge. I will be opening PRs against the `next` branch in preparation for release. We hope to release an alpha version on Monday, October 16. The `nipreps/fmriprep:next` Docker image is available now for testing, and I will be syncing it to this branch ~nightly.
2 parents 61a7d98 + a67853f commit 742d327

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+5451
-4083
lines changed

‎.circleci/config.yml

Lines changed: 37 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -83,33 +83,35 @@ _setup_fmriprep_docker: &setup_fmriprep_docker
8383
export PY3=$(pyenv versions | grep '3\.' |
8484
sed -e 's/.* 3\./3./' -e 's/ .*//')
8585
pyenv local $PY3
86-
mkdir -p /tmp/${DATASET}/derivatives
8786
pip install --upgrade pip
8887
pip install --upgrade /tmp/src/fmriprep/wrapper/
8988
9089
_check_outputs: &check_outputs
91-
name: Checking outputs of fMRIPrep run
90+
name: Check outputs of fMRIPrep run
9291
command: |
9392
set +e -x
9493
mkdir -p /tmp/${DATASET}/test
95-
find /tmp/${DATASET}/${SUBDIR:-derivatives} \( -path '*/figures' -o -path '*/log' -o -path '*/sourcedata' \) -prune -o -name "*" -print | cut -d/ -f 5- | sort > /tmp/${DATASET}/test/outputs.out
94+
find /tmp/${DATASET}/${SUBDIR:-fmriprep} \
95+
\( -path '*/figures' -o -path '*/log' -o -path '*/sourcedata' \) \
96+
-prune -o -name "*" -print | cut -d/ -f 5- | sort > /tmp/${DATASET}/test/outputs.out
9697
9798
FASTTRACK_OUTPUTS="${DATASET}${VARIANT}_fasttrack_outputs.txt"
9899
FULL_OUTPUTS="${DATASET}${VARIANT}_outputs.txt"
99100
RET=1
100101
if [ ! -f /tmp/.nofasttrack ]; then
101102
echo "Checking fasttrack..."
102-
diff /tmp/src/fmriprep/.circleci/${FASTTRACK_OUTPUTS} /tmp/${DATASET}/test/outputs.out
103+
sort /tmp/src/fmriprep/.circleci/${FASTTRACK_OUTPUTS} > /tmp/${DATASET}/test/fasttrack.out
104+
diff /tmp/${DATASET}/test/{fasttrack,outputs}.out
103105
RET=$?
104106
fi
105107
if [ $RET -ne 0 ]; then
106108
echo "Checking full..."
107-
diff /tmp/src/fmriprep/.circleci/${FULL_OUTPUTS} /tmp/${DATASET}/test/outputs.out
109+
sort /tmp/src/fmriprep/.circleci/${FULL_OUTPUTS} > /tmp/${DATASET}/test/full.out
110+
diff /tmp/${DATASET}/test/{full,outputs}.out
108111
RET=$?
109112
fi
110113
exit $RET
111114
112-
113115
version: 2.1
114116
aliases:
115117
- &src "/tmp/src/fmriprep"
@@ -455,14 +457,13 @@ jobs:
455457
name: Run anatomical workflow on ds005
456458
no_output_timeout: 2h
457459
command: |
458-
mkdir -p /tmp/${DATASET}/work /tmp/${DATASET}/derivatives
460+
mkdir -p /tmp/${DATASET}/work /tmp/${DATASET}/fmriprep
459461
if [ -f /tmp/.nofasttrack ]; then
460462
fmriprep-docker -i nipreps/fmriprep:latest \
461463
-e FMRIPREP_DEV 1 --user $(id -u):$(id -g) \
462464
--network none \
463465
--config $PWD/nipype.cfg -w /tmp/${DATASET}/work \
464-
/tmp/data/${DATASET} /tmp/${DATASET}/derivatives participant \
465-
--output-layout legacy \
466+
/tmp/data/${DATASET} /tmp/${DATASET}/fmriprep participant \
466467
--fs-subjects-dir /tmp/${DATASET}/freesurfer \
467468
--skull-strip-template OASIS30ANTs:res-1 \
468469
--output-spaces MNI152NLin2009cAsym MNI152NLin6Asym \
@@ -474,7 +475,7 @@ jobs:
474475
command: |
475476
rm -rf /tmp/${DATASET}/work/fmriprep_*_wf/fsdir*
476477
rm -rf /tmp/${DATASET}/work/reportlets
477-
rm -rf /tmp/${DATASET}/derivatives/fmriprep
478+
rm -rf /tmp/${DATASET}/fmriprep
478479
- save_cache:
479480
key: ds005-anat-v0-{{ .Branch }}-{{ .Revision }}
480481
paths:
@@ -488,7 +489,7 @@ jobs:
488489
name: Run full fMRIPrep on ds005 (LegacyMultiProc plugin)
489490
no_output_timeout: 2h
490491
command: |
491-
FASTRACK_ARG="--anat-derivatives /tmp/${DATASET}/smriprep"
492+
FASTRACK_ARG="--derivatives anat=/tmp/${DATASET}/smriprep"
492493
if [ -f /tmp/.nofasttrack ]; then
493494
FASTRACK_ARG=""
494495
fi
@@ -502,89 +503,54 @@ jobs:
502503
-e FMRIPREP_DEV 1 --user $(id -u):$(id -g) \
503504
--network none \
504505
--config $PWD/nipype.cfg -w /tmp/${DATASET}/work \
505-
/tmp/data/${DATASET} /tmp/${DATASET}/derivatives participant \
506+
/tmp/data/${DATASET} /tmp/${DATASET}/fmriprep participant \
506507
--fs-subjects-dir /tmp/${DATASET}/freesurfer \
507508
${FASTRACK_ARG} \
508-
--output-layout legacy \
509509
--sloppy --write-graph --mem-mb 14336 \
510510
--skull-strip-template OASIS30ANTs:res-1 \
511511
--output-spaces fsaverage5 fsnative \
512512
--use-plugin /tmp/src/fmriprep/.circleci/legacy.yml \
513513
--nthreads 4 -vv
514514
- store_artifacts:
515-
path: /tmp/ds005/derivatives
515+
path: /tmp/ds005/fmriprep
516516
destination: full-run
517517
- run:
518518
name: Copy intermediate results for re-runs
519519
command: |
520-
mkdir -p /tmp/${DATASET}/derivatives_partial
520+
mkdir -p /tmp/${DATASET}/fmriprep-partial
521521
sudo cp -a /tmp/${DATASET}/work /tmp/${DATASET}/work_partial
522522
sudo cp -a /tmp/${DATASET}/work /tmp/${DATASET}/work_bids
523523
# Nipype sometimes fails to pick up when the base directory changes
524524
# This is a cheap workflow, so let's not fuss
525525
sudo rm -rf /tmp/${DATASET}/work /tmp/${DATASET}/work_bids/fmriprep_*_wf/single_subject_01_wf/anat_preproc_wf/surface_recon_wf/gifti_surface_wf || true
526-
- run:
527-
<<: *check_outputs
528-
environment:
529-
VARIANT: '_legacy'
530-
- run:
531-
name: Prepare BIDS Derivatives dataset
532-
command: |
533-
mkdir -p /tmp/${DATASET}/bids/sourcedata
534-
cp -r /tmp/${DATASET}/freesurfer /tmp/${DATASET}/bids/sourcedata/freesurfer
535-
- run:
536-
name: Re-run with BIDS layout
537-
no_output_timeout: 2h
538-
command: |
539-
FASTRACK_ARG="--anat-derivatives /tmp/${DATASET}/smriprep"
540-
if [ -f /tmp/.nofasttrack ]; then
541-
FASTRACK_ARG=""
542-
fi
543-
fmriprep-docker -i nipreps/fmriprep:latest \
544-
-e FMRIPREP_DEV 1 --user $(id -u):$(id -g) \
545-
--network none \
546-
--config $PWD/nipype.cfg -w /tmp/${DATASET}/work_bids \
547-
/tmp/data/${DATASET}/ /tmp/${DATASET}/bids participant \
548-
${FASTRACK_ARG} \
549-
--sloppy --write-graph --mem-mb 14336 \
550-
--output-spaces fsaverage5 \
551-
--nthreads 4 -vv
552-
- store_artifacts:
553-
path: /tmp/ds005/bids
554-
destination: bids-run
555-
- run:
556-
<<: *check_outputs
557-
environment:
558-
VARIANT: '_bids'
559-
SUBDIR: 'bids'
526+
- run: *check_outputs
560527
- run:
561528
name: Re-run fMRIPrep on single run of task data
562529
no_output_timeout: 2h
563530
command: |
564531
rm /tmp/data/${DATASET}/sub-01/func/*_run-01_*
565-
FASTRACK_ARG="--anat-derivatives /tmp/${DATASET}/smriprep"
532+
FASTRACK_ARG="--derivatives anat=/tmp/${DATASET}/smriprep"
566533
if [ -f /tmp/.nofasttrack ]; then
567534
FASTRACK_ARG=""
568535
fi
569536
fmriprep-docker -i nipreps/fmriprep:latest \
570537
-e FMRIPREP_DEV 1 --user $(id -u):$(id -g) \
571538
--network none \
572539
--config $PWD/nipype.cfg -w /tmp/${DATASET}/work_partial \
573-
/tmp/data/${DATASET} /tmp/${DATASET}/derivatives_partial participant \
540+
/tmp/data/${DATASET} /tmp/${DATASET}/fmriprep-partial participant \
574541
--fs-subjects-dir /tmp/${DATASET}/freesurfer \
575542
${FASTRACK_ARG} \
576-
--output-layout legacy \
577543
--sloppy --write-graph --use-syn-sdc --mem-mb 14336 \
578544
--output-spaces MNI152NLin2009cAsym fsaverage5 fsnative MNI152NLin6Asym anat \
579545
--nthreads 4 --cifti-output --project-goodvoxels -vv
580546
- store_artifacts:
581-
path: /tmp/ds005/derivatives_partial
547+
path: /tmp/ds005/fmriprep-partial
582548
destination: partial-run
583549
- run:
584550
<<: *check_outputs
585551
environment:
586-
VARIANT: '_legacy_partial'
587-
SUBDIR: 'derivatives_partial'
552+
VARIANT: '_partial'
553+
SUBDIR: 'fmriprep-partial'
588554
- run:
589555
name: Clean working directory
590556
when: on_success
@@ -643,13 +609,12 @@ jobs:
643609
name: Run anatomical workflow on ds054
644610
no_output_timeout: 2h
645611
command: |
646-
mkdir -p /tmp/${DATASET}/work /tmp/${DATASET}/derivatives
612+
mkdir -p /tmp/${DATASET}/work /tmp/${DATASET}/fmriprep
647613
if [ -f /tmp/.nofasttrack ]; then
648614
fmriprep-docker -i nipreps/fmriprep:latest \
649615
-e FMRIPREP_DEV 1 --user $(id -u):$(id -g) \
650616
--config $PWD/nipype.cfg -w /tmp/${DATASET}/work \
651-
/tmp/data/${DATASET} /tmp/${DATASET}/derivatives participant \
652-
--output-layout legacy \
617+
/tmp/data/${DATASET} /tmp/${DATASET}/fmriprep participant \
653618
--fs-no-reconall --sloppy --write-graph \
654619
--output-spaces MNI152NLin2009cAsym \
655620
--mem-mb 14336 --nthreads 4 --anat-only -vv
@@ -659,7 +624,7 @@ jobs:
659624
command: |
660625
rm -rf /tmp/${DATASET}/work/fmriprep_*_wf/fsdir*
661626
rm -rf /tmp/${DATASET}/work/reportlets
662-
rm -rf /tmp/${DATASET}/derivatives/fmriprep
627+
rm -rf /tmp/${DATASET}/fmriprep
663628
- save_cache:
664629
key: ds054-anat-v0-{{ .Branch }}-{{ .Revision }}
665630
paths:
@@ -669,16 +634,15 @@ jobs:
669634
name: Run full fMRIPrep on ds054
670635
no_output_timeout: 2h
671636
command: |
672-
FASTRACK_ARG="--anat-derivatives /tmp/${DATASET}/smriprep"
637+
FASTRACK_ARG="--derivatives anat=/tmp/${DATASET}/smriprep"
673638
if [ -f /tmp/.nofasttrack ]; then
674639
FASTRACK_ARG=""
675640
fi
676641
fmriprep-docker -i nipreps/fmriprep:latest \
677642
-e FMRIPREP_DEV 1 --user $(id -u):$(id -g) \
678643
--config $PWD/nipype.cfg -w /tmp/${DATASET}/work \
679-
/tmp/data/${DATASET} /tmp/${DATASET}/derivatives participant \
644+
/tmp/data/${DATASET} /tmp/${DATASET}/fmriprep participant \
680645
${FASTRACK_ARG} \
681-
--output-layout legacy \
682646
--fs-no-reconall --sloppy \
683647
--output-spaces MNI152NLin2009cAsym:res-2 anat func \
684648
--mem-mb 14336 --nthreads 4 -vv --debug compcor
@@ -687,18 +651,17 @@ jobs:
687651
name: Generate report with one artificial error
688652
command: |
689653
set -x
690-
sudo mv /tmp/${DATASET}/derivatives/fmriprep/sub-100185.html \
691-
/tmp/${DATASET}/derivatives/fmriprep/sub-100185_noerror.html
654+
sudo mv /tmp/${DATASET}/fmriprep/sub-100185.html \
655+
/tmp/${DATASET}/fmriprep/sub-100185_noerror.html
692656
UUID=$(grep uuid /tmp/${DATASET}/work/*/config.toml | cut -d\" -f 2 | tail -n 1)
693-
mkdir -p /tmp/${DATASET}/derivatives/fmriprep/sub-100185/log/$UUID/
657+
mkdir -p /tmp/${DATASET}/fmriprep/sub-100185/log/$UUID/
694658
cp /tmp/src/fmriprep/fmriprep/data/tests/crash_files/*.txt \
695-
/tmp/${DATASET}/derivatives/fmriprep/sub-100185/log/$UUID/
659+
/tmp/${DATASET}/fmriprep/sub-100185/log/$UUID/
696660
set +e
697661
fmriprep-docker -i nipreps/fmriprep:latest \
698662
-e FMRIPREP_DEV 1 --user $(id -u):$(id -g) \
699663
--config $PWD/nipype.cfg -w /tmp/${DATASET}/work \
700-
/tmp/data/${DATASET} /tmp/${DATASET}/derivatives participant \
701-
--output-layout legacy \
664+
/tmp/data/${DATASET} /tmp/${DATASET}/fmriprep participant \
702665
--fs-no-reconall --sloppy --write-graph \
703666
--output-spaces MNI152NLin2009cAsym:res-2 anat func \
704667
--reports-only --config-file /tmp/${DATASET}/work/${UUID}/config.toml -vv
@@ -759,13 +722,12 @@ jobs:
759722
name: Run anatomical workflow on ds000210
760723
no_output_timeout: 2h
761724
command: |
762-
mkdir -p /tmp/${DATASET}/work /tmp/${DATASET}/derivatives
725+
mkdir -p /tmp/${DATASET}/work /tmp/${DATASET}/fmriprep
763726
if [ -f /tmp/.nofasttrack ]; then
764727
fmriprep-docker -i nipreps/fmriprep:latest \
765728
-e FMRIPREP_DEV 1 --user $(id -u):$(id -g) \
766729
--config $PWD/nipype.cfg -w /tmp/${DATASET}/work \
767-
/tmp/data/${DATASET} /tmp/${DATASET}/derivatives participant \
768-
--output-layout legacy \
730+
/tmp/data/${DATASET} /tmp/${DATASET}/fmriprep participant \
769731
--fs-no-reconall --sloppy --write-graph \
770732
--output-spaces MNI152NLin2009cAsym \
771733
--mem-mb 14336 --nthreads 4 --anat-only -vv --notrack
@@ -775,7 +737,7 @@ jobs:
775737
command: |
776738
rm -rf /tmp/${DATASET}/work/fmriprep_*_wf/fsdir*
777739
rm -rf /tmp/${DATASET}/work/reportlets
778-
rm -rf /tmp/${DATASET}/derivatives/fmriprep
740+
rm -rf /tmp/${DATASET}/fmriprep
779741
- save_cache:
780742
key: ds210-anat-v1-{{ .Branch }}-{{ .Revision }}
781743
paths:
@@ -791,7 +753,7 @@ jobs:
791753
name: Run full fMRIPrep on ds000210
792754
no_output_timeout: 2h
793755
command: |
794-
FASTRACK_ARG="--anat-derivatives /tmp/${DATASET}/smriprep"
756+
FASTRACK_ARG="--derivatives anat=/tmp/${DATASET}/smriprep"
795757
if [ -f /tmp/.nofasttrack ]; then
796758
FASTRACK_ARG=""
797759
fi
@@ -805,9 +767,9 @@ jobs:
805767
fmriprep-docker -i nipreps/fmriprep:latest \
806768
-e FMRIPREP_DEV 1 --user $(id -u):$(id -g) \
807769
--config $PWD/nipype.cfg -w /tmp/${DATASET}/work \
808-
/tmp/data/${DATASET} /tmp/${DATASET}/derivatives participant \
770+
/tmp/data/${DATASET} /tmp/${DATASET}/fmriprep participant \
809771
${FASTRACK_ARG} \
810-
--output-layout legacy --me-output-echos \
772+
--me-output-echos \
811773
--fs-no-reconall --use-syn-sdc --ignore slicetiming \
812774
--dummy-scans 1 --sloppy --write-graph \
813775
--output-spaces MNI152NLin2009cAsym \

‎.circleci/ds005_bids_fasttrack_outputs.txt

Lines changed: 0 additions & 33 deletions
This file was deleted.

‎.circleci/ds005_bids_outputs.txt

Lines changed: 0 additions & 67 deletions
This file was deleted.

0 commit comments

Comments
 (0)