Merge pull request #1 from poldracklab/master

hjjh

Author: bpinsard

.circleci/config.yml

@@ -226,6 +226,7 @@ jobs:
           no_output_timeout: 2h
           command: |
             docker run -ti --rm=false \
+              -e TEST_READONLY_FILESYSTEM=1 -v $HOME:/home/readonly:ro \
               --entrypoint="pytest" poldracklab/fmriprep:latest \
               /src/fmriprep/fmriprep -svx --doctest-modules
 
@@ -260,7 +261,7 @@ jobs:
 
   build_docs:
     docker:
-        - image: python:latest
+      - image: python:3.7.4
     working_directory: /tmp/src/fmriprep
     environment:
       - FSLOUTPUTTYPE: 'NIFTI'
@@ -771,16 +772,15 @@ jobs:
             fi
 
   test_deploy_pypi:
-    machine:
-      image: circleci/classic:201711-01
+    docker:
+      - image: circleci/python:3.7.4
     working_directory: /tmp/src/fmriprep
     steps:
       - checkout
       - run:
           name: Build fMRIPrep
           command: |
-            pyenv local 3.5.2
-            pip install twine  # For use in checking distributions
+            pip install --user twine  # For use in checking distributions
             THISVERSION=$( python get_version.py )
             THISVERSION=${THISVERSION%.dirty*}
             THISVERSION=${CIRCLE_TAG:-$THISVERSION}
@@ -795,7 +795,6 @@ jobs:
       - run:
           name: Check sdist distribution
           command: |
-            pyenv local 3.5.2
             THISVERSION=$( python get_version.py )
             THISVERSION=${THISVERSION%.dirty*}
             THISVERSION=${CIRCLE_TAG:-$THISVERSION}
@@ -814,7 +813,6 @@ jobs:
       - run:
           name: Check wheel distribution
           command: |
-            pyenv local 3.5.2
             THISVERSION=$( python get_version.py )
             THISVERSION=${THISVERSION%.dirty*}
             THISVERSION=${CIRCLE_TAG:-$THISVERSION}
@@ -832,7 +830,6 @@ jobs:
       - run:
           name: Build fmriprep-docker
           command: |
-            pyenv local 3.5.2
             THISVERSION=$( python get_version.py )
             THISVERSION=${THISVERSION%.dirty*}
             cd wrapper
@@ -846,27 +843,24 @@ jobs:
           path: /tmp/src/fmriprep/wrapper/dist
 
   deploy_pypi:
-    machine:
-      image: circleci/classic:201711-01
+    docker:
+      - image: circleci/python:3.7.4
     working_directory: /tmp/src/fmriprep
     steps:
       - checkout
       - run:
           name: Build fMRIPrep
           command: |
-            pyenv local 3.5.2
             THISVERSION=$( python get_version.py )
             virtualenv --python=python build
             source build/bin/activate
-            pip install --upgrade "pip>=19.1" numpy
+            pip install --upgrade "pip>=19.1"
             echo "${CIRCLE_TAG:-$THISVERSION}" > fmriprep/VERSION
             python setup.py sdist
-            docker run --rm=false -e PLAT='manylinux1_x86_64' -e PKGNAME='fmriprep' -v `pwd`:/io \
-                quay.io/pypa/manylinux1_x86_64 /io/.circleci/pypi_wheel/build-wheels.sh
+            pip wheel --no-deps -w dist/ .
       - run:
           name: Build fmriprep-docker
           command: |
-            pyenv local 3.5.2
             THISVERSION=$( python get_version.py )
             cd wrapper
             virtualenv --python=python build
@@ -878,8 +872,7 @@ jobs:
       - run:
           name: Upload packages to PyPI
           command: |
-            pyenv local 3.5.2
-            pip install twine
+            pip install --user twine
             twine upload dist/fmriprep* wrapper/dist/fmriprep*
 
   deployable:

.circleci/ds005_outputs.txt

@@ -67,7 +67,6 @@ fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-MNI152NLin2009cAs
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-MNI152NLin2009cAsym_desc-preproc_bold.json
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz
-fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-MNI152NLin2009cAsym_desc-smoothAROMAnonaggr_bold.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-MNI152NLin6Asym_boldref.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-MNI152NLin6Asym_desc-aparcaseg_dseg.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-01_space-MNI152NLin6Asym_desc-aseg_dseg.nii.gz
@@ -100,7 +99,6 @@ fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin2009cAs
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin2009cAsym_desc-preproc_bold.json
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz
-fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin2009cAsym_desc-smoothAROMAnonaggr_bold.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin6Asym_boldref.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin6Asym_desc-aparcaseg_dseg.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin6Asym_desc-aseg_dseg.nii.gz

.circleci/ds005_partial_outputs.txt

@@ -67,7 +67,6 @@ fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin2009cAs
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin2009cAsym_desc-preproc_bold.json
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz
-fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin2009cAsym_desc-smoothAROMAnonaggr_bold.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin6Asym_boldref.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin6Asym_desc-aparcaseg_dseg.nii.gz
 fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-MNI152NLin6Asym_desc-aseg_dseg.nii.gz

.circleci/pypi_wheel/build-wheels.sh

@@ -0,0 +1,77 @@
+pullRequestOpened: |
+  Thank your for raising your pull request.
+  
+  Some of the *fMRIPRep* maintainers will review your changes as soon as time permits.
+  I'm attaching below a *Review Checklist* for the reviewers, to copy and paste 
+  it in their review comment.
+  
+  ```markdown
+  
+  ## PR Review
+  
+  *Please check off boxes as applicable, and elaborate in comments below.  Your review is not limited to these topics, as described in the reviewer guide*
+  
+  - [ ] As the reviewer I confirm that there are no conflicts of interest for me to review this work.
+  
+  Please check what applies in the following aspects of the PR:
+  
+  #### Code documentation
+  
+  - [ ] New functions and modules are documented following the guidelines.
+  - [ ] Existing code required amendments in documentation and has been updated.
+  - [ ] Sufficient inline comments ensure readability by other contributors in the long term.
+  
+  #### Documentation site
+  
+  - [ ] The modifications to *fMRIPrep* in this PR **do not require extra documentation**. This is a common situation when a bug fix that does not change the code base substantially is submitted.
+  - [ ] This PR requires documentation. The corresponding documentation will be submitted as an additional PR in the future.
+  - [ ] This PR requires extra documentation, and will be included within this PR. Please check the boxes that apply best:
+    - [ ] An existing feature is substantially modified, changing the interface and/or logic of a module.
+    - [ ] A new feature is being added, therefore full documentation of it will be included
+    - [ ] This PR addresses an error in existing documentation.
+  - [ ] Yes, all expected sections of documentation were generated by the CI build, and look as expected
+  
+  #### Tests
+  
+  - [ ] The new functionalities in this PR are covered by the existing tests
+  - [ ] New test batteries have been included with this PR
+  
+  #### Data
+  
+  - [ ] This PR does not require any additional data to be uploaded to OSF.
+  - [ ] This PR requires additional data for tests.
+  
+  #### Dependencies: smriprep
+  
+  - [ ] This PR does not require any update on `smriprep`; or
+  - [ ] `smriprep` has correctly been pinned.
+  
+  #### Dependencies: niworkflows
+  
+  - [ ] This PR does not require any update on `niworkflows`; or
+  - [ ] `niworkflows` has correctly been pinned.
+  
+  #### Dependencies: sdcflows
+  
+  - [ ] This PR does not require any update on `sdcflows`; or
+  - [ ] `sdcflows` has correctly been pinned.
+  
+  #### Dependencies: Nipype
+  
+  - [ ] This PR does not require any update on `nipype`; or
+  - [ ] `nipype` has correctly been pinned.
+  
+  #### Dependencies: other
+  
+  - [ ] This PR does not require any update on other dependencies; or
+  - [ ] other dependencies have correctly been pinned.
+  
+  #### Reports generated within CI tests
+  
+  - [ ] Yes, I have checked the reports of ds005 and they are not modified or the changes are as expected
+  - [ ] Yes, I have checked the reports of ds054 and they are not modified or the changes are as expected
+  - [ ] Yes, I have checked the reports of ds010 and they are not modified or the changes are as expected
+  
+  ### Review Comments (other than those left inline with the code)
+  
+  ```

.github/auto-comment.yml

@@ -27,6 +27,7 @@ var/
 *.egg-info/
 .installed.cfg
 *.egg
+pip-wheel-metadata/
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

.gitignore

@@ -11,7 +11,6 @@ RUN apt-get update && \
                     bzip2 \
                     ca-certificates \
                     xvfb \
-                    cython3 \
                     build-essential \
                     autoconf \
                     libtool \
@@ -109,10 +108,11 @@ RUN npm install -g [email protected]
 
 # Installing and setting up ICA_AROMA
 RUN mkdir -p /opt/ICA-AROMA && \
-  curl -sSL "https://github.com/maartenmennes/ICA-AROMA/archive/v0.4.4-beta.tar.gz" \
+  curl -sSL "https://github.com/oesteban/ICA-AROMA/archive/v0.4.5.tar.gz" \
   | tar -xzC /opt/ICA-AROMA --strip-components 1 && \
   chmod +x /opt/ICA-AROMA/ICA_AROMA.py
-ENV PATH=/opt/ICA-AROMA:$PATH
+ENV PATH="/opt/ICA-AROMA:$PATH" \
+    AROMA_VERSION="0.4.5"
 
 # Installing and setting up miniconda
 RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \

Dockerfile

@@ -1,5 +1,5 @@
-fmriprep: A Robust Preprocessing Pipeline for fMRI Data
-=======================================================
+*fMRIPrep*: A Robust Preprocessing Pipeline for fMRI Data
+=========================================================
 
 This pipeline is developed by the `Poldrack lab at Stanford University
 <https://poldracklab.stanford.edu/>`_ for use at the `Center for Reproducible
@@ -43,7 +43,7 @@ About
 .. image:: https://github.com/oesteban/fmriprep/raw/38a63e9504ab67812b63813c5fe9af882109408e/docs/_static/fmriprep-workflow-all.png
 
 
-``fmriprep`` is a functional magnetic resonance imaging (fMRI) data
+*fMRIPrep* is a functional magnetic resonance imaging (fMRI) data
 preprocessing pipeline that is designed to provide an easily accessible,
 state-of-the-art interface that is robust to variations in scan acquisition
 protocols and that requires minimal user input, while providing easily
@@ -56,12 +56,12 @@ volume-based statistics, etc.
 
 .. note::
 
-   fmriprep performs minimal preprocessing.
+   *fMRIPrep* performs minimal preprocessing.
    Here we define 'minimal preprocessing'  as motion correction, field
    unwarping, normalization, bias field correction, and brain extraction.
    See the workflows_ for more details.
 
-The ``fmriprep`` pipeline uses a combination of tools from well-known software
+The *fMRIPrep* pipeline uses a combination of tools from well-known software
 packages, including FSL_, ANTs_, FreeSurfer_ and AFNI_.
 This pipeline was designed to provide the best software implementation for each
 state of preprocessing, and will be updated as newer and better neuroimaging
@@ -86,7 +86,7 @@ https://fmriprep.readthedocs.io/
 Principles
 ----------
 
-``fmriprep`` is built around three principles:
+*fMRIPrep* is built around three principles:
 
 1. **Robustness** - The pipeline adapts the preprocessing steps depending on
    the input dataset and should provide results as good as possible
@@ -97,22 +97,22 @@ Principles
    automatic fashion.
 3. **"Glass box"** philosophy - Automation should not mean that one should not
    visually inspect the results or understand the methods.
-   Thus, ``fmriprep`` provides visual reports for each subject, detailing the
+   Thus, *fMRIPrep* provides visual reports for each subject, detailing the
    accuracy of the most important processing steps.
    This, combined with the documentation, can help researchers to understand
    the process and decide which subjects should be kept for the group level
    analysis.
 
 
-Limitations and reasons not to use ``fmriprep``
------------------------------------------------
+Limitations and reasons not to use *fMRIPrep*
+---------------------------------------------
 
 1. Very narrow :abbr:`FoV (field-of-view)` images oftentimes do not contain
    enough information for standard image registration methods to work correctly.
    Also, problems may arise when extracting the brain from these data.
    Supporting these particular images is already a future line of the development
    road-map.
-2. ``fmriprep`` may also underperform for particular populations (e.g., infants) and
+2. *fMRIPrep* may also underperform for particular populations (e.g., infants) and
    non-human brains, although appropriate templates can be provided to overcome the
    issue.
 3. The "EPInorm" approach is currently not supported, although we plan to implement
@@ -131,6 +131,6 @@ Acknowledgements
 ----------------
 
 Please acknowledge this work by mentioning explicitly the name of this software
-(fmriprep) and the version, along with a link to the `GitHub repository
+(*fMRIPrep*) and the version, along with a link to the `GitHub repository
 <https://github.com/poldracklab/fmriprep>`_ or the Zenodo reference.
 For more details, please see :ref:`citation`.

README.rst

@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+#SBATCH -J fmriprep
+#SBATCH --time=48:00:00
+#SBATCH -n 1
+#SBATCH --cpus-per-task=16
+#SBATCH --mem-per-cpu=4G
+#SBATCH -p normal,mygroup  # Queue names you can submit to
+# Outputs ----------------------------------
+#SBATCH -o log/%x-%A-%a.out
+#SBATCH -e log/%x-%A-%a.err
+#SBATCH --mail-user=%[email protected]
+#SBATCH --mail-type=ALL
+# ------------------------------------------
+
+BIDS_DIR="$STUDY/data"
+DERIVS_DIR="derivatives/fmriprep-1.5.0"
+
+# Prepare some writeable bind-mount points.
+TEMPLATEFLOW_HOST_HOME=$HOME/.cache/templateflow
+FMRIPREP_HOST_CACHE=$HOME/.cache/fmriprep
+mkdir -p ${TEMPLATEFLOW_HOST_HOME}
+mkdir -p ${FMRIPREP_HOST_CACHE}
+
+# Prepare derivatives folder
+mkdir -p ${BIDS_DIR}/${DERIVS_DIR}
+
+# This trick will help you reuse freesurfer results across pipelines and fMRIPrep versions
+mkdir -p ${BIDS_DIR}/derivatives/freesurfer-6.0.1
+if [ ! -d ${BIDS_DIR}/${DERIVS_DIR}/freesurfer ]; then
+    ln -s ${BIDS_DIR}/derivatives/freesurfer-6.0.1 ${BIDS_DIR}/${DERIVS_DIR}/freesurfer
+fi
+
+# Make sure FS_LICENSE is defined in the container.
+export SINGULARITYENV_FS_LICENSE=$HOME/.freesurfer.txt
+
+# Designate a templateflow bind-mount point
+export SINGULARITYENV_TEMPLATEFLOW_HOME="/templateflow"
+SINGULARITY_CMD="singularity run --cleanenv -B $BIDS_DIR:/data -B ${TEMPLATEFLOW_HOST_HOME}:${SINGULARITYENV_TEMPLATEFLOW_HOME} -B $L_SCRATCH:/work $STUDY/images/poldracklab_fmriprep_1.5.0.simg"
+
+# Parse the participants.tsv file and extract one subject ID from the line corresponding to this SLURM task.
+subject=$( sed -n -E "$((${SLURM_ARRAY_TASK_ID} + 1))s/sub-(\S*)\>.*/\1/gp" ${BIDS_DIR}/participants.tsv )
+
+# Remove IsRunning files from FreeSurfer
+find ${BIDS_DIR}/derivatives/freesurfer-6.0.1/sub-$subject/ -name "*IsRunning*" -type f -delete
+
+# Compose the command line
+cmd="${SINGULARITY_CMD} /data /data/${DERIVS_DIR} participant --participant-label $subject -w /work/ -vv --omp-nthreads 8 --nthreads 12 --mem_mb 30000 --output-spaces MNI152NLin2009cAsym:res-2 anat fsnative fsaverage5 --use-aroma"
+
+# Setup done, run the command
+echo Running task ${SLURM_ARRAY_TASK_ID}
+echo Commandline: $cmd
+eval $cmd
+exitcode=$?
+
+# Output results to a table
+echo "sub-$subject   ${SLURM_ARRAY_TASK_ID}    $exitcode" \
+      >> ${SLURM_JOB_NAME}.${SLURM_ARRAY_JOB_ID}.tsv
+echo Finished tasks ${SLURM_ARRAY_TASK_ID} with exit code $exitcode
+exit $exitcode