[BACKPORT] CI: Use datalad-managed test data (#653) [test masks]

* MAINT: Move mask-regressions test-data to datalad + nipreps-data

* fix: set TEMPLATEFLOW_AUTOUPDATE=off

* fix: revise templateflow installation and set TEMPLATEFLOW_AUTOUPDATE=off

* fix: test preamble, failing to discover data

* fix: trigger run after reseting github's default branches

* enh: precache all necessary templateflow resources

* enh: various improvements to Docker image and Circle settings

* maint: roll back config and sty [skip ci]

* MAINT: Migrate ``ds003_downsampled`` to ``nipreps-data``

Finalizes the migration from zipfiles to datalad datasets.

* maint: [test masks] now triggers regression tests too

* fix: existing datalad datasets

Co-authored-by: Oscar Esteban <[email protected]>

Author: effigies

.circleci/config.yml

@@ -95,7 +95,8 @@ jobs:
               docker build --rm --cache-from=niworkflows:latest \
                 -t niworkflows:latest \
                 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \
-                --build-arg VCS_REF=`git rev-parse --short HEAD` . \
+                --build-arg VCS_REF=`git rev-parse --short HEAD` \
+                --build-arg VERSION=`python3 get_version.py` . \
               && e=0 && break || sleep 15
             done && [ "$e" -eq "0" ]
             docker tag niworkflows:latest localhost:5000/niworkflows
@@ -112,26 +113,45 @@ jobs:
             - /tmp/images
 
   get_data:
-    machine:
-      # Ubuntu 14.04 with Docker 17.10.0-ce
-      image: ubuntu-1604:202007-01
-    working_directory: /home/circleci/data
+    docker:
+      - image: continuumio/miniconda3:4.9.2-alpine
+    working_directory: /tmp/data
+    environment:
+      - TEMPLATEFLOW_HOME: /tmp/templateflow
     steps:
       - restore_cache:
           keys:
-            - data-v1-{{ epoch }}
-            - data-v1-
+            - env-v1-{{ .Branch }}-
+            - env-v1-master-
+            - env-v1-
+      - run:
+          name: Setup git-annex, DataLad & TemplateFlow
+          command: |
+            conda install -y -c anaconda -c conda-forge git-annex datalad wget
+            python -m pip install --no-cache-dir -U datalad-osf
+            git config --global user.name 'NiPreps Bot'
+            git config --global user.email '[email protected]'
+            python -m pip install --no-cache-dir -U templateflow
+      - save_cache:
+          key: env-v1-{{ .Branch }}-{{ .BuildNum }}
+          paths:
+            - /opt/conda
+
+      - restore_cache:
+          keys:
+            - data-v3-{{ .Branch }}-{{ epoch }}
+            - data-v3-{{ .Branch }}-
+            - data-v3-master-
+            - data-v3-
       - run:
           name: Get test data from ds000003
           command: |
-            mkdir -p /tmp/data
-            if [[ ! -d /tmp/data/ds003_downsampled ]]; then
-              wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
-                -O ds003_downsampled.tar.gz "https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/57f328f6b83f6901ef94cf70"
-              tar xvzf ds003_downsampled.tar.gz -C /tmp/data/
-            else
-              echo "Dataset ds000003 was cached"
+            if [[ ! -d ds000003 ]]; then
+              datalad install -r https://github.com/nipreps-data/ds000003.git
             fi
+            datalad update -r --merge -d ds000003/
+            datalad get -J 2 -r -d ds000003/ ds000003/*
+
       - run:
           name: Get BIDS test data stub
           command: |
@@ -143,57 +163,67 @@ jobs:
             else
               echo "BIDS stub was cached"
             fi
+      - save_cache:
+         key: data-v3-{{ .Branch }}-{{ epoch }}
+         paths:
+            - /tmp/data/ds000003
+            - /tmp/data/BIDS-examples-1-enh-ds054
       - run:
           name: Store FreeSurfer license file
           command: |
             mkdir -p /tmp/fslicense
             cd /tmp/fslicense
             echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IGxpY2Vuc2UudHh0Cg==" | base64 -d | sh
+
+      - checkout:
+          path: /tmp/src/niworkflows
+
       - persist_to_workspace:
           root: /tmp
           paths:
-            - data
             - fslicense
-      - save_cache:
-         key: data-v1-{{ epoch }}
-         paths:
-            - /tmp/data
+            - src/
+
+      - run:
+          name: Check if regression data must be downloaded
+          command: |
+            cd /tmp/src/niworkflows
+            set +e
+            test_masks="$( git log --format=oneline -n 1 $CIRCLE_SHA1 | grep -i -E '\[test[ _]?masks?\]' )$( echo $CIRCLE_BRANCH | grep -i -E '^masks?/' )"
+            set -e
+            if [[ "x${CIRCLE_TAG}" = "x" && "${CIRCLE_BRANCH}" != "master" && "x${test_masks}" = "x" ]]; then
+              echo "Not a tag or master branch, not a ``masks?/`` branch, no ``[test_masks]`` label in commit message"
+              echo "Skipping download of test data for mask regressions."
+              circleci step halt
+            fi
 
-  get_regression_data:
-    machine:
-      # Ubuntu 14.04 with Docker 17.10.0-ce
-      image: ubuntu-1604:202007-01
-    working_directory: /home/circleci/data
-    steps:
       - restore_cache:
           keys:
-            - regression-v5-{{ .Revision }}
-            - regression-v5-
+            - mask-regressions-v0-{{ .Branch }}-{{ .Revision }}-
+            - mask-regressions-v0-{{ .Branch }}-
+            - mask-regressions-v0-master-
+            - mask-regressions-v0-
+            - mask-regressions-
       - run:
-          name: Get truncated BOLD series
+          name: Setup TemplateFlow
           command: |
-            mkdir -p /tmp/data
-            if [[ ! -d /tmp/data/fmriprep_bold_truncated ]]; then
-              wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
-                -O fmriprep_bold_truncated.tar.gz "https://osf.io/286yr/download"
-              tar xvzf fmriprep_bold_truncated.tar.gz -C /tmp/data/
-            else
-              echo "Truncated BOLD series were cached"
-            fi
+            python -c "from templateflow import api as tfapi; \
+                       tfapi.get('MNI152NLin2009cAsym', resolution=1, label='brain', suffix='probseg'); \
+                       tfapi.get('MNI152NLin2009cAsym', resolution=2, desc='fMRIPrep', suffix='boldref'); \
+                       tfapi.get('MNI152NLin2009cAsym', resolution=2, desc='brain', suffix='mask'); "
       - run:
-          name: Get pre-computed masks
+          name: Install epi-references
           command: |
-            if [[ ! -d /tmp/data/fmriprep_bold_mask ]]; then
-              wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
-                -O fmriprep_bold_mask.tar.gz "https://osf.io/s4f7b/download"
-              tar xvzf fmriprep_bold_mask.tar.gz -C /tmp/data/
-            else
-              echo "Pre-computed masks were cached"
+            if [[ ! -d epi-references ]]; then
+              datalad install -r https://github.com/nipreps-data/epi-references.git
             fi
+            datalad update -r --merge -d epi-references/
+            datalad get -J 2 -r -d epi-references/ epi-references/*
       - save_cache:
-         key: regression-v5-{{ .Revision }}-{{ epoch }}
-         paths:
-            - /tmp/data
+          key: mask-regressions-v0-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
+          paths:
+            - /tmp/templateflow
+            - /tmp/data/epi-references
 
   test_pytest:
     machine:
@@ -202,8 +232,6 @@ jobs:
     steps:
       - attach_workspace:
           at: /tmp
-      - checkout:
-          path: /tmp/src/niworkflows
       - run:
           name: Get codecov
           command: python -m pip install codecov
@@ -232,10 +260,18 @@ jobs:
             docker run -d -p 5000:5000 --restart=always --name=registry \
                 -v /tmp/docker:/var/lib/registry registry:2
       - run:
-          name: Pull images to registry
+          name: Pull images from registry
           command: |
             docker pull localhost:5000/niworkflows
             docker tag localhost:5000/niworkflows niworkflows:latest
+
+      - restore_cache:
+          keys:
+            - data-v3-{{ .Branch }}-{{ epoch }}
+            - data-v3-{{ .Branch }}-
+            - data-v3-master-
+            - data-v3-
+
       - run:
           name: Run unit tests
           no_output_timeout: 2h
@@ -300,6 +336,25 @@ jobs:
     steps:
       - attach_workspace:
           at: /tmp
+
+      - run:
+          name: Determine whether regression tests (masks) must be executed
+          command: |
+            cd /tmp/src/niworkflows
+            set +e
+            test_masks="$( git log --format=oneline -n 1 $CIRCLE_SHA1 | grep -i -E '\[test[ _]?masks?\]' )$( echo $CIRCLE_BRANCH | grep -i -E '^masks?/' )"
+            set -e
+            if [[ "x${CIRCLE_TAG}" = "x" && "${CIRCLE_BRANCH}" != "master" && "x${test_masks}" = "x" ]]; then
+              echo "Not a tag or master branch, not a ``masks?/`` branch, no ``[test_masks]`` label in commit message"
+              echo "Skipping download of test data for mask regressions."
+              circleci step halt
+            fi
+
+      - restore_cache:
+          keys:
+            - mask-regressions-v0-{{ .Branch }}-{{ .Revision }}-
+            - mask-regressions-v0-{{ .Branch }}-
+
       - restore_cache:
           keys:
             - build-v1-{{ .Branch }}-{{ epoch }}
@@ -331,10 +386,6 @@ jobs:
           command: |
             docker pull localhost:5000/niworkflows
             docker tag localhost:5000/niworkflows niworkflows:latest
-
-      - restore_cache:
-          keys:
-            - regression-v5-{{ .Revision }}
       - restore_cache:
           keys:
             - masks-workdir-v3-{{ .Branch }}-{{epoch}}
@@ -347,10 +398,11 @@ jobs:
           command: |
             mkdir -p /tmp/masks/{reports,workdir} && \
             docker run -ti -u $(id -u) -w /src/niworkflows \
+              -v /tmp/templateflow:/templateflow \
+              -v /tmp/data/epi-references:/data -v /tmp/masks/reports:/tmp/masks/reports \
               -e COVERAGE_FILE=/tmp/masks/reports/.coverage \
-              -v /tmp/data:/tmp/data -v /tmp/masks/reports:/tmp/masks/reports \
-              -e FMRIPREP_REGRESSION_SOURCE=/tmp/data/fmriprep_bold_truncated \
-              -e FMRIPREP_REGRESSION_TARGETS=/tmp/data/fmriprep_bold_mask \
+              -e TEMPLATEFLOW_HOME=/templateflow -e TEMPLATEFLOW_AUTOUPDATE=off \
+              -e FMRIPREP_REGRESSION_SOURCE=/data \
               -e FMRIPREP_REGRESSION_REPORTS=/tmp/masks/reports \
               -e CACHED_WORK_DIRECTORY=/tmp/work -v /tmp/masks/workdir:/tmp/work \
               niworkflows:latest \
@@ -385,8 +437,6 @@ jobs:
               -e COVERAGE_FILE=/tmp/masks/reports/.coverage \
               -v /tmp/masks/reports:/tmp/masks/reports \
               niworkflows:latest coverage xml -o coverage.xml
-      - checkout:
-          path: /tmp/src/niworkflows
       - run:
           name: Get codecov
           command: python -m pip install codecov
@@ -587,21 +637,9 @@ workflows:
               only: /.*/
       - get_data:
           filters:
-            branches:
-              ignore:
-                - /masks?\/.*/
             tags:
               only: /.*/
 
-      - get_regression_data:
-          filters:
-            branches:
-              only:
-                - /master/
-                - /masks?\/.*/
-            tags:
-              ignore: /.*/
-
       - test_package:
           filters:
             branches:
@@ -625,12 +663,8 @@ workflows:
       - test_masks:
           requires:
             - build
-            - get_regression_data
+            - get_data
           filters:
-            branches:
-              only:
-                - /master/
-                - /masks?\/.*/
             tags:
               ignore: /.*/
 

.dockerignore

@@ -9,6 +9,7 @@ __pycache__
 build/
 dist/
 docs/
+examples/
 niworkflows.egg-info/
 pip-wheel-metadata/
 src/
@@ -36,5 +37,6 @@ circle.yml
 codecov.yml
 
 # other
+Makefile
 .zenodo.json
 

Dockerfile

@@ -146,22 +146,12 @@ ENV MKL_NUM_THREADS=1 \
 RUN python -c "from matplotlib import font_manager" && \
     sed -i 's/\(backend *: \).*$/\1Agg/g' $( python -c "import matplotlib; print(matplotlib.matplotlib_fname())" )
 
-# Installing dev requirements (packages that are not in pypi)
-# Precaching atlases
-RUN pip install --no-cache-dir "templateflow >= 0.6" && \
-    rm -rf $HOME/.cache/pip && \
-    python -c "from templateflow import api as tfapi; \
-               tfapi.get(['MNI152Lin', 'MNI152NLin2009cAsym', 'OASIS30ANTs'], suffix='T1w'); \
-               tfapi.get(['MNI152Lin', 'MNI152NLin2009cAsym', 'OASIS30ANTs'], desc='brain', suffix='mask'); \
-               tfapi.get(['MNI152NLin2009cAsym'], desc='fMRIPrep', suffix='boldref'); \
-               tfapi.get('OASIS30ANTs', resolution=1, desc='4', suffix='dseg'); \
-               tfapi.get(['OASIS30ANTs', 'NKI'], resolution=1, label='brain', suffix='probseg'); \
-               tfapi.get(['OASIS30ANTs', 'NKI'], resolution=1, desc='BrainCerebellumRegistration', suffix='mask'); "
-
-WORKDIR /src/
-COPY . niworkflows/
 WORKDIR /src/niworkflows/
-RUN pip install --no-cache-dir -e .[all] && \
+COPY . /src/niworkflows/
+ARG VERSION
+RUN echo "${VERSION}" > /src/niworkflows/niworkflows/VERSION && \
+    echo "include niworkflows/VERSION" >> /src/niworkflows/MANIFEST.in && \
+    pip install --no-cache-dir -e .[all] && \
     rm -rf $HOME/.cache/pip
 
 COPY docker/files/nipype.cfg /home/niworkflows/.nipype/nipype.cfg

Makefile

@@ -0,0 +1,17 @@
+.PHONY: help docker
+.DEFAULT: help
+
+tag="latest"
+
+help:
+	@echo "Premade recipes"
+	@echo
+	@echo "make docker [tag=TAG]"
+	@echo "\tBuilds a docker image from source. Defaults to 'latest' tag."
+
+
+docker:
+	docker build --rm -t nipreps/niworkflows:$(tag) \
+	--build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \
+	--build-arg VCS_REF=`git rev-parse --short HEAD` \
+	--build-arg VERSION=`python setup.py --version` .