Merge pull request #697 from effigies/ci/test_python_versions

CI: Install package across Python versions and run pytest

Author: effigies

.github/workflows/pythonpackage.yml

@@ -14,85 +14,148 @@ jobs:
   build:
     if: "!contains(github.event.head_commit.message, '[skip ci]')"
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Display Python version
+      run: python -c "import sys; print(sys.version)"
+    - name: Check python version and install build
+      run: |
+        python --version
+        python -m pip install -U build twine
+    - name: Build niworkflows
+      run: python -m build
+    - name: Check distributions
+      run: twine check dist/*
+    - uses: actions/upload-artifact@v3
+      with:
+        name: dist
+        path: dist/
+
+  get_data:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Create test data directory
+      run: mkdir -p $HOME/.cache/stanford-crn
+    - name: Load test data cache
+      uses: actions/cache@v2
+      id: stanford-crn
+      with:
+        path: ~/.cache/stanford-crn/
+        key: data-v0-${{ github.ref_name }}-${{ github.sha }}
+        restore-keys: |
+          data-v0-${{ github.ref_name }}-
+          data-v0-
+    - name: Install datalad
+      run: |
+        sudo apt-get update -y
+        sudo apt-get install -y git-annex --no-install-recommends
+        python -m pip install datalad==0.14.7 datalad-osf
+        datalad wtf
+    - name: Fetch test data
+      run: |
+        DS=$HOME/.cache/stanford-crn
+        datalad install -r -s https://github.com/nipreps-data/niworkflows-data.git $DS
+        cd $DS
+        git -C BIDS-examples-1-enh-ds054 checkout enh/ds054
+        datalad update -r --merge -d .
+        datalad get -J 2 -r ds000003 ds000030/sub-10228/func
+
+  test:
+    needs: [build, get_data]
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: [3.7, 3.8, 3.9]
+        install: [repo]
+        include:
+          - python-version: 3.9
+            install: sdist
+          - python-version: 3.9
+            install: wheel
+          - python-version: 3.9
+            install: editable
+    env:
+      INSTALL_TYPE: ${{ matrix.install }}
 
     steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - uses: actions/checkout@v2
+    - name: Load test data cache
+      uses: actions/cache@v2
+      id: stanford-crn
       with:
-        ssh-key: "${{ secrets.NIPREPS_DEPLOY }}"
-        fetch-depth: 0
-    - name: Build in confined, updated environment and interpolate version
+        path: ~/.cache/stanford-crn/
+        key: data-v0-${{ github.ref_name }}-${{ github.sha }}
+    - name: Load TemplateFlow cache
+      uses: actions/cache@v2
+      id: templateflow
+      with:
+        path: ~/.cache/templateflow
+        key: templateflow-v0-${{ github.ref_name }}-${{ strategy.job-index }}-${{ github.sha }}
+        restore-keys: |
+          templateflow-v0-${{ github.ref_name }}-
+          templateflow-v0-
+    - name: Fetch packages
+      uses: actions/download-artifact@v3
+      with:
+        name: dist
+        path: dist/
+    - name: Select archive
+      run: |
+        if [ "$INSTALL_TYPE" = "sdist" ]; then
+          ARCHIVE=$( ls dist/*.tar.gz )
+        elif [ "$INSTALL_TYPE" = "wheel" ]; then
+          ARCHIVE=$( ls dist/*.whl )
+        elif [ "$INSTALL_TYPE" = "repo" ]; then
+          ARCHIVE="."
+        elif [ "$INSTALL_TYPE" = "editable" ]; then
+          ARCHIVE="-e ."
+        fi
+        echo "ARCHIVE=$ARCHIVE" >> $GITHUB_ENV
+    - name: Install package
+      run: python -m pip install $ARCHIVE
+    - name: Check version
       run: |
-        python -m venv /tmp/buildenv
-        source /tmp/buildenv/bin/activate
-        python -m pip install -U setuptools pip wheel twine docutils
-        python setup.py sdist bdist_wheel
-        python -m twine check dist/niworkflows*
-
         # Interpolate version
         if [[ "$GITHUB_REF" == refs/tags/* ]]; then
           TAG=${GITHUB_REF##*/}
         fi
         THISVERSION=$( python get_version.py )
         THISVERSION=${TAG:-$THISVERSION}
-        echo "Expected VERSION: \"${THISVERSION}\""
-        echo "THISVERSION=${THISVERSION}" >> ${GITHUB_ENV}
-
-    - name: Install in confined environment [sdist]
-      run: |
-        python -m venv /tmp/install_sdist
-        source /tmp/install_sdist/bin/activate
-        python -m pip install --upgrade pip wheel
-        python -m pip install dist/niworkflows*.tar.gz
         INSTALLED_VERSION=$(python -c 'import niworkflows; print(niworkflows.__version__, end="")')
         echo "VERSION: \"${THISVERSION}\""
         echo "INSTALLED: \"${INSTALLED_VERSION}\""
         test "${INSTALLED_VERSION}" = "${THISVERSION}"
-
-    - name: Install in confined environment [wheel]
-      run: |
-        python -m venv /tmp/install_wheel
-        source /tmp/install_wheel/bin/activate
-        python -m pip install --upgrade pip wheel
-        python -m pip install dist/niworkflows*.whl
-        INSTALLED_VERSION=$(python -c 'import niworkflows; print(niworkflows.__version__, end="")')
-        echo "INSTALLED: \"${INSTALLED_VERSION}\""
-        test "${INSTALLED_VERSION}" = "${THISVERSION}"
-
-    - name: Install in confined environment [pip install .]
-      run: |
-        python -m venv /tmp/setup_install
-        source /tmp/setup_install/bin/activate
-        python -m pip install --upgrade pip wheel
-        python -m pip install .
-        INSTALLED_VERSION=$(python -c 'import niworkflows; print(niworkflows.__version__, end="")')
-        echo "INSTALLED: \"${INSTALLED_VERSION}\""
-        test "${INSTALLED_VERSION}" = "${THISVERSION}"
-
-    - name: Install in confined environment [pip install -e .]
-      run: |
-        python -m venv /tmp/setup_develop
-        source /tmp/setup_develop/bin/activate
-        python -m pip install pip
-        python -m pip install --upgrade pip wheel
-        python -m pip install -e .
-        INSTALLED_VERSION=$(python -c 'import niworkflows; print(niworkflows.__version__, end="")')
-        echo "INSTALLED: \"${INSTALLED_VERSION}\""
-        test "${INSTALLED_VERSION}" = "${THISVERSION}"
+    - name: Install test dependencies
+      run: python -m pip install "niworkflows[tests]"
+    - name: Run tests
+      uses: GabrielBB/xvfb-action@v1
+      with:
+        run: pytest -sv --no-xvfb --doctest-modules --cov niworkflows niworkflows
+    - uses: codecov/codecov-action@v2
+      name: Submit to CodeCov
 
   flake8:
     if: "!contains(github.event.head_commit.message, '[skip ci]')"
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python 3.7
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v3
       with:
         python-version: 3.7
     - run: pip install flake8

docs/conf.py

@@ -225,7 +225,7 @@
 
 apidoc_module_dir = "../niworkflows"
 apidoc_output_dir = "api"
-apidoc_excluded_paths = ["conftest.py", "*/tests/*", "tests/*", "data/*"]
+apidoc_excluded_paths = ["conftest.py", "*/tests/*", "tests/*", "data/*", "testing.py"]
 apidoc_separate_modules = True
 apidoc_extra_args = ["--module-first", "-d 1", "-T"]
 

niworkflows/conftest.py

@@ -32,14 +32,10 @@
 import pkg_resources
 
 from .utils.bids import collect_data
-
-test_data_env = os.getenv(
-    "TEST_DATA_HOME", str(Path.home() / ".cache" / "stanford-crn")
+from .testing import (
+    test_data_env, test_output_dir, test_workdir, data_dir,
+    data_env_canary, data_dir_canary
 )
-test_output_dir = os.getenv("TEST_OUTPUT_DIR")
-test_workdir = os.getenv("TEST_WORK_DIR")
-
-data_dir = Path(test_data_env) / "BIDS-examples-1-enh-ds054"
 
 
 @pytest.fixture(autouse=True)
@@ -52,6 +48,7 @@ def add_np(doctest_namespace):
     doctest_namespace["pytest"] = pytest
     doctest_namespace["Path"] = Path
     doctest_namespace["datadir"] = data_dir
+    doctest_namespace["data_dir_canary"] = data_dir_canary
     doctest_namespace["bids_collect_data"] = collect_data
     doctest_namespace["test_data"] = pkg_resources.resource_filename(
         "niworkflows", "tests/data"
@@ -82,6 +79,7 @@ def testdata_dir():
 
 @pytest.fixture
 def ds000030_dir():
+    data_env_canary()
     return Path(test_data_env) / "ds000030"
 
 

niworkflows/interfaces/bids.py

@@ -105,6 +105,10 @@ class BIDSInfo(SimpleInterface):
     This interface uses only the basename, not the path, to determine the
     subject, session, task, run, acquisition or reconstruction.
 
+    .. testsetup::
+
+        >>> data_dir_canary()
+
     >>> bids_info = BIDSInfo(bids_dir=str(datadir / 'ds054'), bids_validate=False)
     >>> bids_info.inputs.in_file = '''\
 sub-01/func/ses-retest/sub-01_ses-retest_task-covertverbgeneration_bold.nii.gz'''
@@ -224,6 +228,10 @@ class BIDSDataGrabber(SimpleInterface):
     """
     Collect files from a BIDS directory structure.
 
+    .. testsetup::
+
+        >>> data_dir_canary()
+
     >>> bids_src = BIDSDataGrabber(anat_only=False)
     >>> bids_src.inputs.subject_data = bids_collect_data(
     ...     str(datadir / 'ds114'), '01', bids_validate=False)[0]
@@ -320,6 +328,10 @@ class DerivativesDataSink(SimpleInterface):
     Saves the ``in_file`` into a BIDS-Derivatives folder provided
     by ``base_directory``, given the input reference ``source_file``.
 
+    .. testsetup::
+
+        >>> data_dir_canary()
+
     >>> import tempfile
     >>> tmpdir = Path(tempfile.mkdtemp())
     >>> tmpfile = tmpdir / 'a_temp_file.nii.gz'
@@ -732,6 +744,10 @@ class ReadSidecarJSON(SimpleInterface):
     """
     Read JSON sidecar files of a BIDS tree.
 
+    .. testsetup::
+
+        >>> data_dir_canary()
+
     >>> fmap = str(datadir / 'ds054' / 'sub-100185' / 'fmap' /
     ...            'sub-100185_phasediff.nii.gz')
 

niworkflows/interfaces/freesurfer.py

@@ -48,6 +48,10 @@ class StructuralReference(fs.RobustTemplate):
     """
     Shortcut RobustTemplate with a copy of the source if a single volume is provided.
 
+    .. testsetup::
+
+        >>> data_dir_canary()
+
     Examples
     --------
     >>> t1w = bids_collect_data(

niworkflows/interfaces/tests/test_bids.py

@@ -32,6 +32,7 @@
 from nipype.interfaces.base import Undefined
 
 from .. import bids as bintfs
+from niworkflows.testing import needs_data_dir
 
 
 XFORM_CODES = {
@@ -608,6 +609,7 @@ def test_DerivativesDataSink_values(tmp_path, dtype):
     assert sha1(out_file.read_bytes()).hexdigest() == checksum
 
 
+@needs_data_dir
 @pytest.mark.parametrize("field", ["RepetitionTime", "UndefinedField"])
 def test_ReadSidecarJSON_connection(testdata_dir, field):
     """

niworkflows/interfaces/tests/test_images.py

@@ -29,17 +29,13 @@
 import pytest
 
 from .. import images as im
+from niworkflows.testing import has_afni
 
 
 @pytest.mark.parametrize(
     "nvols, nmasks, ext, factor",
     [
-        (500, 10, ".nii", 2),
-        (500, 10, ".nii.gz", 5),
         (200, 3, ".nii", 1.1),
-        (200, 3, ".nii.gz", 2),
-        (200, 10, ".nii", 1.1),
-        (200, 10, ".nii.gz", 2),
     ],
 )
 def test_signal_extraction_equivalence(tmp_path, nvols, nmasks, ext, factor):
@@ -158,6 +154,7 @@ def test_conform_set_zooms(tmpdir):
     assert np.allclose(out_img.header.get_zooms(), conform.inputs.target_zooms)
 
 
+@pytest.mark.skipif(not has_afni, reason="Needs AFNI")
 @pytest.mark.parametrize("shape", [
     (10, 10, 10),
     (10, 10, 10, 1),

niworkflows/testing.py

@@ -0,0 +1,43 @@
+import pytest
+from functools import wraps
+import os
+from pathlib import Path
+from nipype.interfaces import fsl, freesurfer as fs, afni
+
+test_data_env = os.getenv(
+    "TEST_DATA_HOME", str(Path.home() / ".cache" / "stanford-crn")
+)
+test_output_dir = os.getenv("TEST_OUTPUT_DIR")
+test_workdir = os.getenv("TEST_WORK_DIR")
+
+data_dir = Path(test_data_env) / "BIDS-examples-1-enh-ds054"
+
+
+def create_canary(predicate, message):
+    def canary():
+        if predicate:
+            pytest.skip(message)
+
+    def decorator(f):
+        @wraps(f)
+        def wrapper(*args, **kwargs):
+            canary()
+            return f(*args, **kwargs)
+        return wrapper
+
+    return canary, decorator
+
+
+data_env_canary, needs_data_env = create_canary(
+    not os.path.isdir(test_data_env),
+    "Test data must be made available in ~/.cache/stanford-crn or in a "
+    "directory referenced by the TEST_DATA_HOME environment variable.")
+
+data_dir_canary, needs_data_dir = create_canary(
+    not os.path.isdir(data_dir),
+    "Test data must be made available in ~/.cache/stanford-crn or in a "
+    "directory referenced by the TEST_DATA_HOME environment variable.")
+
+has_fsl = fsl.Info.version() is not None
+has_freesurfer = fs.Info.version() is not None
+has_afni = afni.Info.version() is not None