From d27181fed6bd20e943ad03d1bb143cddb8f74514 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 30 Mar 2020 13:59:22 -0400 Subject: [PATCH 01/14] MNT: Add update_requirements tool to copy from setup.cfg --- tools/update_requirements.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100755 tools/update_requirements.py diff --git a/tools/update_requirements.py b/tools/update_requirements.py new file mode 100755 index 0000000000..32eab85ec5 --- /dev/null +++ b/tools/update_requirements.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +import sys +from copy import copy +from configparser import ConfigParser +from pathlib import Path +from packaging.requirements import Requirement, SpecifierSet + +repo_root = Path(__file__).parent.parent +setup_cfg = repo_root / "setup.cfg" +reqs = repo_root / "requirements.txt" +min_reqs = repo_root / "min-requirements.txt" + +config = ConfigParser() +config.read(setup_cfg) +requirements = [Requirement(req) + for req in config.get("options", "install_requires").strip().splitlines()] + +script_name = Path(__file__).relative_to(repo_root) + +def to_min(req): + if req.specifier: + req = copy(req) + min_spec = [spec for spec in req.specifier if spec.operator in ('>=', '~=')][0] + min_spec._spec = ('==', ) + min_spec._spec[1:] + req.specifier = SpecifierSet(str(min_spec)) + return req + +lines = [f"# Auto-generated by {script_name}", ""] + +# Write requirements +lines[1:-1] = [str(req) for req in requirements] +reqs.write_text("\n".join(lines)) + +# Write minimum requirements +lines[1:-1] = [str(to_min(req)) for req in requirements] +min_reqs.write_text("\n".join(lines)) From e60df9a7e24fa4472a127d5acae67cfff39506f8 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 30 Mar 2020 13:59:47 -0400 Subject: [PATCH 02/14] MNT: Add requirements, minimum requirements --- min-requirements.txt | 8 ++++++++ requirements.txt | 8 ++++++++ 2 files changed, 16 insertions(+) create mode 100644 min-requirements.txt create mode 100644 requirements.txt diff --git a/min-requirements.txt b/min-requirements.txt new file mode 100644 index 0000000000..2b241247db --- /dev/null +++ b/min-requirements.txt @@ -0,0 +1,8 @@ +# Auto-generated by tools/update_requirements.py +nibabel==2.4.1 +niflow-nipype1-workflows==0.0.1 +nipype==1.3.1 +niworkflows==1.1.4 +numpy +pybids==0.9.2 +templateflow==0.4 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..de2d6b18e7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +# Auto-generated by tools/update_requirements.py +nibabel>=2.4.1 +niflow-nipype1-workflows~=0.0.1 +nipype>=1.3.1 +niworkflows~=1.1.4 +numpy +pybids~=0.9.2 +templateflow>=0.4 From 92d7b28695819655edd3c32eb5ff705eec617412 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 30 Mar 2020 14:54:59 -0400 Subject: [PATCH 03/14] TEST: Skip tests when layouts are absent --- sdcflows/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdcflows/conftest.py b/sdcflows/conftest.py index 92bbe19b44..99aa3ebebc 100644 --- a/sdcflows/conftest.py +++ b/sdcflows/conftest.py @@ -47,7 +47,9 @@ def outdir(): @pytest.fixture def bids_layouts(): - return layouts + if layouts: + return layouts + pytest.skip() @pytest.fixture From 4a214782681be50da98e5b0b2cf954809bb56b27 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 30 Mar 2020 14:59:36 -0400 Subject: [PATCH 04/14] CI: Add Travis config --- .travis.yml | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..953823fdc8 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,42 @@ +# vim ft=yaml +os: linux +dist: xenial + +language: python +cache: pip + +python: + - 3.5 + - 3.6 + - 3.7 + - 3.8 + +env: + global: + - CHECK_TYPE="tests" + - INSTALL_DEPENDS="pip setuptools" + - DEPENDS="-r min-requirements.txt" + +before_install: + - python -m pip install --upgrade pip virtualenv + - virtualenv --python=python /tmp/venv + - source /tmp/venv/bin/activate + - python --version + - python -m pip --version + - python -m pip install --upgrade $INSTALL_DEPENDS + - python -m pip --version + +install: + - if [ -n "$DEPENDS" ]; then python -m pip install $DEPENDS; fi + - python -m pip install . + - python -c "import sdcflows; print(sdcflows.__version__)" + +before_script: + - travis_retry python -m pip install "sdcflows[$CHECK_TYPE]" + +script: + - pytest -n 2 -v --cov sdcflows --cov-report xml:cov.xml --doctest-modules sdcflows + +after_script: + - python -m pip install codecov + - python -m codecov --flags travis --file cov.xml -e $TRAVIS_JOB_NUMBER From b94d49fcb54b5dadae259edb1d228db12d2d7f36 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 30 Mar 2020 15:02:08 -0400 Subject: [PATCH 05/14] STY: PEP8 --- tools/update_requirements.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/update_requirements.py b/tools/update_requirements.py index 32eab85ec5..8eb202b8fa 100755 --- a/tools/update_requirements.py +++ b/tools/update_requirements.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -import sys from copy import copy from configparser import ConfigParser from pathlib import Path @@ -17,6 +16,7 @@ script_name = Path(__file__).relative_to(repo_root) + def to_min(req): if req.specifier: req = copy(req) @@ -25,6 +25,7 @@ def to_min(req): req.specifier = SpecifierSet(str(min_spec)) return req + lines = [f"# Auto-generated by {script_name}", ""] # Write requirements From f4b48a2c1cb5c552e865a771be9f0f96da63a8cc Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 30 Mar 2020 17:06:30 -0400 Subject: [PATCH 06/14] CI: Attempt to grab data and tools for testing --- .travis.yml | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 953823fdc8..4a47d2b6fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,12 @@ os: linux dist: xenial +addons: + apt: + packages: + - libgomp1 + + language: python cache: pip @@ -16,6 +22,9 @@ env: - CHECK_TYPE="tests" - INSTALL_DEPENDS="pip setuptools" - DEPENDS="-r min-requirements.txt" + - MRI_ROBUST_TEMPLATE=sx2n7/providers/osfstorage/5e825301d0e35400ebb481f2 + - FS_LICENSE=/tmp/freesurfer/license.txt + - TEST_DATA_HOME=/tmp/data before_install: - python -m pip install --upgrade pip virtualenv @@ -30,9 +39,26 @@ install: - if [ -n "$DEPENDS" ]; then python -m pip install $DEPENDS; fi - python -m pip install . - python -c "import sdcflows; print(sdcflows.__version__)" + - travis_retry python -m pip install "sdcflows[$CHECK_TYPE]" before_script: - - travis_retry python -m pip install "sdcflows[$CHECK_TYPE]" + # External dependencies + - travis_retry bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh); + - sudo apt-get update + - sudo apt-get install -y --no-install-recommends git-annex-standalone fsl afni ants + - curl https://files.osf.io/v1/resources/$MRI_ROBUST_TEMPLATE?direct > mri_robust_template + - sudo install mri_robust_template /usr/local/bin + - mkdir /tmp/freesurfer + - echo "b2VzdGViYW5Ac3RhbmZvcmQuZWR1CjMwNzU2CiAqQ1MzYkJ5VXMxdTVNCiBGU2kvUGJsejJxR1V3Cg==" | base64 -d > $FS_LICENSE + - python -m pip install datalad + # Data dependencies + - python -c "from templateflow import api as tfapi; + tfapi.get('MNI152NLin2009cAsym', resolution=2, desc='brain', suffix='mask'); + tfapi.get('MNI152NLin2009cAsym', resolution=2, desc='fMRIPrep', suffix='boldref');" + - mkdir -p $TEST_DATA_HOME + - datalad install -rg ///openneuro/ds001600 $TEST_DATA_HOME/ds001600 + - curl https://files.osf.io/v1/resources/9sy2a/providers/osfstorage/5d44b940bcd6d900198ed6be/?zip= --output testdata.zip + - unzip testdata.zip $TEST_DATA_HOME/testdata script: - pytest -n 2 -v --cov sdcflows --cov-report xml:cov.xml --doctest-modules sdcflows From 08bde538cb686c58382b8d2f488842d3745820c8 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Tue, 31 Mar 2020 12:04:39 -0400 Subject: [PATCH 07/14] CI: Fetch dataset with a single thread --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4a47d2b6fd..fc9690e24f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,7 +56,7 @@ before_script: tfapi.get('MNI152NLin2009cAsym', resolution=2, desc='brain', suffix='mask'); tfapi.get('MNI152NLin2009cAsym', resolution=2, desc='fMRIPrep', suffix='boldref');" - mkdir -p $TEST_DATA_HOME - - datalad install -rg ///openneuro/ds001600 $TEST_DATA_HOME/ds001600 + - datalad install -rg -J 1 ///openneuro/ds001600 $TEST_DATA_HOME/ds001600 - curl https://files.osf.io/v1/resources/9sy2a/providers/osfstorage/5d44b940bcd6d900198ed6be/?zip= --output testdata.zip - unzip testdata.zip $TEST_DATA_HOME/testdata From 7a09b33bd5c8e938732b427afd9c095e0e7a5cce Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Tue, 31 Mar 2020 12:15:15 -0400 Subject: [PATCH 08/14] CI: Fix datalad invocation --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fc9690e24f..ef45e20aea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,7 +56,7 @@ before_script: tfapi.get('MNI152NLin2009cAsym', resolution=2, desc='brain', suffix='mask'); tfapi.get('MNI152NLin2009cAsym', resolution=2, desc='fMRIPrep', suffix='boldref');" - mkdir -p $TEST_DATA_HOME - - datalad install -rg -J 1 ///openneuro/ds001600 $TEST_DATA_HOME/ds001600 + - datalad install -rg -J 1 -s ///openneuro/ds001600 $TEST_DATA_HOME/ds001600 - curl https://files.osf.io/v1/resources/9sy2a/providers/osfstorage/5d44b940bcd6d900198ed6be/?zip= --output testdata.zip - unzip testdata.zip $TEST_DATA_HOME/testdata From 3e7d1380c8238b2db2e9da14dfaa8b999effa484 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Tue, 31 Mar 2020 12:21:24 -0400 Subject: [PATCH 09/14] CI: Fix unzip command --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ef45e20aea..222ad5cb09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,7 +58,7 @@ before_script: - mkdir -p $TEST_DATA_HOME - datalad install -rg -J 1 -s ///openneuro/ds001600 $TEST_DATA_HOME/ds001600 - curl https://files.osf.io/v1/resources/9sy2a/providers/osfstorage/5d44b940bcd6d900198ed6be/?zip= --output testdata.zip - - unzip testdata.zip $TEST_DATA_HOME/testdata + - unzip testdata.zip -d $TEST_DATA_HOME/testdata script: - pytest -n 2 -v --cov sdcflows --cov-report xml:cov.xml --doctest-modules sdcflows From 0fe2cd08b126449a40c4d1f0d21b749dadebb892 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Wed, 18 Nov 2020 15:23:45 +0100 Subject: [PATCH 10/14] maint: update minimal Python to 3.6 + run python tools/update_requirements.py --- .travis.yml | 1 - min-requirements.txt | 6 +++--- requirements.txt | 8 ++++---- setup.cfg | 8 ++++---- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 222ad5cb09..fc834cdbcc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,6 @@ language: python cache: pip python: - - 3.5 - 3.6 - 3.7 - 3.8 diff --git a/min-requirements.txt b/min-requirements.txt index 2b241247db..2ad91e2dd8 100644 --- a/min-requirements.txt +++ b/min-requirements.txt @@ -1,8 +1,8 @@ # Auto-generated by tools/update_requirements.py -nibabel==2.4.1 +nibabel==3.0.1 niflow-nipype1-workflows==0.0.1 nipype==1.3.1 -niworkflows==1.1.4 +niworkflows==1.2.0 numpy -pybids==0.9.2 +pybids==0.10.2 templateflow==0.4 diff --git a/requirements.txt b/requirements.txt index de2d6b18e7..0be1f4c1a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ # Auto-generated by tools/update_requirements.py -nibabel>=2.4.1 +nibabel>=3.0.1 niflow-nipype1-workflows~=0.0.1 -nipype>=1.3.1 -niworkflows~=1.1.4 +nipype<2.0,>=1.3.1 +niworkflows<1.4,>=1.2.0 numpy -pybids~=0.9.2 +pybids>=0.10.2 templateflow>=0.4 diff --git a/setup.cfg b/setup.cfg index a07cf2bdf6..2d9039607a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,9 +6,9 @@ classifiers = Intended Audience :: Science/Research Topic :: Scientific/Engineering :: Image Recognition License :: OSI Approved :: BSD License - Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 description = Susceptibility Distortion Correction (SDC) workflows for EPI MR schemes. license = Apache-2.0 long_description = file:README.rst @@ -20,7 +20,7 @@ project_urls = url = https://www.nipreps.org/sdcflows [options] -python_requires = >=3.5 +python_requires = >=3.6 setup_requires = setuptools >= 42.0 setuptools_scm >= 3.4 @@ -56,8 +56,8 @@ docs = %(doc)s tests = pytest - pytest-xdist - pytest-cov == 2.5.1 + pytest-xdist >= 2.0 + pytest-cov == 2.10.1 coverage all = %(doc)s From 38c0faaf88f32fe455f49c02f9b40bb52d18a889 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Wed, 18 Nov 2020 17:59:23 +0100 Subject: [PATCH 11/14] fix: pytest-xdist does not execute testsetup of doctests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fc834cdbcc..15f1c92a63 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,7 @@ before_script: - unzip testdata.zip -d $TEST_DATA_HOME/testdata script: - - pytest -n 2 -v --cov sdcflows --cov-report xml:cov.xml --doctest-modules sdcflows + - pytest -v --cov sdcflows --cov-report xml:cov.xml --doctest-modules sdcflows after_script: - python -m pip install codecov From 067f2bd783fe3e0b047cd62bdfbdd752f25edba1 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Wed, 18 Nov 2020 18:10:30 +0100 Subject: [PATCH 12/14] fix(tests): skip some tests on Travis --- sdcflows/models/tests/test_pepolar.py | 2 ++ sdcflows/models/tests/test_phdiff.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/sdcflows/models/tests/test_pepolar.py b/sdcflows/models/tests/test_pepolar.py index b2453d7261..cbe7aec36f 100644 --- a/sdcflows/models/tests/test_pepolar.py +++ b/sdcflows/models/tests/test_pepolar.py @@ -1,4 +1,5 @@ """Test pepolar type of fieldmaps.""" +import os from pathlib import Path from json import loads import pytest @@ -9,6 +10,7 @@ from ..pepolar import Workflow, init_topup_wf +@pytest.mark.skipif(os.getenv("TRAVIS") == "true", reason="this is TravisCI") @pytest.mark.parametrize( "epi_path", [ diff --git a/sdcflows/models/tests/test_phdiff.py b/sdcflows/models/tests/test_phdiff.py index a4dc0e1cd7..fa72eaccfd 100644 --- a/sdcflows/models/tests/test_phdiff.py +++ b/sdcflows/models/tests/test_phdiff.py @@ -1,4 +1,5 @@ """Test phase-difference type of fieldmaps.""" +import os from pathlib import Path from json import loads @@ -9,6 +10,7 @@ from ..fieldmap import init_fmap_wf, Workflow +@pytest.mark.skipif(os.getenv("TRAVIS") == "true", reason="this is TravisCI") @pytest.mark.parametrize( "fmap_path", [ From 151932624f6975a43ab0fbdf2c5f1e02b3ca75dd Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Wed, 18 Nov 2020 19:04:30 +0100 Subject: [PATCH 13/14] fix(tests): add missing templateflow requirement --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 15f1c92a63..418b83fd89 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,6 +53,7 @@ before_script: # Data dependencies - python -c "from templateflow import api as tfapi; tfapi.get('MNI152NLin2009cAsym', resolution=2, desc='brain', suffix='mask'); + tfapi.get('MNI152NLin2009cAsym', resolution=1, label='brain', suffix='probseg'); tfapi.get('MNI152NLin2009cAsym', resolution=2, desc='fMRIPrep', suffix='boldref');" - mkdir -p $TEST_DATA_HOME - datalad install -rg -J 1 -s ///openneuro/ds001600 $TEST_DATA_HOME/ds001600 From 553a80689d156133a0d759e91f36eef8041365d7 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Wed, 18 Nov 2020 19:13:00 +0100 Subject: [PATCH 14/14] maint: update auto-generated files --- min-requirements.txt | 8 ++++---- requirements.txt | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/min-requirements.txt b/min-requirements.txt index 2ad91e2dd8..415823bb8d 100644 --- a/min-requirements.txt +++ b/min-requirements.txt @@ -1,8 +1,8 @@ # Auto-generated by tools/update_requirements.py nibabel==3.0.1 niflow-nipype1-workflows==0.0.1 -nipype==1.3.1 -niworkflows==1.2.0 +nipype==1.5.1 +niworkflows==1.3.0 numpy -pybids==0.10.2 -templateflow==0.4 +pybids==0.11.1 +templateflow==0.6 diff --git a/requirements.txt b/requirements.txt index 0be1f4c1a7..7a16a46837 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ # Auto-generated by tools/update_requirements.py nibabel>=3.0.1 niflow-nipype1-workflows~=0.0.1 -nipype<2.0,>=1.3.1 -niworkflows<1.4,>=1.2.0 +nipype<2.0,>=1.5.1 +niworkflows~=1.3.0 numpy -pybids>=0.10.2 -templateflow>=0.4 +pybids>=0.11.1 +templateflow>=0.6