enh(packaging): Infrastructure to run like fMRIPrep

Author: oesteban

dmriprep/cli/__init__.py

@@ -37,7 +37,7 @@ def get_parser():
     from templateflow.api import templates
     from packaging.version import Version
     from ..__about__ import __version__
-    from ..workflows.bold.resampling import NONSTANDARD_REFERENCES
+    from ..config import NONSTANDARD_REFERENCES
     from .version import check_latest, is_flagged
 
     verstr = 'dmriprep v{}'.format(__version__)
@@ -280,7 +280,7 @@ def main():
         work_dir = Path(retval.get('work_dir'))
         plugin_settings = retval.get('plugin_settings', None)
         subject_list = retval.get('subject_list', None)
-        fmriprep_wf = retval.get('workflow', None)
+        dmriprep_wf = retval.get('workflow', None)
         run_uuid = retval.get('run_uuid', None)
 
     if opts.reports_only:
@@ -289,15 +289,15 @@ def main():
     if opts.boilerplate:
         sys.exit(int(retcode > 0))
 
-    if fmriprep_wf and opts.write_graph:
-        fmriprep_wf.write_graph(graph2use="colored", format='svg', simple_form=True)
+    if dmriprep_wf and opts.write_graph:
+        dmriprep_wf.write_graph(graph2use="colored", format='svg', simple_form=True)
 
-    retcode = retcode or int(fmriprep_wf is None)
+    retcode = retcode or int(dmriprep_wf is None)
     if retcode != 0:
         sys.exit(retcode)
 
     # Check workflow for missing commands
-    missing = check_deps(fmriprep_wf)
+    missing = check_deps(dmriprep_wf)
     if missing:
         print("Cannot run dMRIPrep. Missing dependencies:", file=sys.stderr)
         for iface, cmd in missing:
@@ -313,7 +313,7 @@ def main():
 
     errno = 1  # Default is error exit unless otherwise set
     try:
-        fmriprep_wf.run(**plugin_settings)
+        dmriprep_wf.run(**plugin_settings)
     except Exception as e:
         if not opts.notrack:
             from ..utils.sentry import process_crashfile
@@ -417,7 +417,7 @@ def build_workflow(opts, retval):
     from niworkflows.utils.bids import collect_participants
     from niworkflows.reports import generate_reports
     from ..__about__ import __version__
-    from ..workflows.base import init_fmriprep_wf
+    from ..workflows.base import init_dmriprep_wf
 
     build_log = nlogging.getLogger('nipype.workflow')
 
@@ -546,7 +546,7 @@ def build_workflow(opts, retval):
         uuid=run_uuid)
     )
 
-    retval['workflow'] = init_fmriprep_wf(
+    retval['workflow'] = init_dmriprep_wf(
         anat_only=opts.anat_only,
         aroma_melodic_dim=opts.aroma_melodic_dimensionality,
         bold2t1w_dof=opts.bold2t1w_dof,

dmriprep/cli/run.py

@@ -0,0 +1,51 @@
+"""Test CLI."""
+from packaging.version import Version
+import pytest
+from .. import version as _version
+from ... import __about__
+from ..run import get_parser
+
+
+@pytest.mark.parametrize(('current', 'latest'), [
+    ('1.0.0', '1.3.2'),
+    ('1.3.2', '1.3.2')
+])
+def test_get_parser_update(monkeypatch, capsys, current, latest):
+    """Make sure the out-of-date banner is shown."""
+    expectation = Version(current) < Version(latest)
+
+    def _mock_check_latest(*args, **kwargs):
+        return Version(latest)
+
+    monkeypatch.setattr(__about__, '__version__', current)
+    monkeypatch.setattr(_version, 'check_latest', _mock_check_latest)
+
+    get_parser()
+    captured = capsys.readouterr().err
+
+    msg = """\
+You are using dMRIPrep-%s, and a newer version of dMRIPrep is available: %s.
+Please check out our documentation about how and when to upgrade:
+https://dmriprep.readthedocs.io/en/latest/faq.html#upgrading""" % (current, latest)
+
+    assert (msg in captured) is expectation
+
+
+@pytest.mark.parametrize('flagged', [
+    (True, None),
+    (True, 'random reason'),
+    (False, None),
+])
+def test_get_parser_blacklist(monkeypatch, capsys, flagged):
+    """Make sure the blacklisting banner is shown."""
+    def _mock_is_bl(*args, **kwargs):
+        return flagged
+
+    monkeypatch.setattr(_version, 'is_flagged', _mock_is_bl)
+
+    get_parser()
+    captured = capsys.readouterr().err
+
+    assert ('FLAGGED' in captured) is flagged[0]
+    if flagged[0]:
+        assert ((flagged[1] or 'reason: unknown') in captured)

dmriprep/cli/tests/__init__.py

@@ -0,0 +1,163 @@
+"""Test version checks."""
+from datetime import datetime
+from pathlib import Path
+from packaging.version import Version
+import pytest
+from .. import version as _version
+from ..version import check_latest, DATE_FMT, requests, is_flagged
+
+
+class MockResponse:
+    """Mocks the requests module so that Pypi is not actually queried."""
+
+    status_code = 200
+    _json = {
+        "releases": {
+            '1.0.0': None,
+            '1.0.1': None,
+            '1.1.0': None,
+            '1.1.1rc1': None
+        }
+    }
+
+    def __init__(self, code=200, json=None):
+        """Allow setting different response codes."""
+        self.status_code = code
+        if json is not None:
+            self._json = json
+
+    def json(self):
+        """Redefine the response object."""
+        return self._json
+
+
+def test_check_latest1(tmpdir, monkeypatch):
+    """Test latest version check."""
+    tmpdir.chdir()
+    monkeypatch.setenv('HOME', str(tmpdir))
+    assert str(Path.home()) == str(tmpdir)
+
+    def mock_get(*args, **kwargs):
+        return MockResponse()
+    monkeypatch.setattr(requests, "get", mock_get)
+
+    # Initially, cache should not exist
+    cachefile = Path.home() / '.cache' / 'dmriprep' / 'latest'
+    assert not cachefile.exists()
+
+    # First check actually fetches from pypi
+    v = check_latest()
+    assert cachefile.exists()
+    assert isinstance(v, Version)
+    assert v == Version('1.1.0')
+    assert cachefile.read_text().split('|') == [str(v), datetime.now().strftime(DATE_FMT)]
+
+    # Second check - test the cache file is read
+    cachefile.write_text('|'.join(('1.0.0', cachefile.read_text().split('|')[1])))
+    v = check_latest()
+    assert isinstance(v, Version)
+    assert v == Version('1.0.0')
+
+    # Third check - forced oudating of cache
+    cachefile.write_text('2.0.0|20180121')
+    v = check_latest()
+    assert isinstance(v, Version)
+    assert v == Version('1.1.0')
+
+    # Mock timeouts
+    def mock_get(*args, **kwargs):
+        raise requests.exceptions.Timeout
+    monkeypatch.setattr(requests, "get", mock_get)
+
+    cachefile.write_text('|'.join(('1.0.0', cachefile.read_text().split('|')[1])))
+    v = check_latest()
+    assert isinstance(v, Version)
+    assert v == Version('1.0.0')
+
+    cachefile.write_text('2.0.0|20180121')
+    v = check_latest()
+    assert v is None
+
+    cachefile.unlink()
+    v = check_latest()
+    assert v is None
+
+
+@pytest.mark.parametrize(('result', 'code', 'json'), [
+    (None, 404, None),
+    (None, 200, {'releases': {'1.0.0rc1': None}}),
+    (Version('1.1.0'), 200, None),
+    (Version('1.0.0'), 200, {'releases': {'1.0.0': None}}),
+])
+def test_check_latest2(tmpdir, monkeypatch, result, code, json):
+    """Test latest version check with varying server responses."""
+    tmpdir.chdir()
+    monkeypatch.setenv('HOME', str(tmpdir))
+    assert str(Path.home()) == str(tmpdir)
+
+    def mock_get(*args, **kwargs):
+        return MockResponse(code=code, json=json)
+    monkeypatch.setattr(requests, "get", mock_get)
+
+    v = check_latest()
+    if result is None:
+        assert v is None
+    else:
+        assert isinstance(v, Version)
+        assert v == result
+
+
+@pytest.mark.parametrize('bad_cache', [
+    '3laj#r???d|3akajdf#',
+    '2.0.0|3akajdf#',
+    '|'.join(('2.0.0', datetime.now().strftime(DATE_FMT), '')),
+    ''
+])
+def test_check_latest3(tmpdir, monkeypatch, bad_cache):
+    """Test latest version check when the cache file is corrupted."""
+    tmpdir.chdir()
+    monkeypatch.setenv('HOME', str(tmpdir))
+    assert str(Path.home()) == str(tmpdir)
+
+    def mock_get(*args, **kwargs):
+        return MockResponse()
+    monkeypatch.setattr(requests, "get", mock_get)
+
+    # Initially, cache should not exist
+    cachefile = Path.home() / '.cache' / 'dmriprep' / 'latest'
+    cachefile.parent.mkdir(parents=True, exist_ok=True)
+    assert not cachefile.exists()
+
+    cachefile.write_text(bad_cache)
+    v = check_latest()
+    assert isinstance(v, Version)
+    assert v == Version('1.1.0')
+
+
+@pytest.mark.parametrize(('result', 'version', 'code', 'json'), [
+    (False, '1.2.1', 200, {'flagged': {'1.0.0': None}}),
+    (True, '1.2.1', 200, {'flagged': {'1.2.1': None}}),
+    (True, '1.2.1', 200, {'flagged': {'1.2.1': 'FATAL Bug!'}}),
+    (False, '1.2.1', 404, {'flagged': {'1.0.0': None}}),
+    (False, '1.2.1', 200, {'flagged': []}),
+    (False, '1.2.1', 200, {}),
+])
+def test_is_flagged(monkeypatch, result, version, code, json):
+    """Test that the flagged-versions check is correct."""
+    monkeypatch.setattr(_version, "__version__", version)
+
+    def mock_get(*args, **kwargs):
+        return MockResponse(code=code, json=json)
+    monkeypatch.setattr(requests, "get", mock_get)
+
+    val, reason = is_flagged()
+    assert val is result
+
+    test_reason = None
+    if val:
+        test_reason = json.get('flagged', {}).get(version, None)
+
+    if test_reason is not None:
+        assert reason == test_reason
+    else:
+        assert reason is None

dmriprep/cli/tests/test_run.py

@@ -0,0 +1,77 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
+# vi: set ft=python sts=4 ts=4 sw=4 et:
+"""Version CLI helpers."""
+
+from pathlib import Path
+from datetime import datetime
+import requests
+from .. import __version__
+
+RELEASE_EXPIRY_DAYS = 14
+DATE_FMT = '%Y%m%d'
+
+
+def check_latest():
+    """Determine whether this is the latest version."""
+    from packaging.version import Version, InvalidVersion
+
+    latest = None
+    date = None
+    outdated = None
+    cachefile = Path.home() / '.cache' / 'dmriprep' / 'latest'
+    cachefile.parent.mkdir(parents=True, exist_ok=True)
+
+    try:
+        latest, date = cachefile.read_text().split('|')
+    except Exception:
+        pass
+    else:
+        try:
+            latest = Version(latest)
+            date = datetime.strptime(date, DATE_FMT)
+        except (InvalidVersion, ValueError):
+            latest = None
+        else:
+            if abs((datetime.now() - date).days) > RELEASE_EXPIRY_DAYS:
+                outdated = True
+
+    if latest is None or outdated is True:
+        try:
+            response = requests.get(url='https://pypi.org/pypi/dmriprep/json', timeout=1.0)
+        except Exception:
+            response = None
+
+        if response and response.status_code == 200:
+            versions = [Version(rel) for rel in response.json()['releases'].keys()]
+            versions = [rel for rel in versions if not rel.is_prerelease]
+            if versions:
+                latest = sorted(versions)[-1]
+        else:
+            latest = None
+
+    if latest is not None:
+        try:
+            cachefile.write_text('|'.join(('%s' % latest, datetime.now().strftime(DATE_FMT))))
+        except Exception:
+            pass
+
+    return latest
+
+
+def is_flagged():
+    """Check whether current version is flagged."""
+    # https://raw.githubusercontent.com/poldracklab/dmriprep/master/.versions.json
+    flagged = tuple()
+    try:
+        response = requests.get(url="""\
+https://raw.githubusercontent.com/poldracklab/dmriprep/master/.versions.json""", timeout=1.0)
+    except Exception:
+        response = None
+
+    if response and response.status_code == 200:
+        flagged = response.json().get('flagged', {}) or {}
+
+    if __version__ in flagged:
+        return True, flagged[__version__]
+
+    return False, None

dmriprep/cli/tests/test_version.py

@@ -0,0 +1,5 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
+# vi: set ft=python sts=4 ts=4 sw=4 et:
+"""Settings."""
+
+NONSTANDARD_REFERENCES = ['anat', 'T1w', 'dwi', 'fsnative']