Merge pull request #59 from mgxd/fix/layout-reindex

FIX: Ensure templateflow Layout is re-indexed when updating

Author: mgxd

.circleci/config.yml

@@ -45,7 +45,7 @@ jobs:
             source /tmp/venv/bin/activate
             pip install -U pip
             pip install -r /tmp/src/templateflow/requirements.txt
-            pip install "datalad ~= 0.11.8" doi2bib
+            pip install "datalad ~= 0.11.8" "doi2bib < 0.4"
             pip install "setuptools>=42.0" "setuptools_scm[toml] >= 3.4" twine codecov
 
       - run:

setup.cfg

@@ -49,7 +49,7 @@ exclude =
 
 [options.extras_require]
 citations =
-    doi2bib
+    doi2bib < 0.4.0
 datalad =
     datalad ~= 0.12.0
 doc =

templateflow/conf/__init__.py

@@ -45,11 +45,19 @@
 def update(local=False, overwrite=True, silent=False):
     """Update an existing DataLad or S3 home."""
     if TF_USE_DATALAD and _update_datalad():
-        return True
-
-    from ._s3 import update as _update_s3
+        success = True
+    else:
+        from ._s3 import update as _update_s3
+        success = _update_s3(TF_HOME, local=local, overwrite=overwrite, silent=silent)
 
-    return _update_s3(TF_HOME, local=local, overwrite=overwrite, silent=silent)
+    # update Layout only if necessary
+    if success and TF_LAYOUT is not None:
+        init_layout()
+        # ensure the api uses the updated layout
+        import importlib
+        from .. import api
+        importlib.reload(api)
+    return success
 
 
 def setup_home(force=False):
@@ -76,9 +84,12 @@ def _update_datalad():
 
 
 TF_LAYOUT = None
-try:
+
+
+def init_layout():
     from .bids import Layout
 
+    global TF_LAYOUT
     TF_LAYOUT = Layout(
         TF_HOME,
         validate=False,
@@ -92,5 +103,9 @@ def _update_datalad():
             "scripts",
         ],
     )
+
+
+try:
+    init_layout()
 except ImportError:
     pass

templateflow/conf/_s3.py

@@ -27,15 +27,15 @@ def _get_skeleton_file():
     import requests
 
     try:
-        r = requests.get(TF_SKEL_URL(release="master", ext="md5", allow_redirects=True))
+        r = requests.get(TF_SKEL_URL(release="master", ext="md5"), allow_redirects=True)
     except requests.exceptions.ConnectionError:
         return
 
     if not r.ok:
         return
 
     if r.content.decode().split()[0] != TF_SKEL_MD5:
-        r = requests.get(TF_SKEL_URL(release="master", ext="zip", allow_redirects=True))
+        r = requests.get(TF_SKEL_URL(release="master", ext="zip"), allow_redirects=True)
         if r.ok:
             from os import close
 

templateflow/tests/test_conf.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+import pytest
+from .. import conf, api
+
+
+@pytest.mark.skipif(conf.TF_USE_DATALAD, reason="S3 only")
+def test_update_s3(tmp_path):
+    conf.TF_HOME = tmp_path / 'templateflow'
+    conf.TF_HOME.mkdir(exist_ok=True)
+
+    # replace TF_SKEL_URL with the path of a legacy skeleton
+    _skel_url = conf._s3.TF_SKEL_URL
+    conf._s3.TF_SKEL_URL = (
+        "https://github.com/templateflow/python-client/raw/0.5.0/"
+        "templateflow/conf/templateflow-skel.{ext}".format
+    )
+    # initialize templateflow home, making sure to pull the legacy skeleton
+    conf.update(local=False)
+    # ensure we can grab a file
+    assert Path(api.get('MNI152NLin2009cAsym', resolution=2, desc='brain', suffix='mask')).exists()
+    # and ensure we can't fetch one that doesn't yet exist
+    assert not api.get('Fischer344', hemi='L', desc='brain', suffix='mask')
+
+    # refresh the skeleton using the most recent skeleton
+    conf._s3.TF_SKEL_URL = _skel_url
+    conf.update(local=True, overwrite=True)
+    assert Path(api.get('Fischer344', hemi='L', desc='brain', suffix='mask')).exists()