Merge pull request #816 from effigies/mnt/data_module_and_docs

ENH: Add data loader class, install at root and data modules

Author: effigies

docs/api.rst

@@ -7,6 +7,7 @@ Information on specific functions, classes, and methods.
 
    api/niworkflows.anat
    api/niworkflows.cli
+   api/niworkflows.data
    api/niworkflows.dwi
    api/niworkflows.engine
    api/niworkflows.func

docs/conf.py

@@ -111,7 +111,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -247,7 +247,7 @@
 
 apidoc_module_dir = "../niworkflows"
 apidoc_output_dir = "api"
-apidoc_excluded_paths = ["conftest.py", "*/tests/*", "tests/*", "data/*", "testing.py"]
+apidoc_excluded_paths = ["conftest.py", "*/tests/*", "tests/*", "testing.py"]
 apidoc_separate_modules = True
 apidoc_extra_args = ["--module-first", "-d 1", "-T"]
 

docs/requirements.txt

@@ -1,10 +1,9 @@
 attrs
-furo ~= 2022.4.7
+furo
 nipype >= 1.5.1
 traits < 6.4
 packaging
 pytest
-sphinx ~= 4.2
+sphinx
 sphinxcontrib-apidoc
-sphinxcontrib-napoleon
 templateflow

niworkflows/__init__.py

@@ -4,6 +4,7 @@
 import logging
 
 from .__about__ import __packagename__, __copyright__, __credits__
+from .data import Loader
 try:
     from ._version import __version__
 except ImportError:  # pragma: no cover
@@ -16,6 +17,7 @@
     "__copyright__",
     "__credits__",
     "NIWORKFLOWS_LOG",
+    "load_resource",
 ]
 
 NIWORKFLOWS_LOG = logging.getLogger(__packagename__)
@@ -27,3 +29,5 @@
     matplotlib.use("Agg")
 except ImportError:
     pass
+
+load_resource = Loader(__package__)

niworkflows/anat/ants.py

@@ -25,7 +25,6 @@
 # general purpose
 from collections import OrderedDict
 from multiprocessing import cpu_count
-from pkg_resources import resource_filename as pkgr_fn
 from warnings import warn
 
 # nipype
@@ -40,6 +39,7 @@
     ThresholdImage,
 )
 
+from ..data import load as load_data
 from ..utils.misc import get_template_specs
 from ..utils.connections import pop_file as _pop
 
@@ -302,9 +302,7 @@ def init_brain_extraction_wf(
         else "antsBrainExtractionNoLaplacian_%s.json"
     )
     norm = pe.Node(
-        Registration(
-            from_file=pkgr_fn("niworkflows.data", settings_file % normalization_quality)
-        ),
+        Registration(from_file=load_data(settings_file % normalization_quality)),
         name="norm",
         n_procs=omp_nthreads,
         mem_gb=mem_gb,

niworkflows/conftest.py

@@ -30,6 +30,8 @@
 import pytest
 import tempfile
 
+from . import load_resource
+
 try:
     import importlib_resources
 except ImportError:
@@ -40,7 +42,7 @@
 
 
 def find_resource_or_skip(resource):
-    pathlike = importlib_resources.files("niworkflows") / resource
+    pathlike = load_resource(resource)
     if not pathlike.exists():
         pytest.skip(f"Missing resource {resource}; run this test from a source repository")
     return pathlike
@@ -63,7 +65,7 @@ def add_np(doctest_namespace):
     doctest_namespace["datadir"] = data_dir
     doctest_namespace["data_dir_canary"] = data_dir_canary
     doctest_namespace["bids_collect_data"] = collect_data
-    doctest_namespace["test_data"] = importlib_resources.files("niworkflows") / "tests" / "data"
+    doctest_namespace["test_data"] = load_resource('tests/data')
 
     tmpdir = tempfile.TemporaryDirectory()
 

niworkflows/data/__init__.py

@@ -0,0 +1,182 @@
+"""Niworkflows data files
+
+.. autofunction:: load
+
+.. automethod:: load.readable
+
+.. automethod:: load.as_path
+
+.. automethod:: load.cached
+
+.. autoclass:: Loader
+"""
+from __future__ import annotations
+
+import atexit
+import os
+from contextlib import AbstractContextManager, ExitStack
+from functools import cached_property
+from pathlib import Path
+from types import ModuleType
+from typing import Union
+
+try:
+    from functools import cache
+except ImportError:  # PY38
+    from functools import lru_cache as cache
+
+try:  # Prefer backport to leave consistency to dependency spec
+    from importlib_resources import as_file, files
+except ImportError:
+    from importlib.resources import as_file, files  # type: ignore
+
+try:  # Prefer stdlib so Sphinx can link to authoritative documentation
+    from importlib.resources.abc import Traversable
+except ImportError:
+    from importlib_resources.abc import Traversable
+
+__all__ = ["load"]
+
+
+class Loader:
+    """A loader for package files relative to a module
+
+    This class wraps :mod:`importlib.resources` to provide a getter
+    function with an interpreter-lifetime scope. For typical packages
+    it simply passes through filesystem paths as :class:`~pathlib.Path`
+    objects. For zipped distributions, it will unpack the files into
+    a temporary directory that is cleaned up on interpreter exit.
+
+    This loader accepts a fully-qualified module name or a module
+    object.
+
+    Expected usage::
+
+        '''Data package
+
+        .. autofunction:: load_data
+
+        .. automethod:: load_data.readable
+
+        .. automethod:: load_data.as_path
+
+        .. automethod:: load_data.cached
+        '''
+
+        from niworkflows.data import Loader
+
+        load_data = Loader(__package__)
+
+    :class:`~Loader` objects implement the :func:`callable` interface
+    and generate a docstring, and are intended to be treated and documented
+    as functions.
+
+    For greater flexibility and improved readability over the ``importlib.resources``
+    interface, explicit methods are provided to access resources.
+
+    +---------------+----------------+------------------+
+    | On-filesystem | Lifetime       | Method           |
+    +---------------+----------------+------------------+
+    | `True`        | Interpreter    | :meth:`cached`   |
+    +---------------+----------------+------------------+
+    | `True`        | `with` context | :meth:`as_path`  |
+    +---------------+----------------+------------------+
+    | `False`       | n/a            | :meth:`readable` |
+    +---------------+----------------+------------------+
+
+    It is also possible to use ``Loader`` directly::
+
+        from niworkflows.data import Loader
+
+        Loader(other_package).readable('data/resource.ext').read_text()
+
+        with Loader(other_package).as_path('data') as pkgdata:
+            # Call function that requires full Path implementation
+            func(pkgdata)
+
+        # contrast to
+
+        from importlib_resources import files, as_file
+
+        files(other_package).joinpath('data/resource.ext').read_text()
+
+        with as_file(files(other_package) / 'data') as pkgdata:
+            func(pkgdata)
+
+    .. automethod:: readable
+
+    .. automethod:: as_path
+
+    .. automethod:: cached
+    """
+
+    def __init__(self, anchor: Union[str, ModuleType]):
+        self._anchor = anchor
+        self.files = files(anchor)
+        self.exit_stack = ExitStack()
+        atexit.register(self.exit_stack.close)
+        # Allow class to have a different docstring from instances
+        self.__doc__ = self._doc
+
+    @cached_property
+    def _doc(self):
+        """Construct docstring for instances
+
+        Lists the public top-level paths inside the location, where
+        non-public means has a `.` or `_` prefix or is a 'tests'
+        directory.
+        """
+        top_level = sorted(
+            os.path.relpath(p, self.files) + "/"[: p.is_dir()]
+            for p in self.files.iterdir()
+            if p.name[0] not in (".", "_") and p.name != "tests"
+        )
+        doclines = [
+            f"Load package files relative to ``{self._anchor}``.",
+            "",
+            "This package contains the following (top-level) files/directories:",
+            "",
+            *(f"* ``{path}``" for path in top_level),
+        ]
+
+        return "\n".join(doclines)
+
+    def readable(self, *segments) -> Traversable:
+        """Provide read access to a resource through a Path-like interface.
+
+        This file may or may not exist on the filesystem, and may be
+        efficiently used for read operations, including directory traversal.
+
+        This result is not cached or copied to the filesystem in cases where
+        that would be necessary.
+        """
+        return self.files.joinpath(*segments)
+
+    def as_path(self, *segments) -> AbstractContextManager[Path]:
+        """Ensure data is available as a :class:`~pathlib.Path`.
+
+        This method generates a context manager that yields a Path when
+        entered.
+
+        This result is not cached, and any temporary files that are created
+        are deleted when the context is exited.
+        """
+        return as_file(self.files.joinpath(*segments))
+
+    @cache
+    def cached(self, *segments) -> Path:
+        """Ensure data is available as a :class:`~pathlib.Path`.
+
+        Any temporary files that are created remain available throughout
+        the duration of the program, and are deleted when Python exits.
+
+        Results are cached so that multiple calls do not unpack the same
+        data multiple times, but the cache is sensitive to the specific
+        argument(s) passed.
+        """
+        return self.exit_stack.enter_context(as_file(self.files.joinpath(*segments)))
+
+    __call__ = cached
+
+
+load = Loader(__package__)

niworkflows/func/util.py

@@ -22,13 +22,13 @@
 #
 """Utility workflows."""
 from packaging.version import parse as parseversion, Version
-from pkg_resources import resource_filename as pkgr_fn
 
 from nipype.pipeline import engine as pe
 from nipype.interfaces import utility as niu, fsl, afni
 
 from templateflow.api import get as get_template
 
+from .. import data
 from ..engine.workflows import LiterateWorkflow as Workflow
 from ..interfaces.fixes import (
     FixHeaderRegistration as Registration,
@@ -452,9 +452,7 @@ def init_enhance_and_skullstrip_bold_wf(
 
         # Set up spatial normalization
         norm = pe.Node(
-            Registration(
-                from_file=pkgr_fn("niworkflows.data", "epi_atlasbased_brainmask.json")
-            ),
+            Registration(from_file=data.load("epi_atlasbased_brainmask.json")),
             name="norm",
             n_procs=omp_nthreads,
         )

niworkflows/interfaces/bids.py

@@ -27,7 +27,6 @@
 from pathlib import Path
 import shutil
 import os
-from pkg_resources import resource_filename as _pkgres
 import re
 
 import nibabel as nb
@@ -50,12 +49,13 @@
 )
 from nipype.interfaces.io import add_traits
 import templateflow as tf
+from .. import data
 from ..utils.bids import _init_layout, relative_to_root
 from ..utils.images import set_consumables, unsafe_write_nifti_header_and_data
 from ..utils.misc import _copy_any, unlink
 
 regz = re.compile(r"\.gz$")
-_pybids_spec = loads(Path(_pkgres("niworkflows", "data/nipreps.json")).read_text())
+_pybids_spec = loads(data.load.readable("nipreps.json").read_text())
 BIDS_DERIV_ENTITIES = _pybids_spec["entities"]
 BIDS_DERIV_PATTERNS = tuple(_pybids_spec["default_path_patterns"])
 

niworkflows/interfaces/norm.py

@@ -24,7 +24,6 @@
 from os import path as op
 
 from multiprocessing import cpu_count
-import pkg_resources as pkgr
 from packaging.version import Version
 import numpy as np
 
@@ -40,6 +39,7 @@
 
 from templateflow.api import get as get_template
 from .. import NIWORKFLOWS_LOG, __version__
+from ..data import load as load_data
 from .fixes import FixHeaderRegistration as Registration
 
 
@@ -166,16 +166,13 @@ def _get_settings(self):
             self.inputs.moving.lower(), self.inputs.flavor
         )
 
+        data_dir = load_data()
         # Get a list of settings files that match the flavor.
         filenames = [
-            i
-            for i in pkgr.resource_listdir("niworkflows", "data")
-            if i.startswith(filestart) and i.endswith(".json")
+            i for i in data_dir.iterdir() if i.startswith(filestart) and i.endswith(".json")
         ]
         # Return the settings files.
-        return [
-            pkgr.resource_filename("niworkflows.data", f) for f in sorted(filenames)
-        ]
+        return [str(data_dir / f) for f in sorted(filenames)]
 
     def _run_interface(self, runtime):
         # Get a list of settings files.