ENH: support importlib.resources for editable wheels

Author: dnicolodi

mesonpy/_editable.py

@@ -23,20 +23,163 @@
     from types import ModuleType
     from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
     NodeBase = Dict[str, Union[Node, str]]
+    PathStr = Union[str, os.PathLike[str]]
 else:
     NodeBase = dict
 
 
+if sys.version_info >= (3, 12):
+    from importlib.resources.abc import Traversable, TraversableResources
+elif sys.version_info >= (3, 9):
+    from importlib.abc import Traversable, TraversableResources
+else:
+    class Traversable:
+        pass
+    class TraversableResources:
+        pass
+
+
 MARKER = 'MESONPY_EDITABLE_SKIP'
 VERBOSE = 'MESONPY_EDITABLE_VERBOSE'
 
+
+class MesonpyOrphan(Traversable):
+    def __init__(self, name: str):
+        self._name = name
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def is_dir(self) -> bool:
+        return False
+
+    def is_file(self) -> bool:
+        return False
+
+    def iterdir(self) -> Iterator[Traversable]:
+        raise FileNotFoundError()
+
+    def open(self, *args, **kwargs):  # type: ignore
+        raise FileNotFoundError()
+
+    def joinpath(self, *descendants: PathStr) -> Traversable:
+        if not descendants:
+            return self
+        name = os.fspath(descendants[-1]).split('/')[-1]
+        return MesonpyOrphan(name)
+
+    def __truediv__(self, child: PathStr) -> Traversable:
+        return self.joinpath(child)
+
+    def read_bytes(self) -> bytes:
+        raise FileNotFoundError()
+
+    def read_text(self, encoding: Optional[str] = None) -> str:
+        raise FileNotFoundError()
+
+
+class MesonpyTraversable(Traversable):
+    def __init__(self, name: str, tree: Node):
+        self._name = name
+        self._tree = tree
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def is_dir(self) -> bool:
+        return True
+
+    def is_file(self) -> bool:
+        return False
+
+    def iterdir(self) -> Iterator[Traversable]:
+        for name, node in self._tree.items():
+            yield MesonpyTraversable(name, node) if isinstance(node, dict) else pathlib.Path(node)  # type: ignore
+
+    def open(self, *args, **kwargs):  # type: ignore
+        raise IsADirectoryError()
+
+    @staticmethod
+    def _flatten(names: Tuple[PathStr, ...]) -> Iterator[str]:
+        for name in names:
+            yield from os.fspath(name).split('/')
+
+    def joinpath(self, *descendants: PathStr) -> Traversable:
+        if not descendants:
+            return self
+        names = self._flatten(descendants)
+        name = next(names)
+        node = self._tree.get(name)
+        if isinstance(node, dict):
+            return MesonpyTraversable(name, node).joinpath(*names)
+        if isinstance(node, str):
+            return pathlib.Path(node).joinpath(*names)
+        return MesonpyOrphan(name).joinpath(*names)
+
+    def __truediv__(self, child: PathStr) -> Traversable:
+        return self.joinpath(child)
+
+    def read_bytes(self) -> bytes:
+        raise IsADirectoryError()
+
+    def read_text(self, encoding: Optional[str] = None) -> str:
+        raise IsADirectoryError()
+
+
+class MesonpyReader(TraversableResources):
+    def __init__(self, name: str, tree: Node):
+        self._name = name
+        self._tree = tree
+
+    def files(self) -> Traversable:
+        return MesonpyTraversable(self._name, self._tree)
+
+
+class ExtensionFileLoader(importlib.machinery.ExtensionFileLoader):
+    def __init__(self, name: str, path: str, tree: Node):
+        super().__init__(name, path)
+        self._tree = tree
+
+    def get_resource_reader(self, name: str) -> TraversableResources:
+        return MesonpyReader(name, self._tree)
+
+
+class SourceFileLoader(importlib.machinery.SourceFileLoader):
+    def __init__(self, name: str, path: str, tree: Node):
+        super().__init__(name, path)
+        self._tree = tree
+
+    def get_resource_reader(self, name: str) -> TraversableResources:
+        return MesonpyReader(name, self._tree)
+
+
+class SourcelessFileLoader(importlib.machinery.SourcelessFileLoader):
+    def __init__(self, name: str, path: str, tree: Node):
+        super().__init__(name, path)
+        self._tree = tree
+
+    def get_resource_reader(self, name: str) -> TraversableResources:
+        return MesonpyReader(name, self._tree)
+
+
 LOADERS = [
-    (importlib.machinery.ExtensionFileLoader, tuple(importlib.machinery.EXTENSION_SUFFIXES)),
-    (importlib.machinery.SourceFileLoader, tuple(importlib.machinery.SOURCE_SUFFIXES)),
-    (importlib.machinery.SourcelessFileLoader, tuple(importlib.machinery.BYTECODE_SUFFIXES)),
+    (ExtensionFileLoader, tuple(importlib.machinery.EXTENSION_SUFFIXES)),
+    (SourceFileLoader, tuple(importlib.machinery.SOURCE_SUFFIXES)),
+    (SourcelessFileLoader, tuple(importlib.machinery.BYTECODE_SUFFIXES)),
 ]
 
 
+def build_module_spec(cls: type, name: str, path: str, tree: Optional[Node]) -> importlib.machinery.ModuleSpec:
+    loader = cls(name, path, tree)
+    spec = importlib.machinery.ModuleSpec(name, loader, origin=path)
+    spec.has_location = True
+    if loader.is_package(name):
+        spec.submodule_search_locations = []
+    return spec
+
+
 class Node(NodeBase):
     """Tree structure to store a virtual filesystem view."""
 

tests/packages/simple/__init__.py

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: 2021 The meson-python developers
+#
+# SPDX-License-Identifier: MIT
+
+from .test import data  # noqa: F401

tests/packages/simple/data.txt

@@ -0,0 +1 @@
+ABC

tests/packages/simple/data.txt.license

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2021 The meson-python developers
+#
+# SPDX-License-Identifier: MIT

tests/packages/simple/meson.build

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: 2021 The meson-python developers
+#
+# SPDX-License-Identifier: MIT
+
+project('simple', version: '1.0.0')
+
+py = import('python').find_installation()
+
+py.install_sources(
+  '__init__.py',
+  'test.py',
+  'data.txt',
+  subdir: 'simple')

tests/packages/simple/pyproject.toml

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2021 The meson-python developers
+#
+# SPDX-License-Identifier: MIT
+
+[build-system]
+build-backend = 'mesonpy'
+requires = ['meson-python']

tests/packages/simple/test.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2021 The meson-python developers
+#
+# SPDX-License-Identifier: MIT
+
+import pathlib
+
+
+def data():
+    with pathlib.Path(__file__).parent.joinpath('data.txt').open() as f:
+        return f.read().rstrip()

tests/test_editable.py

@@ -3,6 +3,10 @@
 # SPDX-License-Identifier: MIT
 
 import os
+import pathlib
+import sys
+
+import pytest
 
 import mesonpy
 
@@ -94,8 +98,37 @@ def test_mesonpy_meta_finder(package_complex, tmp_build_path):
         # remove finder from the meta path
         del sys.meta_path[0]
 
+
+def test_mesonpy_traversable():
+    tree = _editable.Node()
+    tree[('package', '__init__.py')] = '/tmp/src/package/__init__.py'
+    tree[('package', 'src.py')] = '/tmp/src/package/src.py'
+    tree[('package', 'data.txt')] = '/tmp/src/package/data.txt'
+    tree[('package', 'nested', '__init__.py')] = '/tmp/src/package/nested/__init__.py'
+    tree[('package', 'nested', 'some.py')] = '/tmp/src/package/nested/some.py'
+    tree[('package', 'nested', 'generated.txt')] = '/tmp/build/generated.txt'
+    traversable = _editable.MesonpyTraversable('package', tree['package'])
+    assert {x.name for x in traversable.iterdir()} == {'__init__.py', 'src.py', 'data.txt', 'nested'}
+    nested = traversable / 'nested'
+    assert nested.is_dir()
+    assert {x.name for x in nested.iterdir()} == {'__init__.py', 'some.py', 'generated.txt'}
+    generated = traversable.joinpath('nested', 'generated.txt')
+    assert isinstance(generated, pathlib.Path)
+    assert generated == pathlib.Path('/tmp/build/generated.txt')
+    bad = traversable / 'bad'
+    assert not bad.is_file()
+    assert not bad.is_dir()
+    with pytest.raises(FileNotFoundError):
+        bad.open()
+
+
+def test_resources(tmp_path):
+    # build a package in a temporary directory
+    package_path = pathlib.Path(__file__).parent / 'packages' / 'simple'
+    mesonpy.Project(package_path, tmp_path)
+
     # point the meta finder to the build directory
-    finder = _editable.MesonpyMetaFinder({'simple'}, os.fspath(tmp_path / 'build'), ['ninja'], False)
+    finder = _editable.MesonpyMetaFinder({'simple'}, os.fspath(tmp_path / 'build'), ['ninja'])
 
     # verify that we can look up resources
     spec = finder.find_spec('simple')
@@ -106,6 +139,30 @@ def test_mesonpy_meta_finder(package_complex, tmp_build_path):
         text = f.read().rstrip()
     assert text == 'ABC'
 
+
+@pytest.mark.skipif(sys.version_info < (3, 9), reason='importlib.resources not available')
+def test_importlib_resources(tmp_path):
+    # build a package in a temporary directory
+    package_path = pathlib.Path(__file__).parent / 'packages' / 'simple'
+    mesonpy.Project(package_path, tmp_path)
+
+    # point the meta finder to the build directory
+    finder = _editable.MesonpyMetaFinder({'simple'}, os.fspath(tmp_path / 'build'), ['ninja'])
+
+    try:
+        # install the finder in the meta path
+        sys.meta_path.insert(0, finder)
+        # verify that we can import the modules
+        import simple
+        assert simple.__spec__.origin == os.fspath(package_path / '__init__.py')
+        assert simple.__file__ == os.fspath(package_path / '__init__.py')
+        assert simple.data() == 'ABC'
+        # load resources via importlib
+        import importlib.resources
+        with importlib.resources.files(simple).joinpath('data.txt').open() as f:
+            text = f.read().rstrip()
+        assert text == 'ABC'
+        assert importlib.resources.files(simple).joinpath('data.txt').read_text().rstrip() == 'ABC'
     finally:
         # remove finder from the meta path
         del sys.meta_path[0]