diff --git a/fsspec_python/fs.py b/fsspec_python/fs.py index 2968e05..0dd7515 100644 --- a/fsspec_python/fs.py +++ b/fsspec_python/fs.py @@ -2,14 +2,15 @@ import inspect -from fsspec import AbstractFileSystem, filesystem +from fsspec import filesystem +from fsspec.implementations.chained import ChainedFileSystem from .importer import install_importer, uninstall_importer __all__ = ("PythonFileSystem",) -class PythonFileSystem(AbstractFileSystem): +class PythonFileSystem(ChainedFileSystem): """Python import filesystem""" def __init__(self, target_protocol=None, target_options=None, fs=None, **kwargs): @@ -29,62 +30,70 @@ def __init__(self, target_protocol=None, target_options=None, fs=None, **kwargs) self.target_protocol = ( target_protocol if isinstance(target_protocol, str) else (fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0]) ) + self.fs = fs if fs is not None else filesystem(target_protocol, **target_options) - if target_protocol and kwargs.get("fo"): - install_importer(f"{self.target_protocol}://{kwargs['fo']}", **target_options) + if kwargs.get("fo"): + self.registered_name = f"{self.target_protocol}://{kwargs['fo']}" + kwargs = {**target_options} else: - install_importer(self.fs, **target_options, **kwargs) + self.registered_name = f"{self.target_protocol}://" + kwargs = {**target_options, **kwargs} + install_importer(self.registered_name, **kwargs) - def close(self): - uninstall_importer(self.target_protocol) - self.fs.close() - super().close() + def exit(self): + uninstall_importer(self.registered_name) + if hasattr(self, "fs") and self.fs is not None and hasattr(self.fs, "exit"): + self.fs.exit() def __getattribute__(self, item): - if item in { - "__init__", - "__getattribute__", - "__reduce__", - "_make_local_details", - "open", - "cat", - "cat_file", - "_cat_file", - "cat_ranges", - "_cat_ranges", - "get", - "read_block", - "tail", - "head", - "info", - "ls", - "exists", - "isfile", - "isdir", - "_check_file", - "_check_cache", - "_mkcache", - "clear_cache", - "clear_expired_cache", - "pop_from_cache", - "local_file", - "_paths_from_path", - "get_mapper", - "open_many", - "commit_many", - "hash_name", - "__hash__", - "__eq__", - "to_json", - "to_dict", - "cache_size", - "pipe_file", - "pipe", - "start_transaction", - "end_transaction", + if item not in { + "__new__", + "fs", + # "__init__", + # "__getattribute__", + # "__reduce__", + # "_make_local_details", + # "open", + # "cat", + # "cat_file", + # "_cat_file", + # "cat_ranges", + # "_cat_ranges", + # "get", + # "read_block", + # "tail", + # "head", + # "info", + # "ls", + # "exists", + # "isfile", + # "isdir", + # "_check_file", + # "_check_cache", + # "_mkcache", + # "clear_cache", + # "clear_expired_cache", + # "pop_from_cache", + # "local_file", + # "_paths_from_path", + # "get_mapper", + # "open_many", + # "commit_many", + # "hash_name", + # "__hash__", + # "__eq__", + # "to_json", + # "to_dict", + # "cache_size", + # "pipe_file", + # "pipe", + # "start_transaction", + # "end_transaction", }: return object.__getattribute__(self, item) + + # Otherwise pull it out of dict d = object.__getattribute__(self, "__dict__") fs = d.get("fs", None) # fs is not immediately defined if item in d: diff --git a/fsspec_python/importer.py b/fsspec_python/importer.py index 724fbad..7447fbc 100644 --- a/fsspec_python/importer.py +++ b/fsspec_python/importer.py @@ -26,10 +26,13 @@ class FSSpecImportFinder(MetaPathFinder): def __init__(self, fsspec: str, **fsspec_args: str) -> None: self.fsspec_fs: AbstractFileSystem self.root: str + self.registered_name: str if isinstance(fsspec, AbstractFileSystem): self.fsspec_fs = fsspec self.root = fsspec_args.get("fo", fsspec.root_marker) + self.registered_name = f"{fsspec.protocol if isinstance(fsspec.protocol, str) else fsspec.protocol[0]}://{self.root}" else: + self.registered_name = fsspec self.fsspec_fs, self.root = url_to_fs(fsspec, **fsspec_args) self.remote_modules: dict[str, str] = {} @@ -83,7 +86,11 @@ def install_importer(fsspec: str, **fsspec_args: str) -> FSSpecImportFinder: """ if isinstance(fsspec, AbstractFileSystem): # Reassemble fsspec and args - fsspec = f"{fsspec.protocol if isinstance(fsspec.protocol, str) else fsspec.protocol[0]}://{fsspec.root_marker}" + if "fo" in fsspec_args: + # if fo is given, use that as root + fsspec = f"{fsspec.protocol if isinstance(fsspec.protocol, str) else fsspec.protocol[0]}://{fsspec_args['fo']}" + else: + fsspec = f"{fsspec.protocol if isinstance(fsspec.protocol, str) else fsspec.protocol[0]}://{fsspec.root_marker}" fsspec_args = fsspec_args or {} global _finders @@ -103,8 +110,10 @@ def uninstall_importer(fsspec: str = "") -> None: return fsspec = list(_finders.keys())[-1] if fsspec in _finders: - finder = _finders[fsspec] - del _finders[fsspec] - if finder in sys.meta_path: + finder = _finders.pop(fsspec, None) + if finder: finder.unload() + if finder in sys.meta_path: sys.meta_path.remove(finder) + else: + raise ValueError(f"No importer found for {fsspec}") diff --git a/fsspec_python/tests/conftest.py b/fsspec_python/tests/conftest.py index bdd0f23..68a79d1 100644 --- a/fsspec_python/tests/conftest.py +++ b/fsspec_python/tests/conftest.py @@ -1,39 +1,61 @@ import os +import sys from pathlib import Path import pytest -from fsspec import open +from fsspec import url_to_fs -from fsspec_python import install_importer, uninstall_importer +from fsspec_python import install_importer, install_open_hook, uninstall_importer, uninstall_open_hook @pytest.fixture() def s3_importer(): + sys_meta_path_length = len(sys.meta_path) if not os.environ.get("FSSPEC_S3_ENDPOINT_URL"): pytest.skip("S3 not configured") install_importer("s3://timkpaine-public/projects/fsspec-python") + if len(sys.meta_path) != sys_meta_path_length + 1: + # reset, some others get registered + sys_meta_path_length = len(sys.meta_path) - 1 + assert len(sys.meta_path) == sys_meta_path_length + 1 yield uninstall_importer() + assert len(sys.meta_path) == sys_meta_path_length @pytest.fixture() def local_importer(): + sys_meta_path_length = len(sys.meta_path) install_importer(f"file://{Path(__file__).parent}/local") yield uninstall_importer() + assert len(sys.meta_path) == sys_meta_path_length @pytest.fixture() -def open_hook(): - from fsspec_python import install_open_hook, uninstall_open_hook +def local_importer_multi(): + sys_meta_path_length = len(sys.meta_path) + install_importer(f"file://{Path(__file__).parent}/local") + install_importer(f"file://{Path(__file__).parent}/local2") + yield + uninstall_importer() + uninstall_importer() + assert len(sys.meta_path) == sys_meta_path_length + +@pytest.fixture() +def open_hook(): + sys_meta_path_length = len(sys.meta_path) install_open_hook(f"file://{Path(__file__).parent}/dump/") yield uninstall_open_hook() + assert len(sys.meta_path) == sys_meta_path_length @pytest.fixture() def fs_importer(): - fs = open(f"python::file://{Path(__file__).parent}/local2") + sys_meta_path_length = len(sys.meta_path) + fs, _ = url_to_fs(f"python::file://{Path(__file__).parent}/local2") yield fs - fs.close() + fs.exit() + assert len(sys.meta_path) == sys_meta_path_length diff --git a/fsspec_python/tests/test_fs.py b/fsspec_python/tests/test_fs.py index 52759c5..f83d2d8 100644 --- a/fsspec_python/tests/test_fs.py +++ b/fsspec_python/tests/test_fs.py @@ -1,8 +1,5 @@ class TestFs: def test_fs(self, fs_importer): - fs = fs_importer import my_local_file2 assert my_local_file2.baz() == "This is a local file." - - fs.close() diff --git a/fsspec_python/tests/test_remote_import.py b/fsspec_python/tests/test_remote_import.py index 10d8a6d..cc3f4e5 100644 --- a/fsspec_python/tests/test_remote_import.py +++ b/fsspec_python/tests/test_remote_import.py @@ -10,7 +10,15 @@ def test_importer_s3(self, s3_importer): assert my_remote_file.foo() == "This is a remote file." - def test_importer_local(self, local_importer): - import my_local_file + # def test_importer_local(self, local_importer): + # import my_local_file - assert my_local_file.bar() == "This is a local file." + # assert my_local_file.bar() == "This is a local file." + + # def test_importer_local_multi(self, local_importer_multi): + # assert len(fsspec_python.importer._finders) == 2 + # import my_local_file + # import my_local_file2 + + # assert my_local_file.bar() == "This is a local file." + # assert my_local_file2.baz() == "This is a local file." diff --git a/pyproject.toml b/pyproject.toml index 3767d34..9b9e827 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,8 @@ classifiers = [ ] dependencies = [ - "fsspec", + "fsspec@git+https://github.com/fsspec/filesystem_spec#egg=c23674c4c7bd60f8c3b41d0c4bbcefe4d18d6c70", + ] [project.entry-points."fsspec.specs"] @@ -95,6 +96,10 @@ exclude_also = [ ignore_errors = true fail_under = 50 +# TODO remove +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.build] artifacts = []