Support correct path when instantiate via chaining

fsspec_python/fs.py

@@ -2,14 +2,15 @@
 
 import inspect
 
-from fsspec import AbstractFileSystem, filesystem
+from fsspec import filesystem
+from fsspec.implementations.chained import ChainedFileSystem
 
 from .importer import install_importer, uninstall_importer
 
 __all__ = ("PythonFileSystem",)
 
 
-class PythonFileSystem(AbstractFileSystem):
+class PythonFileSystem(ChainedFileSystem):
     """Python import filesystem"""
 
     def __init__(self, target_protocol=None, target_options=None, fs=None, **kwargs):
@@ -29,62 +30,70 @@ def __init__(self, target_protocol=None, target_options=None, fs=None, **kwargs)
         self.target_protocol = (
             target_protocol if isinstance(target_protocol, str) else (fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0])
         )
+
         self.fs = fs if fs is not None else filesystem(target_protocol, **target_options)
 
-        if target_protocol and kwargs.get("fo"):
-            install_importer(f"{self.target_protocol}://{kwargs['fo']}", **target_options)
+        if kwargs.get("fo"):
+            self.registered_name = f"{self.target_protocol}://{kwargs['fo']}"
+            kwargs = {**target_options}
         else:
-            install_importer(self.fs, **target_options, **kwargs)
+            self.registered_name = f"{self.target_protocol}://"
+            kwargs = {**target_options, **kwargs}
+        install_importer(self.registered_name, **kwargs)
 
-    def close(self):
-        uninstall_importer(self.target_protocol)
-        self.fs.close()
-        super().close()
+    def exit(self):
+        uninstall_importer(self.registered_name)
+        if hasattr(self, "fs") and self.fs is not None and hasattr(self.fs, "exit"):
+            self.fs.exit()
 
     def __getattribute__(self, item):
-        if item in {
-            "__init__",
-            "__getattribute__",
-            "__reduce__",
-            "_make_local_details",
-            "open",
-            "cat",
-            "cat_file",
-            "_cat_file",
-            "cat_ranges",
-            "_cat_ranges",
-            "get",
-            "read_block",
-            "tail",
-            "head",
-            "info",
-            "ls",
-            "exists",
-            "isfile",
-            "isdir",
-            "_check_file",
-            "_check_cache",
-            "_mkcache",
-            "clear_cache",
-            "clear_expired_cache",
-            "pop_from_cache",
-            "local_file",
-            "_paths_from_path",
-            "get_mapper",
-            "open_many",
-            "commit_many",
-            "hash_name",
-            "__hash__",
-            "__eq__",
-            "to_json",
-            "to_dict",
-            "cache_size",
-            "pipe_file",
-            "pipe",
-            "start_transaction",
-            "end_transaction",
+        if item not in {
+            "__new__",
+            "fs",
+            # "__init__",
+            # "__getattribute__",
+            # "__reduce__",
+            # "_make_local_details",
+            # "open",
+            # "cat",
+            # "cat_file",
+            # "_cat_file",
+            # "cat_ranges",
+            # "_cat_ranges",
+            # "get",
+            # "read_block",
+            # "tail",
+            # "head",
+            # "info",
+            # "ls",
+            # "exists",
+            # "isfile",
+            # "isdir",
+            # "_check_file",
+            # "_check_cache",
+            # "_mkcache",
+            # "clear_cache",
+            # "clear_expired_cache",
+            # "pop_from_cache",
+            # "local_file",
+            # "_paths_from_path",
+            # "get_mapper",
+            # "open_many",
+            # "commit_many",
+            # "hash_name",
+            # "__hash__",
+            # "__eq__",
+            # "to_json",
+            # "to_dict",
+            # "cache_size",
+            # "pipe_file",
+            # "pipe",
+            # "start_transaction",
+            # "end_transaction",
         }:
             return object.__getattribute__(self, item)
+
+        # Otherwise pull it out of dict
         d = object.__getattribute__(self, "__dict__")
         fs = d.get("fs", None)  # fs is not immediately defined
         if item in d:

fsspec_python/importer.py

@@ -26,10 +26,13 @@ class FSSpecImportFinder(MetaPathFinder):
     def __init__(self, fsspec: str, **fsspec_args: str) -> None:
         self.fsspec_fs: AbstractFileSystem
         self.root: str
+        self.registered_name: str
         if isinstance(fsspec, AbstractFileSystem):
             self.fsspec_fs = fsspec
             self.root = fsspec_args.get("fo", fsspec.root_marker)
+            self.registered_name = f"{fsspec.protocol if isinstance(fsspec.protocol, str) else fsspec.protocol[0]}://{self.root}"
         else:
+            self.registered_name = fsspec
             self.fsspec_fs, self.root = url_to_fs(fsspec, **fsspec_args)
         self.remote_modules: dict[str, str] = {}
 
@@ -83,7 +86,11 @@ def install_importer(fsspec: str, **fsspec_args: str) -> FSSpecImportFinder:
     """
     if isinstance(fsspec, AbstractFileSystem):
         # Reassemble fsspec and args
-        fsspec = f"{fsspec.protocol if isinstance(fsspec.protocol, str) else fsspec.protocol[0]}://{fsspec.root_marker}"
+        if "fo" in fsspec_args:
+            # if fo is given, use that as root
+            fsspec = f"{fsspec.protocol if isinstance(fsspec.protocol, str) else fsspec.protocol[0]}://{fsspec_args['fo']}"
+        else:
+            fsspec = f"{fsspec.protocol if isinstance(fsspec.protocol, str) else fsspec.protocol[0]}://{fsspec.root_marker}"
         fsspec_args = fsspec_args or {}
 
     global _finders
@@ -103,8 +110,10 @@ def uninstall_importer(fsspec: str = "") -> None:
             return
         fsspec = list(_finders.keys())[-1]
     if fsspec in _finders:
-        finder = _finders[fsspec]
-        del _finders[fsspec]
-        if finder in sys.meta_path:
+        finder = _finders.pop(fsspec, None)
+        if finder:
             finder.unload()
+        if finder in sys.meta_path:
             sys.meta_path.remove(finder)
+    else:
+        raise ValueError(f"No importer found for {fsspec}")

fsspec_python/tests/conftest.py

@@ -1,39 +1,61 @@
 import os
+import sys
 from pathlib import Path
 
 import pytest
-from fsspec import open
+from fsspec import url_to_fs
 
-from fsspec_python import install_importer, uninstall_importer
+from fsspec_python import install_importer, install_open_hook, uninstall_importer, uninstall_open_hook
 
 
 @pytest.fixture()
 def s3_importer():
+    sys_meta_path_length = len(sys.meta_path)
     if not os.environ.get("FSSPEC_S3_ENDPOINT_URL"):
         pytest.skip("S3 not configured")
     install_importer("s3://timkpaine-public/projects/fsspec-python")
+    if len(sys.meta_path) != sys_meta_path_length + 1:
+        # reset, some others get registered
+        sys_meta_path_length = len(sys.meta_path) - 1
+    assert len(sys.meta_path) == sys_meta_path_length + 1
     yield
     uninstall_importer()
+    assert len(sys.meta_path) == sys_meta_path_length
 
 
 @pytest.fixture()
 def local_importer():
+    sys_meta_path_length = len(sys.meta_path)
     install_importer(f"file://{Path(__file__).parent}/local")
     yield
     uninstall_importer()
+    assert len(sys.meta_path) == sys_meta_path_length
 
 
 @pytest.fixture()
-def open_hook():
-    from fsspec_python import install_open_hook, uninstall_open_hook
+def local_importer_multi():
+    sys_meta_path_length = len(sys.meta_path)
+    install_importer(f"file://{Path(__file__).parent}/local")
+    install_importer(f"file://{Path(__file__).parent}/local2")
+    yield
+    uninstall_importer()
+    uninstall_importer()
+    assert len(sys.meta_path) == sys_meta_path_length
+
 
+@pytest.fixture()
+def open_hook():
+    sys_meta_path_length = len(sys.meta_path)
     install_open_hook(f"file://{Path(__file__).parent}/dump/")
     yield
     uninstall_open_hook()
+    assert len(sys.meta_path) == sys_meta_path_length
 
 
 @pytest.fixture()
 def fs_importer():
-    fs = open(f"python::file://{Path(__file__).parent}/local2")
+    sys_meta_path_length = len(sys.meta_path)
+    fs, _ = url_to_fs(f"python::file://{Path(__file__).parent}/local2")
     yield fs
-    fs.close()
+    fs.exit()
+    assert len(sys.meta_path) == sys_meta_path_length

fsspec_python/tests/test_fs.py

@@ -1,8 +1,5 @@
 class TestFs:
     def test_fs(self, fs_importer):
-        fs = fs_importer
         import my_local_file2
 
         assert my_local_file2.baz() == "This is a local file."
-
-        fs.close()

fsspec_python/tests/test_remote_import.py

@@ -10,7 +10,15 @@ def test_importer_s3(self, s3_importer):
 
         assert my_remote_file.foo() == "This is a remote file."
 
-    def test_importer_local(self, local_importer):
-        import my_local_file
+    # def test_importer_local(self, local_importer):
+    #     import my_local_file
 
-        assert my_local_file.bar() == "This is a local file."
+    #     assert my_local_file.bar() == "This is a local file."
+
+    # def test_importer_local_multi(self, local_importer_multi):
+    #     assert len(fsspec_python.importer._finders) == 2
+    #     import my_local_file
+    #     import my_local_file2
+
+    #     assert my_local_file.bar() == "This is a local file."
+    #     assert my_local_file2.baz() == "This is a local file."

pyproject.toml

@@ -27,7 +27,8 @@ classifiers = [
 ]
 
 dependencies = [
-    "fsspec",
+    "fsspec@git+https://github.com/fsspec/filesystem_spec#egg=c23674c4c7bd60f8c3b41d0c4bbcefe4d18d6c70",
+
 ]
 
 [project.entry-points."fsspec.specs"]
@@ -95,6 +96,10 @@ exclude_also = [
 ignore_errors = true
 fail_under = 50
 
+# TODO remove
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [tool.hatch.build]
 artifacts = []