Merge pull request #1 from 1kbgz/tkp/init

Add support for union of python filesystems

Author: timkpaine

fsspec_union/fs.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from fsspec import filesystem
+from fsspec.implementations.chained import ChainedFileSystem
+
+__all__ = ("UnionFileSystem",)
+
+
+class UnionFileSystem(ChainedFileSystem):
+    """Union filesystem"""
+
+    def __init__(self, target_protocol=None, target_options=None, fs=None, **kwargs):
+        """
+        Args:
+            target_protocol: str (optional) Target filesystem protocol. Provide either this or ``fs``.
+            target_options: dict or None Passed to the instantiation of the FS, if fs is None.
+            fs: filesystem instance The target filesystem to run against. Provide this or ``protocol``.
+        """
+        super().__init__(**kwargs)
+        if fs is None and target_protocol is None:
+            raise ValueError("Please provide filesystem instance(fs) or target_protocol")
+        if not (fs is None) ^ (target_protocol is None):
+            raise ValueError("Both filesystems (fs) and target_protocol may not be both given.")
+
+        if target_protocol:
+            # unpack the targets and then instantiate in reverse order
+            fs_options = [{"target_protocol": target_protocol, "target_options": kwargs}]
+            fss = []
+
+            while "target_options" in target_options:
+                target_protocol = target_options.pop("target_protocol")
+                new_target_options = target_options.pop("target_options")
+                kwargs = target_options
+                fs_options.append({"target_protocol": target_protocol, "target_options": kwargs})
+                target_options = new_target_options
+
+            # instantiate in reverse order
+            for fspec in reversed(fs_options):
+                target_protocol = fspec["target_protocol"]
+                target_options = fspec["target_options"]
+                fss.append(filesystem(target_protocol, fs=fss[-1] if fss else None, **target_options))
+            fss.reverse()
+            self.fss = fss
+            self.fs = fss[0]
+        else:
+            self.fss = [fs]
+            self.fs = fs
+
+    def exit(self):
+        for fs in self.fss:
+            if hasattr(fs, "exit"):
+                fs.exit()

fsspec_union/tests/conftest.py

@@ -0,0 +1,32 @@
+import importlib
+import sys
+from pathlib import Path
+
+import pytest
+from fsspec import url_to_fs
+
+
+@pytest.fixture(scope="function")
+def fs_union_importer():
+    sys_meta_path_length = len(sys.meta_path)
+    fs, _ = url_to_fs(f"union::python::file://{Path(__file__).parent}/local1::python::file://{Path(__file__).parent}/local2")
+    import masked
+
+    importlib.reload(masked)
+    yield fs
+    fs.exit()
+    sys.modules.pop("masked", None)
+    assert len(sys.meta_path) == sys_meta_path_length
+
+
+@pytest.fixture(scope="function")
+def fs_union_importer_inverse():
+    sys_meta_path_length = len(sys.meta_path)
+    fs, _ = url_to_fs(f"union::python::file://{Path(__file__).parent}/local2::python::file://{Path(__file__).parent}/local1")
+    import masked
+
+    importlib.reload(masked)
+    yield fs
+    fs.exit()
+    sys.modules.pop("masked", None)
+    assert len(sys.meta_path) == sys_meta_path_length

fsspec_union/tests/local1/masked.py

@@ -0,0 +1,2 @@
+def masked():
+    return 1

fsspec_union/tests/local1/my_local_file1.py

@@ -0,0 +1,2 @@
+def foo1():
+    return "This is a local file 1."

fsspec_union/tests/local2/masked.py

@@ -0,0 +1,2 @@
+def masked():
+    return 2

fsspec_union/tests/local2/my_local_file2.py

@@ -0,0 +1,2 @@
+def foo2():
+    return "This is a local file 2."

fsspec_union/tests/test_fs.py

@@ -0,0 +1,22 @@
+class TestFs:
+    def test_fs_union(self, fs_union_importer):
+        import my_local_file1
+        import my_local_file2
+
+        assert my_local_file1.foo1() == "This is a local file 1."
+        assert my_local_file2.foo2() == "This is a local file 2."
+
+        import masked
+
+        assert masked.masked() == 1
+
+    def test_fs_union_inverse(self, fs_union_importer_inverse):
+        import my_local_file1
+        import my_local_file2
+
+        assert my_local_file1.foo1() == "This is a local file 1."
+        assert my_local_file2.foo2() == "This is a local file 2."
+
+        import masked
+
+        assert masked.masked() == 2

pyproject.toml

@@ -26,7 +26,13 @@ classifiers = [
     "Programming Language :: Python :: 3.14",
 ]
 
-dependencies = []
+dependencies = [
+    "fsspec@git+https://github.com/fsspec/filesystem_spec#egg=c23674c4c7bd60f8c3b41d0c4bbcefe4d18d6c70",
+    "fsspec-python@git+https://github.com/1kbgz/fsspec-python#egg=9dfe6ecc5410ef871083f2b0d0cbb008acdb7f4d",
+]
+
+[project.entry-points."fsspec.specs"]
+union = "fsspec_union.fs.UnionFileSystem"
 
 [project.optional-dependencies]
 develop = [
@@ -88,6 +94,10 @@ exclude_also = [
 ignore_errors = true
 fail_under = 50
 
+# TODO remove
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [tool.hatch.build]
 artifacts = []