localmemory: init

fsspec/implementations/localmemory.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import logging
+
+from fsspec.implementations.memory import MemoryFileSystem
+
+logger = logging.getLogger("fsspec.localmemoryfs")
+
+
+class LocalMemoryFileSystem(MemoryFileSystem):
+    """A filesystem based on a dict of BytesIO objects
+
+    This is a local filesystem so different instances of this class
+    point to different memory filesystems.
+    """
+
+    pseudo_dirs = None
+    protocol = "localmemory"
+    root_marker = "/"
+    _intrans = False
+    cachable = False  # same as: skip_instance_cache = True
+
+    def __init__(self, *args, **kwargs):
+        self.logger = logger  # global
+        self.store = {}  # local
+        self.pseudo_dirs = [""]  # local
+        super().__init__(*args, **kwargs)

fsspec/implementations/memory.py

@@ -25,6 +25,11 @@ class MemoryFileSystem(AbstractFileSystem):
     pseudo_dirs = [""]  # global, do not overwrite!
     protocol = "memory"
     root_marker = "/"
+    _intrans = False
+
+    def __init__(self, *args, **kwargs):
+        self.logger = logger
+        super().__init__(*args, **kwargs)
 
     @classmethod
     def _strip_protocol(cls, path):
@@ -147,7 +152,7 @@ def rmdir(self, path):
             raise FileNotFoundError(path)
 
     def info(self, path, **kwargs):
-        logger.debug("info: %s", path)
+        self.logger.debug("info: %s", path)
         path = self._strip_protocol(path)
         if path in self.pseudo_dirs or any(
             p.startswith(path + "/") for p in list(self.store) + self.pseudo_dirs
@@ -202,7 +207,7 @@ def _open(
         elif mode in {"wb", "xb"}:
             if mode == "xb" and self.exists(path):
                 raise FileExistsError
-            m = MemoryFile(self, path, kwargs.get("data"))
+            m = MemoryFile(self, path, kwargs.get("data"), self.logger)
             if not self._intrans:
                 m.commit()
             return m
@@ -215,7 +220,7 @@ def cp_file(self, path1, path2, **kwargs):
         path2 = self._strip_protocol(path2)
         if self.isfile(path1):
             self.store[path2] = MemoryFile(
-                self, path2, self.store[path1].getvalue()
+                self, path2, self.store[path1].getvalue(), self.logger
             )  # implicit copy
         elif self.isdir(path1):
             if path2 not in self.pseudo_dirs:
@@ -224,7 +229,7 @@ def cp_file(self, path1, path2, **kwargs):
             raise FileNotFoundError(path1)
 
     def cat_file(self, path, start=None, end=None, **kwargs):
-        logger.debug("cat: %s", path)
+        self.logger.debug("cat: %s", path)
         path = self._strip_protocol(path)
         try:
             return bytes(self.store[path].getbuffer()[start:end])
@@ -283,8 +288,9 @@ class MemoryFile(BytesIO):
     No need to provide fs, path if auto-committing (default)
     """
 
-    def __init__(self, fs=None, path=None, data=None):
-        logger.debug("open file %s", path)
+    def __init__(self, fs=None, path=None, data=None, logger=logger):
+        self.logger = logger
+        self.logger.debug("open file %s", path)
         self.fs = fs
         self.path = path
         self.created = datetime.now(tz=timezone.utc)

fsspec/implementations/tests/test_localmemory.py

@@ -0,0 +1,22 @@
+import fsspec
+from fsspec.implementations.localmemory import LocalMemoryFileSystem
+
+
+def test_protocol():
+    # this should not throw: ValueError: Protocol not known: localmemory
+    fsspec.filesystem("localmemory")
+
+
+def test_init():
+    fs1 = LocalMemoryFileSystem()
+    fs2 = LocalMemoryFileSystem()
+
+    # check that fs1 and fs2 are different instances of LocalMemoryFileSystem
+    assert id(fs1) != id(fs2)
+    assert id(fs1.store) != id(fs2.store)
+    assert id(fs1.pseudo_dirs) != id(fs2.pseudo_dirs)
+
+    fs1.touch("/fs1.txt")
+    fs2.touch("/fs2.txt")
+    assert fs1.ls("/", detail=False) == ["/fs1.txt"]
+    assert fs2.ls("/", detail=False) == ["/fs2.txt"]

fsspec/registry.py

@@ -171,6 +171,9 @@ def register_implementation(name, cls, clobber=False, errtxt=None):
         "err": "LibArchive requires to be installed",
     },
     "local": {"class": "fsspec.implementations.local.LocalFileSystem"},
+    "localmemory": {
+        "class": "fsspec.implementations.localmemory.LocalMemoryFileSystem"
+    },
     "memory": {"class": "fsspec.implementations.memory.MemoryFileSystem"},
     "oci": {
         "class": "ocifs.OCIFileSystem",