Apply lru_cache manually per LazyReferenceMapper instance (#1985)

Author: honzaflash

fsspec/implementations/reference.py

@@ -157,6 +157,13 @@ def __init__(
         if self.engine == "pyarrow" and find_spec("pyarrow") is None:
             raise ImportError("engine choice `pyarrow` is not installed.")
 
+        # Apply `lru_cache` decorator manually per instance.
+        # This way `self` reference is not held on class level.
+        # WARNING: However, this means that self and its members are not reflected
+        # in the cache key, so we expect they won't be mutated once a value is cached.
+        self.listdir = lru_cache()(self.listdir)
+        self._key_to_record = lru_cache(maxsize=4096)(self._key_to_record)
+
     def __getattr__(self, item):
         if item in ("_items", "record_size", "zmetadata"):
             self.setup()
@@ -219,7 +226,6 @@ def create(root, storage_options=None, fs=None, record_size=10000, **kwargs):
         fs.pipe("/".join([root, ".zmetadata"]), json.dumps(met).encode())
         return LazyReferenceMapper(root, fs, **kwargs)
 
-    @lru_cache
     def listdir(self):
         """List top-level directories"""
         dirs = (p.rsplit("/", 1)[0] for p in self.zmetadata if not p.startswith(".z"))
@@ -331,7 +337,6 @@ def _load_one_key(self, key):
         # URL, offset, size
         return selection[:3]
 
-    @lru_cache(4096)
     def _key_to_record(self, key):
         """Details needed to construct a reference for one key"""
         field, chunk = key.rsplit("/", 1)

pyproject.toml

@@ -189,8 +189,6 @@ select = [
 ignore = [
     # Loop control variable `loop` not used within loop body
     "B007",
-    # Use of `functools.lru_cache` or `functools.cache` on methods can lead to memory leaks
-    "B019",
     # Star-arg unpacking after a keyword argument is strongly discouraged
     "B026",
     # No explicit `stacklevel` keyword argument found