Merge pull request #53 from machow/feat-cache

feat: basic path preserving cache

Author: machow

.env.dev

@@ -1,3 +1,7 @@
+# Pins optional config ----
+#PINS_CACHE_DIR=.pins_cache
+#PINS_DATA_DIR=.pins_data
+
 # AWS S3 backend ----
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=

docs/getting_started.Rmd

@@ -250,3 +250,7 @@ You can read this data by combining `pin_download()` with `read.csv()`[^1]:
 
 # pd.read_csv(fname)
 ```
+
+```{python}
+my_data.pin_download("penguins")
+```

pins/boards.py

@@ -12,7 +12,7 @@
 from typing import Protocol, Sequence, Optional, Mapping
 
 from .versions import VersionRaw, guess_version
-from .meta import Meta, MetaFactory
+from .meta import Meta, MetaRaw, MetaFactory
 from .errors import PinsError
 from .drivers import load_data, save_data, default_title
 
@@ -45,9 +45,13 @@ def info(self, path):
 
 class BaseBoard:
     def __init__(
-        self, board: str, fs: IFileSystem, versioned=True, meta_factory=MetaFactory(),
+        self,
+        board: "str | Path",
+        fs: IFileSystem,
+        versioned=True,
+        meta_factory=MetaFactory(),
     ):
-        self.board = board
+        self.board = str(board)
         self.fs = fs
         self.meta_factory = meta_factory
 
@@ -185,6 +189,12 @@ def pin_read(self, name, version: Optional[str] = None, hash: Optional[str] = No
         """
         meta = self.pin_fetch(name, version)
 
+        if isinstance(meta, MetaRaw):
+            raise TypeError(
+                "Could not find metadata for this pin version. If this is an individual "
+                "file, may need to use pin_download()."
+            )
+
         if hash is not None:
             raise NotImplementedError("TODO: validate hash")
 
@@ -577,16 +587,41 @@ def pin_meta(self, name, version=None):
         pin_name = self.path_to_pin(name)
         meta_name = self.meta_factory.get_meta_name()
 
+        # special case where we are using http protocol to fetch what may be
+        # a file. here we need to create a stripped down form of metadata, since
+        # a metadata file does not exist (and we can't pull files from a version dir).
+        path_to_pin = self.construct_path([pin_name])
+        if self.fs.protocol == "http" and not path_to_pin.rstrip().endswith("/"):
+            # create metadata, rather than read from a file
+            return self.meta_factory.create_raw(path_to_pin, type="file",)
+
         path_meta = self.construct_path([pin_name, meta_name])
         f = self.fs.open(path_meta)
         return self.meta_factory.read_pin_yaml(f, pin_name, VersionRaw(""))
 
+    def pin_download(self, name, version=None, hash=None) -> Sequence[str]:
+        meta = self.pin_meta(name, version)
+
+        if isinstance(meta, MetaRaw):
+            return load_data(meta, self.fs, None)
+
+        raise NotImplementedError("TODO: allow download beyond MetaRaw.")
+
     def construct_path(self, elements):
         # TODO: in practice every call to construct_path has the first element of
         # pin name. to make this safer, we should enforce that in its signature.
         pin_name, *others = elements
         pin_path = self.pin_paths[pin_name]
 
+        if self.board.strip() == "":
+            return pin_path
+
+        if len(others):
+            # this is confusing, but R pins url board has a final "/" indicate that
+            # something is a pin version, rather than a single file. but since other
+            # boards forbid a final /, we need to strip it off to join elements
+            pin_path = pin_path.rstrip().rstrip("/")
+
         return super().construct_path([pin_path, *others])
 
 
@@ -626,7 +661,7 @@ def pin_search(self, search=None, as_df=True):
                 # verify code is for inadequate permission to access
                 if e.args[0]["code"] != 19:
                     raise e
-                # TODO(question): should this be a RawMeta class or something?
+                # TODO(question): should this be a MetaRaw class or something?
                 #                 that fixes our isinstance Meta below.
                 # TODO(compatibility): R pins errors instead, see #27
                 res.append({"name": pin_name, "meta": None})

pins/cache.py

@@ -0,0 +1,232 @@
+import humanize
+import os
+import time
+import shutil
+import urllib.parse
+
+from fsspec.implementations.cached import SimpleCacheFileSystem, hash_name
+from fsspec import register_implementation
+from pathlib import Path
+
+from .config import get_cache_dir
+
+
+# used if needed to preserve board path structure in the cache
+PLACEHOLDER_VERSION = "v"
+PLACEHOLDER_FILE = "file"
+
+
+def touch_access_time(path, access_time: "float | None" = None):
+    """Update access time of file.
+
+    Returns the new access time.
+    """
+
+    if access_time is None:
+        access_time = time.time()
+
+    p = Path(path)
+
+    if not p.exists():
+        p.touch()
+
+    stat = p.stat()
+    os.utime(path, (access_time, stat.st_mtime))
+
+    return access_time
+
+
+def protocol_to_string(protocol):
+    if isinstance(protocol, str):
+        return protocol
+
+    return protocol[0]
+
+
+class PinsCache(SimpleCacheFileSystem):
+    protocol = "pinscache"
+
+    def __init__(self, *args, hash_prefix=None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.hash_prefix = hash_prefix
+
+    def _open(self, path, *args, **kwargs):
+        # For some reason, the open method of SimpleCacheFileSystem doesn't
+        # call _make_local_details, so we need to patch in here.
+        # Note that methods like .cat() do call it. Other Caches don't have this issue.
+        path = self._strip_protocol(path)
+        self._make_local_details(path)
+
+        return super()._open(path, *args, **kwargs)
+
+    def _make_local_details(self, path):
+        # modifies method to create any parent directories needed by the cached file
+        # note that this is called in ._open(), at the point it's known the file
+        # will be cached
+        fn = super()._make_local_details(path)
+        Path(fn).parent.mkdir(parents=True, exist_ok=True)
+
+        return fn
+
+    def hash_name(self, path, same_name):
+        # the main change in this function is that, for same_name, it returns
+        # the full path
+        if same_name:
+            if self.hash_prefix:
+                # optionally make the name relative to a parent path
+                # using the hash of parent path as a prefix, to flatten a bit
+                suffix = Path(path).relative_to(Path(self.hash_prefix))
+                # TODO(compat): R pins uses xxh128 hash here, but fsspec uses sha256
+                prefix = hash_name(self.hash_prefix, False)
+
+                # TODO: hacky to automatically tack on protocol here
+                # but this is what R pins boards do. Could make a bool arg?
+                proto_name = protocol_to_string(self.fs.protocol)
+                full_prefix = "_".join([proto_name, prefix])
+                return str(full_prefix / suffix)
+
+            return path
+        else:
+            return hash_name(path, same_name)
+
+    def touch_access_time(path):
+        return touch_access_time(path)
+
+
+class PinsUrlCache(PinsCache):
+    protocol = "pinsurlcache"
+
+    def hash_name(self, path, same_name):
+        # strip final arg from path
+        # note that R pins uses fs::path_file, and I'm not sure exactly how it
+        # behaves for the many forms url paths can take.
+        # e.g. path_file(.../extdata/) -> extdata
+        # e.g. path_file(.../extdata?123) -> extdata?123
+        path_parts = urllib.parse.urlparse(path)[2]
+
+        # strip off final whitespace and / (if it exists)
+        # TODO(compat): python pins currently not keeping query part of url
+        final_part = path_parts.rstrip().rstrip("/").rsplit("/", 1)[-1]
+
+        # TODO: what happens in R pins if no final part?
+        if final_part == "":
+            final_part = PLACEHOLDER_FILE
+
+        # hash url
+        prefix = hash_name(path, False)
+
+        # note that we include an extra version folder, so it conforms with
+        # pin board path form: <board_path>/<pin_name>/<version_name>/<file>
+        proto_name = protocol_to_string(self.fs.protocol)
+        full_prefix = "_".join([proto_name, prefix])
+        return str(Path(full_prefix) / PLACEHOLDER_VERSION / final_part)
+
+
+class CachePruner:
+    """Prunes the cache directory, across multiple boards.
+
+    Note
+    ----
+
+    `pins` assumes that all boards cache using these rules:
+
+    * path structure: `<cache_root>/<board_hash>/<pin>/<version>`.
+    * each version has a data.txt file in it.
+    """
+
+    meta_path = "data.txt"
+
+    def __init__(self, cache_dir: "str | Path"):
+        self.cache_dir = Path(cache_dir)
+
+    def versions(self) -> "iter[Path]":
+        for p_version in self.cache_dir.glob("*/*"):
+            if p_version.is_dir() and (p_version / self.meta_path).exists():
+                yield p_version
+
+    def should_prune_version(self, days, path: "str | Path"):
+        path = Path(path)
+
+        expiry_time_sec = days * 60 * 60 * 24
+        prune_before = time.time() - expiry_time_sec
+
+        p_meta = path / self.meta_path
+
+        if not p_meta.exists():
+            raise FileNotFoundError(f"No metadata file: {p_meta.absolute()}")
+
+        access_time = p_meta.stat().st_atime
+        return access_time < prune_before
+
+    def old_versions(self, days):
+        return [p for p in self.versions() if self.should_prune_version(days, p)]
+
+    def prune(self, days=30):
+        to_prune = self.old_versions(days)
+        size = sum(map(disk_usage, to_prune))
+
+        # TODO: clean this up, general approach to prompting
+        confirmed = prompt_cache_prune(to_prune, size)
+        if confirmed:
+            for path in to_prune:
+                delete_version(to_prune)
+
+        print("Skipping cache deletion")
+
+
+def delete_version(path: "str | Path"):
+    path = Path(path)
+    shutil.rmtree(str(path.absolute()))
+
+
+def disk_usage(path):
+    return sum(p.stat().st_size for p in path.glob("**/*") if p.is_file() or p.is_dir())
+
+
+def prompt_cache_prune(to_prune, size) -> bool:
+    print(to_prune)
+    human_size = humanize.naturalsize(size, binary=True)
+    resp = input(f"Delete {len(to_prune)} pin versions, freeing {human_size}?")
+    return resp == "yes"
+
+
+def cache_prune(days=30, cache_root=None, prompt=True):
+    if cache_root is None:
+        cache_root = get_cache_dir()
+
+    final_delete = []
+    for p_board in Path(cache_root).glob("*"):
+        pruner = CachePruner(p_board)
+        final_delete.extend(pruner.old_versions(days))
+
+    size = sum(map(disk_usage, final_delete))
+
+    if prompt:
+        confirmed = prompt_cache_prune(final_delete, size)
+    else:
+        confirmed = True
+    if confirmed:
+        for p in final_delete:
+            delete_version(p)
+
+
+# def prune_files(days = 30, path = None):
+#     if path is None:
+#         for p_cache in Path(get_cache_dir()).glob("*"):
+#             return prune_files(days=days, path=str(p_cache.absolute()))
+#
+#     expiry_time_sec = days * 60 * 60 * 24
+#     fs_cache = PinsCache(
+#         target_protocol=None,
+#         cache_storage=path,
+#         check_files=True,
+#         expiry_time=expiry_time_sec
+#     )
+#
+#     # note that fsspec considers only the last entry in cached_files deletable
+#     for hash_path, entry in fs_cache.cached_files[-1].items():
+#         if time.time() - detail["time"] > self.expiry:
+#             fs_cache.pop_from_cache(entry["original"])
+
+# TODO: swap to use entrypoint
+register_implementation("pinscache", PinsCache)

pins/config.py

@@ -0,0 +1,12 @@
+import appdirs
+import os
+
+PINS_NAME = "pins-py"
+
+
+def get_data_dir():
+    return os.environ.get("PINS_DATA_DIR", appdirs.user_data_dir(PINS_NAME))
+
+
+def get_cache_dir():
+    return os.environ.get("PINS_CACHE_DIR", appdirs.user_cache_dir(PINS_NAME))