fix: implement cache touch in board, fix rsc cache

Author: machow

pins/__init__.py

@@ -11,3 +11,4 @@
 # Imports ----
 from .boards import BaseBoard
 from .constructors import *
+from .cache import PinsCache, PinsUrlCache

pins/boards.py

@@ -148,9 +148,14 @@ def pin_meta(self, name, version: str = None) -> Meta:
         components = [pin_name, selected_version.version]
         meta_name = self.meta_factory.get_meta_name(*components)
 
-        path_version = self.construct_path([*components, meta_name])
-        f = self.fs.open(path_version)
-        return self.meta_factory.read_pin_yaml(f, pin_name, selected_version)
+        path_meta = self.construct_path([*components, meta_name])
+        f = self.fs.open(path_meta)
+
+        meta = self.meta_factory.read_pin_yaml(f, pin_name, selected_version)
+
+        self._touch_cache(path_meta)
+
+        return meta
 
     def pin_list(self):
         """List names of all pins in a board.
@@ -561,6 +566,23 @@ def _extract_search_meta(self, meta):
         d["meta"] = meta
         return d
 
+    def _get_cache_path(self, pin_name, version):
+        p_version = self.construct_path([self.path_to_pin(pin_name), version])
+        hash = self.fs.hash_name(p_version, True)
+        return str(Path(self.fs.storage[-1]) / hash)
+
+    def _touch_cache(self, path):
+        from pins.cache import touch_access_time
+
+        # TODO: assumes same_name set to True. Let's require this be set to
+        # instantiate a pins cache.
+        if not hasattr(self.fs, "cached_files"):
+            return
+
+        hash = self.fs.hash_name(path, True)
+        path_to_hashed = Path(self.fs.storage[-1]) / hash
+        return touch_access_time(path_to_hashed)
+
 
 class BoardManual(BaseBoard):
     """Simple board that accepts a dictionary of form pin_name: path.
@@ -614,7 +636,9 @@ def pin_meta(self, name, version=None):
 
         path_meta = self.construct_path([pin_name, meta_name])
         f = self.fs.open(path_meta)
-        return self.meta_factory.read_pin_yaml(f, pin_name, VersionRaw(""))
+        meta = self.meta_factory.read_pin_yaml(f, pin_name, VersionRaw(""))
+
+        return meta
 
     def pin_download(self, name, version=None, hash=None) -> Sequence[str]:
         meta = self.pin_meta(name, version)

pins/cache.py

@@ -16,7 +16,7 @@
 PLACEHOLDER_FILE = "file"
 
 
-def touch_access_time(path, access_time: "float | None" = None):
+def touch_access_time(path, access_time: "float | None" = None, strict=True):
     """Update access time of file.
 
     Returns the new access time.
@@ -27,7 +27,7 @@ def touch_access_time(path, access_time: "float | None" = None):
 
     p = Path(path)
 
-    if not p.exists():
+    if not p.exists() and not strict:
         p.touch()
 
     stat = p.stat()
@@ -64,6 +64,7 @@ def _make_local_details(self, path):
         # note that this is called in ._open(), at the point it's known the file
         # will be cached
         fn = super()._make_local_details(path)
+        print(f"cache file: {fn}")
         Path(fn).parent.mkdir(parents=True, exist_ok=True)
 
         return fn
@@ -72,7 +73,7 @@ def hash_name(self, path, same_name):
         # the main change in this function is that, for same_name, it returns
         # the full path
         if same_name:
-            if self.hash_prefix:
+            if self.hash_prefix is not None:
                 # optionally make the name relative to a parent path
                 # using the hash of parent path as a prefix, to flatten a bit
                 suffix = Path(path).relative_to(Path(self.hash_prefix))
@@ -87,10 +88,37 @@ def hash_name(self, path, same_name):
 
             return path
         else:
-            return hash_name(path, same_name)
+            raise NotImplementedError()
 
-    def touch_access_time(path):
-        return touch_access_time(path)
+
+class PinsRscCache(PinsCache):
+    """Modifies the PinsCache to allow hash_prefix to be an RSC server url.
+
+    Note that this class also modifies the first / in a path to be a -, so that
+    pin contents will not be put into subdirectories, for e.g. michael/mtcars/data.txt.
+    """
+
+    protocol = "pinsrsccache"
+
+    def hash_name(self, path, same_name):
+        # the main change in this function is that, for same_name, it returns
+        # the full path
+        if same_name:
+            if self.hash_prefix is None:
+                raise NotImplementedError()
+
+            # change pin path of form <user>/<content> to <user>+<content>
+            suffix = path.replace("/", "+", 1)
+            prefix = hash_name(self.hash_prefix, False)
+
+            # TODO: hacky to automatically tack on protocol here
+            # but this is what R pins boards do. Could make a bool arg?
+            proto_name = protocol_to_string(self.fs.protocol)
+            full_prefix = "_".join([proto_name, prefix])
+            return str(full_prefix / Path(suffix))
+
+        else:
+            raise NotImplementedError()
 
 
 class PinsUrlCache(PinsCache):

pins/constructors.py

@@ -3,7 +3,7 @@
 import tempfile
 
 from .boards import BaseBoard, BoardRsConnect, BoardManual
-from .cache import PinsCache, PinsUrlCache
+from .cache import PinsCache, PinsUrlCache, PinsRscCache
 from .config import get_data_dir, get_cache_dir
 
 
@@ -58,6 +58,8 @@ def board(
     if storage_options is None:
         storage_options = {}
 
+    # TODO: at this point should just manually construct the rsc board directly
+    # from board_rsconnect...
     if protocol == "rsc":
         # TODO: register RsConnectFs with fsspec
         from pins.rsconnect.fs import RsConnectFs
@@ -71,9 +73,17 @@ def board(
 
     if cache is DEFAULT:
         cache_dir = get_cache_dir()
-        fs = PinsCache(
-            cache_storage=cache_dir, fs=fs, hash_prefix=path, same_names=True
-        )
+
+        # manually create a subdirectory for rsc server
+        if protocol == "rsc":
+            hash_prefix = storage_options["server_url"]
+            fs = PinsRscCache(
+                cache_storage=cache_dir, fs=fs, hash_prefix=hash_prefix, same_names=True
+            )
+        else:
+            fs = PinsCache(
+                cache_storage=cache_dir, fs=fs, hash_prefix=path, same_names=True
+            )
     elif cache is None:
         pass
     else:
@@ -272,7 +282,9 @@ def board_rsconnect(
         server_url = os.environ.get("CONNECT_SERVER")
 
     kwargs = dict(server_url=server_url, api_key=api_key)
-    return board("rsc", "", versioned, cache, allow_pickle_read, storage_options=kwargs)
+    return board(
+        "rsc", None, versioned, cache, allow_pickle_read, storage_options=kwargs
+    )
 
 
 def board_s3(path, versioned=True, cache=DEFAULT, allow_pickle_read=None):

pins/tests/test_boards.py

@@ -11,6 +11,7 @@
 
 from datetime import datetime, timedelta
 from time import sleep
+from pathlib import Path
 
 # using pytest cases, so that we can pass in fixtures as parameters
 # TODO: this seems like maybe overkill
@@ -288,6 +289,36 @@ def test_board_pin_search_name(board, df, search, matches):
     assert sorted_meta_names == sorted(matches)
 
 
+# BaseBoard specific ==========================================================
+
+from pins.boards import BaseBoard  # noqa
+from pins.cache import PinsCache  # noqa
+
+
+def test_board_base_pin_meta_cache_touch(tmp_dir2, df):
+
+    cache = fsspec.filesystem(
+        "pinscache", target_protocol="file", same_names=True, hash_prefix=str(tmp_dir2),
+    )
+    board = BaseBoard(str(tmp_dir2), fs=cache)
+
+    board.pin_write(df, "some_df", type="csv")
+    meta = board.pin_meta("some_df")
+    v = meta.version.version
+
+    p_cache_version = board._get_cache_path(meta.name, v)
+    p_cache_meta = Path(p_cache_version) / "data.txt"
+
+    orig_access = p_cache_meta.stat().st_atime
+
+    board.pin_meta("some_df")
+
+    new_access = p_cache_meta.stat().st_atime
+
+    assert orig_access < new_access
+    assert False
+
+
 # RStudio Connect specific ====================================================
 
 # import fixture that builds / tearsdown user "susan"

pins/tests/test_constructors.py

@@ -100,38 +100,62 @@ def board(backend):
 
 
 def test_constructor_board(board, df_csv, tmp_cache):
-    prot = board.fs.protocol
+    # TODO: would be nice to have fixtures for each board constructor
+    # doesn't need to copy over pins-compat content
+
+    # create board from constructor -------------------------------------------
 
+    prot = board.fs.protocol
     fs_name = prot if isinstance(prot, str) else prot[0]
 
     if fs_name == "file":
-        con_name = "folder"
+        board = c.board_folder(board.board)
     elif fs_name == "rsc":
-        con_name = "rsconnect"
-        pytest.xfail()
+        board = c.board_rsconnect(
+            server_url=board.fs.api.server_url, api_key=board.fs.api.api_key
+        )
     else:
-        con_name = fs_name
+        board = getattr(c, f"board_{fs_name}")(board.board)
+
+    # read a pin and check its contents ---------------------------------------
 
-    board = getattr(c, f"board_{con_name}")(board.board)
     df = board.pin_read("df_csv")
 
     # check data
     assert_frame_equal(df, df_csv)
 
+    # check the cache structure -----------------------------------------------
+
     # check cache
     if fs_name == "file":
         # no caching for local file boards
         pass
     else:
+        # check path structure ----
+
         options = list(tmp_cache.glob("*"))
         assert len(options) == 1
 
         cache_dir = options[0]
-        res = list(cache_dir.rglob("**/*.csv"))
+        res = list(cache_dir.rglob("*/*.csv"))
         assert len(res) == 1
 
         check_cache_file_path(res[0], cache_dir)
 
+        # check cache touch on access time ----
+
+        meta = board.pin_meta("df_csv")
+        p_cache_meta = (
+            Path(board._get_cache_path(meta.name, meta.version.version)) / "data.txt"
+        )
+        orig_access = p_cache_meta.stat().st_atime
+
+        board.pin_meta("df_csv")
+
+        new_access = p_cache_meta.stat().st_atime
+
+        assert orig_access < new_access
+
 
 # Board particulars ===========================================================