Merge pull request #78 from machow/feat-cache-touch

Feat cache touch

Author: machow

pins/__init__.py

@@ -11,3 +11,4 @@
 # Imports ----
 from .boards import BaseBoard
 from .constructors import *
+from .cache import PinsCache, PinsUrlCache, cache_prune

pins/boards.py

@@ -1,3 +1,4 @@
+import logging
 import tempfile
 import shutil
 import inspect
@@ -148,9 +149,14 @@ def pin_meta(self, name, version: str = None) -> Meta:
         components = [pin_name, selected_version.version]
         meta_name = self.meta_factory.get_meta_name(*components)
 
-        path_version = self.construct_path([*components, meta_name])
-        f = self.fs.open(path_version)
-        return self.meta_factory.read_pin_yaml(f, pin_name, selected_version)
+        path_meta = self.construct_path([*components, meta_name])
+        f = self.fs.open(path_meta)
+
+        meta = self.meta_factory.read_pin_yaml(f, pin_name, selected_version)
+
+        self._touch_cache(path_meta)
+
+        return meta
 
     def pin_list(self):
         """List names of all pins in a board.
@@ -385,9 +391,9 @@ def pin_versions_prune(
         # TODO(question): how to pin_inform? Log or warning?
         if to_delete:
             str_vers = ", ".join([v.version for v in to_delete])
-            print(f"Deleting versions: {str_vers}.")
+            logging.info(f"Deleting versions: {str_vers}.")
         if not to_delete:
-            print("No old versions to delete")
+            logging.info("No old versions to delete")
 
         for version in to_delete:
             self.pin_version_delete(name, version.version)
@@ -561,6 +567,23 @@ def _extract_search_meta(self, meta):
         d["meta"] = meta
         return d
 
+    def _get_cache_path(self, pin_name, version):
+        p_version = self.construct_path([self.path_to_pin(pin_name), version])
+        hash = self.fs.hash_name(p_version, True)
+        return str(Path(self.fs.storage[-1]) / hash)
+
+    def _touch_cache(self, path):
+        from pins.cache import touch_access_time
+
+        # TODO: assumes same_name set to True. Let's require this be set to
+        # instantiate a pins cache.
+        if not hasattr(self.fs, "cached_files"):
+            return
+
+        hash = self.fs.hash_name(path, True)
+        path_to_hashed = Path(self.fs.storage[-1]) / hash
+        return touch_access_time(path_to_hashed)
+
 
 class BoardManual(BaseBoard):
     """Simple board that accepts a dictionary of form pin_name: path.
@@ -614,7 +637,9 @@ def pin_meta(self, name, version=None):
 
         path_meta = self.construct_path([pin_name, meta_name])
         f = self.fs.open(path_meta)
-        return self.meta_factory.read_pin_yaml(f, pin_name, VersionRaw(""))
+        meta = self.meta_factory.read_pin_yaml(f, pin_name, VersionRaw(""))
+
+        return meta
 
     def pin_download(self, name, version=None, hash=None) -> Sequence[str]:
         meta = self.pin_meta(name, version)
@@ -692,12 +717,13 @@ def pin_search(self, search=None, as_df=True):
 
         paged_res = self.fs.api.misc_get_applications("content_type:pin", search=search)
         results = paged_res.results
-        names = [f"{cont['owner_username']}/{cont['name']}" for cont in results]
 
         res = []
-        for pin_name in names:
+        for content in results:
+            pin_name = f"{content['owner_username']}/{content['name']}"
+            version = str(content["bundle_id"])
             try:
-                meta = self.pin_meta(pin_name)
+                meta = self.pin_meta(pin_name, version)
                 res.append(meta)
 
             except RsConnectApiRequestError as e:

pins/cache.py

@@ -1,4 +1,5 @@
 import humanize
+import logging
 import os
 import time
 import shutil
@@ -16,7 +17,7 @@
 PLACEHOLDER_FILE = "file"
 
 
-def touch_access_time(path, access_time: "float | None" = None):
+def touch_access_time(path, access_time: "float | None" = None, strict=True):
     """Update access time of file.
 
     Returns the new access time.
@@ -27,7 +28,7 @@ def touch_access_time(path, access_time: "float | None" = None):
 
     p = Path(path)
 
-    if not p.exists():
+    if not p.exists() and not strict:
         p.touch()
 
     stat = p.stat()
@@ -64,6 +65,7 @@ def _make_local_details(self, path):
         # note that this is called in ._open(), at the point it's known the file
         # will be cached
         fn = super()._make_local_details(path)
+        logging.info(f"cache file: {fn}")
         Path(fn).parent.mkdir(parents=True, exist_ok=True)
 
         return fn
@@ -72,7 +74,7 @@ def hash_name(self, path, same_name):
         # the main change in this function is that, for same_name, it returns
         # the full path
         if same_name:
-            if self.hash_prefix:
+            if self.hash_prefix is not None:
                 # optionally make the name relative to a parent path
                 # using the hash of parent path as a prefix, to flatten a bit
                 suffix = Path(path).relative_to(Path(self.hash_prefix))
@@ -87,10 +89,37 @@ def hash_name(self, path, same_name):
 
             return path
         else:
-            return hash_name(path, same_name)
+            raise NotImplementedError()
 
-    def touch_access_time(path):
-        return touch_access_time(path)
+
+class PinsRscCache(PinsCache):
+    """Modifies the PinsCache to allow hash_prefix to be an RSC server url.
+
+    Note that this class also modifies the first / in a path to be a -, so that
+    pin contents will not be put into subdirectories, for e.g. michael/mtcars/data.txt.
+    """
+
+    protocol = "pinsrsccache"
+
+    def hash_name(self, path, same_name):
+        # the main change in this function is that, for same_name, it returns
+        # the full path
+        if same_name:
+            if self.hash_prefix is None:
+                raise NotImplementedError()
+
+            # change pin path of form <user>/<content> to <user>+<content>
+            suffix = path.replace("/", "+", 1)
+            prefix = hash_name(self.hash_prefix, False)
+
+            # TODO: hacky to automatically tack on protocol here
+            # but this is what R pins boards do. Could make a bool arg?
+            proto_name = protocol_to_string(self.fs.protocol)
+            full_prefix = "_".join([proto_name, prefix])
+            return str(full_prefix / Path(suffix))
+
+        else:
+            raise NotImplementedError()
 
 
 class PinsUrlCache(PinsCache):
@@ -171,7 +200,7 @@ def prune(self, days=30):
             for path in to_prune:
                 delete_version(to_prune)
 
-        print("Skipping cache deletion")
+        logging.info("Skipping cache deletion")
 
 
 def delete_version(path: "str | Path"):
@@ -184,7 +213,7 @@ def disk_usage(path):
 
 
 def prompt_cache_prune(to_prune, size) -> bool:
-    print(to_prune)
+    logging.info(f"Pruning items: {to_prune}")
     human_size = humanize.naturalsize(size, binary=True)
     resp = input(f"Delete {len(to_prune)} pin versions, freeing {human_size}?")
     return resp == "yes"

pins/config.py

@@ -29,3 +29,12 @@ def get_allow_pickle_read(flag):
         flag = bool(env_int)
 
     return flag
+
+
+def _enable_logs():
+    import logging
+
+    format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    handlers = [logging.FileHandler("filename.log"), logging.StreamHandler()]
+
+    logging.basicConfig(level=logging.INFO, format=format, handlers=handlers)

pins/constructors.py

@@ -3,7 +3,7 @@
 import tempfile
 
 from .boards import BaseBoard, BoardRsConnect, BoardManual
-from .cache import PinsCache, PinsUrlCache
+from .cache import PinsCache, PinsUrlCache, PinsRscCache
 from .config import get_data_dir, get_cache_dir
 
 
@@ -58,6 +58,8 @@ def board(
     if storage_options is None:
         storage_options = {}
 
+    # TODO: at this point should just manually construct the rsc board directly
+    # from board_rsconnect...
     if protocol == "rsc":
         # TODO: register RsConnectFs with fsspec
         from pins.rsconnect.fs import RsConnectFs
@@ -71,9 +73,17 @@ def board(
 
     if cache is DEFAULT:
         cache_dir = get_cache_dir()
-        fs = PinsCache(
-            cache_storage=cache_dir, fs=fs, hash_prefix=path, same_names=True
-        )
+
+        # manually create a subdirectory for rsc server
+        if protocol == "rsc":
+            hash_prefix = storage_options["server_url"]
+            fs = PinsRscCache(
+                cache_storage=cache_dir, fs=fs, hash_prefix=hash_prefix, same_names=True
+            )
+        else:
+            fs = PinsCache(
+                cache_storage=cache_dir, fs=fs, hash_prefix=path, same_names=True
+            )
     elif cache is None:
         pass
     else:
@@ -272,7 +282,9 @@ def board_rsconnect(
         server_url = os.environ.get("CONNECT_SERVER")
 
     kwargs = dict(server_url=server_url, api_key=api_key)
-    return board("rsc", "", versioned, cache, allow_pickle_read, storage_options=kwargs)
+    return board(
+        "rsc", None, versioned, cache, allow_pickle_read, storage_options=kwargs
+    )
 
 
 def board_s3(path, versioned=True, cache=DEFAULT, allow_pickle_read=None):

pins/meta.py

@@ -112,8 +112,8 @@ def to_pin_yaml(self, f: Optional[IOBase] = None) -> "str | None":
 class MetaV0:
     file: Union[str, Sequence[str]]
     type: str
+    description: "str | None"
 
-    description: str
     name: str
 
     version: VersionRaw
@@ -134,9 +134,10 @@ def to_dict(self):
     @classmethod
     def from_pin_dict(cls, data, pin_name, version) -> "MetaV0":
         # could infer from dataclasses.fields(), but seems excessive.
-        req_fields = {"type", "description", "name"}
+        req_fields = {"type", "description"}
 
-        req_inputs = {k: v for k, v in data.items() if k in req_fields}
+        # Note that we need to .get(), since fields may not be in metadata
+        req_inputs = {k: data.get(k) for k in req_fields}
         req_inputs["file"] = data["path"]
 
         return cls(**req_inputs, name=pin_name, original_fields=data, version=version)

pins/rsconnect/api.py

@@ -1,3 +1,4 @@
+import logging
 import os
 import requests
 import tempfile
@@ -221,6 +222,7 @@ def _raw_query(self, url, method="GET", return_request=False, **kwargs):
 
         headers = self._get_headers()
 
+        logging.info(f"RSConnect API {method}: {url} -- {kwargs}")
         r = self.session.request(method, url, headers=headers, **kwargs)
 
         if return_request:
@@ -246,13 +248,11 @@ def walk_paginated_offsets(self, f_query, endpoint, method, params=None, **kwarg
         all_results.extend(data["results"])
 
         while data["results"]:
-            print("FETCHING")
             page_kwargs = {"page_number": data["current_page"] + 1}
             new_params = {**params, **page_kwargs}
             data = f_query(endpoint, method, params=new_params)
 
             all_results.extend(data["results"])
-            print(data["results"])
 
         return all_results
 

pins/tests/test_boards.py

@@ -11,6 +11,7 @@
 
 from datetime import datetime, timedelta
 from time import sleep
+from pathlib import Path
 
 # using pytest cases, so that we can pass in fixtures as parameters
 # TODO: this seems like maybe overkill
@@ -288,6 +289,35 @@ def test_board_pin_search_name(board, df, search, matches):
     assert sorted_meta_names == sorted(matches)
 
 
+# BaseBoard specific ==========================================================
+
+from pins.boards import BaseBoard  # noqa
+from pins.cache import PinsCache  # noqa
+
+
+def test_board_base_pin_meta_cache_touch(tmp_dir2, df):
+
+    cache = fsspec.filesystem(
+        "pinscache", target_protocol="file", same_names=True, hash_prefix=str(tmp_dir2),
+    )
+    board = BaseBoard(str(tmp_dir2), fs=cache)
+
+    board.pin_write(df, "some_df", type="csv")
+    meta = board.pin_meta("some_df")
+    v = meta.version.version
+
+    p_cache_version = board._get_cache_path(meta.name, v)
+    p_cache_meta = Path(p_cache_version) / "data.txt"
+
+    orig_access = p_cache_meta.stat().st_atime
+
+    board.pin_meta("some_df")
+
+    new_access = p_cache_meta.stat().st_atime
+
+    assert orig_access < new_access
+
+
 # RStudio Connect specific ====================================================
 
 # import fixture that builds / tearsdown user "susan"