tests: add more cache tests

Author: machow

pins/boards.py

@@ -586,11 +586,10 @@ def pin_meta(self, name, version=None):
         # special case where we are using http protocol to fetch what may be
         # a file. here we need to create a stripped down form of metadata, since
         # a metadata file does not exist (and we can't pull files from a version dir).
-        if self.fs.protocol == "http" and not pin_name.rstrip().endswith("/"):
+        path_to_pin = self.construct_path([pin_name])
+        if self.fs.protocol == "http" and not path_to_pin.rstrip().endswith("/"):
             # create metadata, rather than read from a file
-            path_download = self.construct_path([pin_name])
-
-            return self.meta_factory.create_raw(path_download, type="file",)
+            return self.meta_factory.create_raw(path_to_pin, type="file",)
 
         path_meta = self.construct_path([pin_name, meta_name])
         f = self.fs.open(path_meta)
@@ -613,6 +612,12 @@ def construct_path(self, elements):
         if self.board.strip() == "":
             return pin_path
 
+        if len(others):
+            # this is confusing, but R pins url board has a final "/" indicate that
+            # something is a pin version, rather than a single file. but since other
+            # boards forbid a final /, we need to strip it off to join elements
+            pin_path = pin_path.rstrip().rstrip("/")
+
         return super().construct_path([pin_path, *others])
 
 

pins/tests/test_boards.py

@@ -1,6 +1,7 @@
-import pytest
-import pandas as pd
+import fsspec
 import uuid
+import pandas as pd
+import pytest
 
 from pins.tests.helpers import DEFAULT_CREATION_DATE
 from pins.errors import PinsError
@@ -290,3 +291,53 @@ def test_board_pin_search_admin_user(df, fs_short, fs_admin):  # noqa
     assert search_res2.shape == (1, 2)
     assert search_res2.loc[0, "name"] == "susan/some_df"
     assert search_res2.loc[0, "meta"] is None
+
+
+# Manual Board Specific =======================================================
+
+from pins.boards import BoardManual  # noqa
+
+
+def test_board_manual_http_file_download():
+    # TODO: change when repo is moved to RStudio
+
+    path = "https://raw.githubusercontent.com/machow/pins-python"
+    license_path = "main/LICENSE"
+
+    # use a simple cache, which automatically creates a temporary directory
+    fs = fsspec.filesystem(
+        "simplecache", target_protocol="http", target_options={"block_size": 0}
+    )
+
+    # with path ----
+    board = BoardManual(path, fs, pin_paths={"license": "main/LICENSE"})
+
+    assert board.pin_list() == ["license"]
+    # TODO: better assert
+    assert len(board.pin_download("license"))
+
+    # no base path ----
+    board2 = BoardManual("", fs, pin_paths={"license": f"{path}/{license_path}"})
+
+    assert board2.pin_list() == ["license"]
+    # TODO better assert
+    assert len(board2.pin_download("license"))
+
+
+def test_board_manual_pin_read():
+    # TODO: block size must be set to 0 to handle gzip encoding from github
+    # see https://github.com/fsspec/filesystem_spec/issues/389
+    fs = fsspec.filesystem("http", block_size=0)
+    board = BoardManual(
+        "https://raw.githubusercontent.com/machow/pins-python/main/pins/tests/pins-compat",
+        fs,
+        pin_paths={
+            "df_csv": "df_csv/20220214T163718Z-eceac/",
+            "df_csv2_v2": "df_csv/20220214T163720Z-9bfad/",
+        },
+    )
+
+    df = board.pin_read("df_csv")
+
+    # do a somewhat data-framey check
+    assert df.shape[0] > 1

pins/tests/test_cache.py

@@ -2,7 +2,18 @@
 import shutil
 
 import pytest
-from pins.cache import CachePruner, touch_access_time, cache_prune
+from pins.cache import (
+    CachePruner,
+    touch_access_time,
+    cache_prune,
+    PinsCache,
+    PinsUrlCache,
+)
+
+from fsspec import filesystem
+
+
+# Utilities ===================================================================
 
 
 @pytest.fixture
@@ -31,6 +42,31 @@ def test_touch_access_time_auto(some_file):
     assert orig_access < new_time
 
 
+# Cache Classes ===============================================================
+
+# Boards w/ default cache =====================================================
+
+
+def test_pins_cache_hash_name_preserves():
+    cache = PinsCache(fs=filesystem("file"))
+    assert cache.hash_name("a/b/c.txt", True) == "a/b/c.txt"
+
+
+def test_pins_cache_url_hash_name():
+    cache = PinsUrlCache(fs=filesystem("file"))
+    hashed = cache.hash_name("http://example.com/a.txt", True)
+
+    # should have form <url_hash>/<version_placeholder>/<filename>
+    assert hashed.endswith("/a.txt")
+    assert hashed.count("/") == 2
+
+
+@pytest.mark.skip("TODO")
+def test_pins_cache_open():
+    # check that opening works and creates the cached file
+    pass
+
+
 # Cache pruning ===============================================================
 
 

pins/tests/test_constructors.py

@@ -33,6 +33,9 @@ def check_dir_writable(p_dir):
     assert os.access(p_dir.parent.absolute(), os.W_OK)
 
 
+# Board particulars ===========================================================
+
+
 @pytest.mark.skip_on_github
 def test_board_constructor_local_default_writable():