Merge pull request #147 from rstudio/feat-rsc-public

feat!: support public rsc pin read, using manual board

Author: machow

docs/api/constructors.rst

@@ -13,4 +13,5 @@ Board Constructors
    ~board_gcs
    ~board_azure
    ~board_rsconnect
+   ~board_url
    ~board

docs/api/index.rst

@@ -32,16 +32,18 @@ Boards abstract over different storage backends, making it easy to share data in
 .. list-table::
    :class: table-align-left
 
-   * - :func:`.board_folder`, :func:`.board_local`
+   * - :func:`.board_azure`
+     - Use an Azure storage container as a board
+   * - :func:`.board_folder`, :func:`.board_local`, :func:`.board_temp`
      - Use a local folder as a board
    * - :func:`.board_rsconnect`
      - Use RStudio Connect as a board
    * - :func:`.board_s3`
      - Use an S3 bucket as a board
    * - :func:`.board_gcs`
-     - Use an Google Cloud Storage bucket as a board
-   * - :func:`.board_azure`
-     - Use an Azure Datalake storage container as a board.
+     - Use a Google Cloud Storage bucket as a board.
+   * - :func:`.board_url`
+     - Use a dictionary of URLs as a board
    * - :func:`.board`
      - Generic board constructor
 

pins/__init__.py

@@ -16,7 +16,8 @@
     board_temp,
     board_local,
     board_github,
-    board_urls,
+    board_urls,  # DEPRECATED
+    board_url,
     board_rsconnect,
     board_azure,
     board_s3,

pins/boards.py

@@ -637,7 +637,7 @@ class BoardManual(BaseBoard):
     >>> import fsspec
     >>> import os
     >>> fs = fsspec.filesystem("github", org = "machow", repo = "pins-python")
-    >>> pin_paths = {"df_csv": "df_csv/20220214T163720Z-9bfad"}
+    >>> pin_paths = {"df_csv": "df_csv/20220214T163720Z-9bfad/"}
     >>> board = BoardManual("pins/tests/pins-compat", fs, pin_paths=pin_paths)
 
     >>> board.pin_list()
@@ -690,7 +690,9 @@ def pin_meta(self, name, version=None):
             # create metadata, rather than read from a file
             return self.meta_factory.create_raw(path_to_pin, type="file", name=pin_name)
 
-        path_meta = self.construct_path([pin_name, meta_name])
+        # note that pins on this board should point to versions, so we use an
+        # empty string to mark version (it ultimately is ignored)
+        path_meta = self.construct_path([pin_name, "", meta_name])
         f = self._open_pin_meta(path_meta)
         meta = self.meta_factory.read_pin_yaml(f, pin_name, VersionRaw(""))
 
@@ -717,21 +719,34 @@ def construct_path(self, elements):
         pin_name, *others = elements
         pin_path = self.pin_paths[pin_name]
 
-        if self.board.strip() == "":
-            return pin_path
+        pre_components = [] if not self.board else [self.board]
 
-        if len(others):
-            # this is confusing, but R pins url board has a final "/" indicate that
-            # something is a pin version, rather than a single file. but since other
-            # boards forbid a final /, we need to strip it off to join elements
-            pin_path = pin_path.rstrip().rstrip("/")
+        # note that for paths where version is specified, it gets omitted,
+        # since pin_path should point to a pin version
+        if not pin_path.endswith("/"):
+            if len(others):
+                raise ValueError(
+                    f"pin path {pin_path} does not end in '/' so is assumed to be a"
+                    f" single file. Cannot construct a path to elements {elements}."
+                )
+            return "/".join(pre_components + [pin_path])
+
+        # handle paths to pins (i.e. end with /) ----
+        stripped = pin_path[:-1]
+
+        if len(others) == 0:
+            return "/".join(pre_components + [pin_path])
+        elif len(others) == 1:
+            version = others[0]
+            return "/".join(pre_components + [pin_path])
+        elif len(others) == 2:
+            version, meta = others
 
-            # this is a bit hacky, but this board only aims at specific pins versions,
-            # so the metadata has the version as "", so we need to remove it.
-            if others[0] == "":
-                return super().construct_path([pin_path, *others[1:]])
+            return "/".join(pre_components + [stripped, meta])
 
-        return super().construct_path([pin_path, *others])
+        raise NotImplementedError(
+            f"Unable to construct path from these elements: {elements}"
+        )
 
 
 class BoardRsConnect(BaseBoard):

pins/constructors.py

@@ -289,7 +289,18 @@ def board_github(
     )
 
 
-def board_urls(path: str, pin_paths: dict, cache=DEFAULT, allow_pickle_read=None):
+def board_urls(*args, **kwargs):
+    """DEPRECATED: This board has been renamed to board_url."""
+    from .utils import warn_deprecated
+
+    warn_deprecated(
+        "board_urls has been renamed to board_url. Please use board_url instead."
+    )
+
+    return board_url(*args, **kwargs)
+
+
+def board_url(path: str, pin_paths: dict, cache=DEFAULT, allow_pickle_read=None):
     """Create a board from individual urls.
 
     Parameters
@@ -309,7 +320,7 @@ def board_urls(path: str, pin_paths: dict, cache=DEFAULT, allow_pickle_read=None
     ...     "df_csv": "df_csv/20220214T163720Z-9bfad/",
     ...     "df_arrow": "df_arrow/20220214T163720Z-ad0c1/",
     ... }
-    >>> board = board_urls(github_raw, pin_paths)
+    >>> board = board_url(github_raw, pin_paths)
     >>> board.pin_list()
     ['df_csv', 'df_arrow']
     """
@@ -350,6 +361,31 @@ def board_rsconnect(
         CONNECT_API_KEY environment variable.
     **kwargs:
         Passed to the pins.board function.
+
+    Examples
+    --------
+    Use a server url or set the CONNECT_SERVER environt variable to connect:
+
+    ::
+
+       server_url = "https://connect.rstudioservices.com"
+       board = board_rsconnect(server_url)
+
+    In order to read a public pin, use board_manual with the public pin url.
+
+    ::
+
+       # for a pin at https://connect.rstudioservices.com/content/3004/
+       board = board_url(
+           "https://connect.rstudioservices.com/content",
+           {"my_df": "3004/"}
+       )
+       board.pin_read("my_df")
+
+    See Also
+    --------
+    board_url : board for connecting to individual pins, using a url or path.
+
     """
 
     # TODO: api_key can be passed in to underlying RscApi, equiv to R's manual mode

pins/rsconnect/fs.py

@@ -201,7 +201,6 @@ def put(
         except RsConnectApiMissingContentError:
             # TODO: this could be seen as analogous to mkdir (which gets
             # called by pins anyway)
-            # TODO: hard-coded acl bad?
             content = self.api.post_content_item(parsed.content, access_type)
 
         # Create bundle (with manifest.json inserted if missing) ----

pins/tests/test_boards.py

@@ -443,6 +443,23 @@ def test_board_rsc_pin_write_acl(df, board_short):
     assert content["access_type"] == "all"
 
 
+@pytest.mark.fs_rsc
+def test_board_rsc_pin_read_public(df, board_short):
+    from pins.boards import BoardManual
+
+    board_short.pin_write(df, "susan/mtcars", type="csv", access_type="all")
+
+    # note that users can also get this from the web ui
+    content_url = board_short.fs.info("susan/mtcars")["content_url"]
+
+    # shouldn't be a key set in env, but remove just in case
+    fs = fsspec.filesystem("http")
+    board_url = BoardManual("", fs, pin_paths={"rsc_public": content_url})
+
+    df_no_key = board_url.pin_read("rsc_public")
+    assert df_no_key.equals(df)
+
+
 # Manual Board Specific =======================================================
 
 from pins.boards import BoardManual  # noqa
@@ -491,3 +508,41 @@ def test_board_manual_pin_read():
 
     # do a somewhat data-framey check
     assert df.shape[0] > 1
+
+
+def test_board_manual_construct_path():
+    fs = fsspec.filesystem("file")
+    root = "pins/tests/pins-compat"
+    path_df_csv = "df_csv/20220214T163718Z-eceac/"
+    path_df_csv_v2 = "df_csv/20220214T163720Z-9bfad/df_csv.csv"
+
+    board = BoardManual(
+        root,
+        fs,
+        pin_paths={
+            "df_csv": path_df_csv,
+            "df_csv2_v2": path_df_csv_v2,
+        },
+    )
+
+    # path to pin folder ----
+    # creates path to pin, ignores version, can include data.txt
+    assert board.construct_path(["df_csv"]) == f"{root}/{path_df_csv}"
+    assert board.construct_path(["df_csv", "v"]) == f"{root}/{path_df_csv}"
+    assert (
+        board.construct_path(["df_csv", "v", "data.txt"])
+        == f"{root}/{path_df_csv}data.txt"
+    )
+
+    with pytest.raises(NotImplementedError) as exc_info:
+        board.construct_path(["df_csv", "v", "data.txt", "too_much"])
+
+    assert "Unable to construct path" in exc_info.value.args[0]
+
+    # path to individual file ----
+    assert board.construct_path(["df_csv2_v2"]) == f"{root}/{path_df_csv_v2}"
+
+    with pytest.raises(ValueError) as exc_info:
+        board.construct_path(["df_csv2_v2", "v"])
+
+    assert "assumed to be a single file" in exc_info.value.args[0]