feat: azure backend (#143)

* feat: azure backend

* fix(azure): use adl (gen2 system) in constructor, clean up

* fix(azure): wait abfs is gen2, change back

* docs: add board_azure to docs

Author: machow

.github/workflows/ci.yml

@@ -69,6 +69,8 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_REGION: "us-east-1"
+          AZURE_STORAGE_ACCOUNT_NAME: ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
+          AZURE_STORAGE_ACCOUNT_KEY: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }}
           PYTEST_OPTS: ${{ matrix.pytest_opts }}
           REQUIREMENTS: ${{ matrix.requirements }}
           ACTION_OS: ${{ matrix.os }}

docs/api/constructors.rst

@@ -11,5 +11,6 @@ Board Constructors
    ~board_temp
    ~board_s3
    ~board_gcs
+   ~board_azure
    ~board_rsconnect
    ~board

docs/api/index.rst

@@ -38,6 +38,10 @@ Boards abstract over different storage backends, making it easy to share data in
      - Use RStudio Connect as a board
    * - :func:`.board_s3`
      - Use an S3 bucket as a board
+   * - :func:`.board_gcs`
+     - Use an Google Cloud Storage bucket as a board
+   * - :func:`.board_azure`
+     - Use an Azure Datalake storage container as a board.
    * - :func:`.board`
      - Generic board constructor
 

docs/getting_started.Rmd

@@ -22,7 +22,7 @@ Getting Started
 
 The pins package helps you publish data sets, models, and other Python objects, making it easy to share them across projects and with your colleagues.
 You can pin objects to a variety of "boards", including local folders (to share on a networked drive or with DropBox), RStudio connect, Amazon S3,
-Google Cloud Storage, and more.
+Google Cloud Storage, Azure Datalake, and more.
 This vignette will introduce you to the basics of pins.
 
 ```{python}

docs/intro.md

@@ -21,7 +21,7 @@ kernelspec:
 ```
 
 The pins package publishes data, models, and other Python objects, making it easy to share
-You can pin objects to a variety of pin *boards*, including folders (to share on a networked drive or with services like DropBox), RStudio Connect, Amazon S3, and Google Cloud Storage.
+You can pin objects to a variety of pin *boards*, including folders (to share on a networked drive or with services like DropBox), RStudio Connect, Amazon S3, Google Cloud Storage, and Azure Datalake.
 Pins can be automatically versioned, making it straightforward to track changes, re-run analyses on historical data, and undo mistakes.
 
 ## Installation
@@ -96,5 +96,6 @@ board.pin_read("hadley/sales-summary")
 
 You can easily control who gets to access the data using the RStudio Connect permissions pane.
 
-The pins package also includes boards that allow you to share data on services like Amazon's S3 (`board_s3()`) and Google Cloud Storage (`board_gcs()`).
+The pins package also includes boards that allow you to share data on services like
+Amazon's S3 (`board_s3()`), Google Cloud Storage (`board_gcs()`), and Azure Datalake (`board_azure()`).
 Learn more in [getting started](getting_started.Rmd).

pins/__init__.py

@@ -18,6 +18,7 @@
     board_github,
     board_urls,
     board_rsconnect,
+    board_azure,
     board_s3,
     board_gcs,
     board,

pins/constructors.py

@@ -111,8 +111,8 @@ def board(
     board_factory:
         An optional board class to use as the constructor.
 
-    Note
-    ----
+    Notes
+    -----
     Many fsspec implementations of filesystems cache the searching of files, which may
     cause you to not see pins saved by other people. Disable this on these file systems
     with `storage_options = {"listings_expiry_time": 0}` on s3, or `{"cache_timeout": 0}`
@@ -256,8 +256,8 @@ def board_github(
     **kwargs:
         Passed to the pins.board function.
 
-    Note
-    ----
+    Notes
+    -----
     This board is read only.
 
 
@@ -374,12 +374,14 @@ def board_s3(path, versioned=True, cache=DEFAULT, allow_pickle_read=None):
     **kwargs:
         Passed to the pins.board function.
 
-    Note
-    ----
+    Notes
+    -----
     The s3 board uses the fsspec library (s3fs) to handle interacting with s3.
     In order to authenticate, set the AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
     and (optionally) AWS_REGION environment variables.
 
+    See https://github.com/fsspec/s3fs
+
     """
     # TODO: user should be able to specify storage options here?
 
@@ -397,8 +399,8 @@ def board_gcs(path, versioned=True, cache=DEFAULT, allow_pickle_read=None):
     **kwargs:
         Passed to the pins.board function.
 
-    Note
-    ----
+    Notes
+    -----
     The gcs board uses the fsspec library (gcsfs) to handle interacting with
     google cloud storage. Currently, its default mode of authentication
     is supported.
@@ -410,3 +412,28 @@ def board_gcs(path, versioned=True, cache=DEFAULT, allow_pickle_read=None):
     # fixes it under the hood
     opts = {"cache_timeout": 0}
     return board("gcs", path, versioned, cache, allow_pickle_read, storage_options=opts)
+
+
+def board_azure(path, versioned=True, cache=DEFAULT, allow_pickle_read=None):
+    """Create a board to read and write pins from an Google Cloud Storage bucket folder.
+
+    Parameters
+    ----------
+    path:
+        Path of form <bucket_name>/<optional>/<subdirectory>.
+    **kwargs:
+        Passed to the pins.board function.
+
+    Notes
+    -----
+    The azure board uses the fsspec library (adlfs) to handle interacting with
+    Azure Datalake Filesystem (abfs). Currently, its default mode of authentication
+    is supported.
+
+    See https://github.com/fsspec/adlfs
+    """
+
+    opts = {"use_listings_cache": False}
+    return board(
+        "abfs", path, versioned, cache, allow_pickle_read, storage_options=opts
+    )

pins/tests/conftest.py

@@ -14,13 +14,14 @@
 
 
 # Based on https://github.com/machow/siuba/blob/main/siuba/tests/helpers.py
-BACKEND_MARKS = ["fs_s3", "fs_file", "fs_gcs", "fs_rsc"]
+BACKEND_MARKS = ["fs_s3", "fs_file", "fs_gcs", "fs_abfs", "fs_rsc"]
 
 # parameters that can be used more than once per session
 params_safe = [
     pytest.param(lambda: BoardBuilder("file"), id="file", marks=m.fs_file),
     pytest.param(lambda: BoardBuilder("s3"), id="s3", marks=m.fs_s3),
-    pytest.param(lambda: BoardBuilder("gcs"), id="s3", marks=m.fs_gcs),
+    pytest.param(lambda: BoardBuilder("gcs"), id="gcs", marks=m.fs_gcs),
+    pytest.param(lambda: BoardBuilder("abfs"), id="abfs", marks=m.fs_abfs),
 ]
 
 # rsc should only be used once, because users are created at docker setup time

pins/tests/helpers.py

@@ -25,11 +25,8 @@
     "file": {"path": ["PINS_TEST_FILE__PATH", None]},
     "s3": {"path": ["PINS_TEST_S3__PATH", "ci-pins"]},
     "gcs": {"path": ["PINS_TEST_GCS__PATH", "ci-pins"]},
+    "abfs": {"path": ["PINS_TEST_AZURE__PATH", "ci-pins"]},
     "rsc": {"path": ["PINS_TEST_RSC__PATH", RSC_SERVER_URL]},
-    # TODO(question): R pins has the whole server a board
-    # but it's a bit easier to test by (optionally) allowing a user
-    # or something else to be a board
-    # "rsc": {"path": ["PINS_TEST_RSC__PATH", ""]}
 }
 
 # TODO: Backend initialization should be independent of helpers, but these
@@ -121,7 +118,7 @@ def create_tmp_board(self, src_board=None) -> BaseBoard:
         if self.fs_name == "gcs":
             opts = {"cache_timeout": 0}
         else:
-            opts = {"listings_expiry_time": 0}
+            opts = {"use_listings_cache": False}
 
         fs = filesystem(self.fs_name, **opts)
         temp_name = str(uuid.uuid4())

pins/tests/test_constructors.py

@@ -45,6 +45,8 @@ def construct_from_board(board):
         board = c.board_rsconnect(
             server_url=board.fs.api.server_url, api_key=board.fs.api.api_key
         )
+    elif fs_name == "abfs":
+        board = c.board_azure(board.board)
     else:
         board = getattr(c, f"board_{fs_name}")(board.board)
 
@@ -214,6 +216,8 @@ def test_constructor_boards_multi_user(board2, df_csv, tmp_cache):
         # TODO: RSConnect writes pin names like <user>/<name>, so would need to
         # modify test
         pytest.skip()
+    elif fs_name == "abfs":
+        fs_name = "azure"
 
     first = construct_from_board(board2)