Merge pull request #122 from rstudio/feat-backend-gcs

feat(gcs): initial google cloud storage support

Author: machow

.env.dev

@@ -16,6 +16,11 @@ AWS_REGION=us-east-1
 AZURE_STORAGE_ACCOUNT_NAME=cipins
 AZURE_STORAGE_ACCOUNT_KEY=
 
+# GCS backend ----
+# Note that this backend uses gcsfs's
+# default auth setting, which requires authenticating
+# via the gcloud cli.
+
 # Rstudio Connect license ----
 RSC_LICENSE=
 

.github/workflows/ci.yml

@@ -48,15 +48,26 @@ jobs:
           fi
 
           python -m pip install -e .[test]
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v0
+        with:
+          project_id: siuba-tests
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+          export_default_credentials: true
+
       - name: Run tests
         run: |
-          pytest pins -m 'not fs_rsc and not skip_on_github' $PYTEST_OPTS
+          pytest pins -m 'not fs_rsc and not skip_on_github' --workers 4 --tests-per-worker 1 $PYTEST_OPTS
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_REGION: "us-east-1"
           PYTEST_OPTS: ${{ matrix.pytest_opts }}
           REQUIREMENTS: ${{ matrix.requirements }}
+          # fixes error on macosx virtual machine with pytest-parallel
+          # https://github.com/browsertron/pytest-parallel/issues/93
+          no_proxy: "*"
 
   test-rsconnect:
     name: "Test RSConnect"

Makefile

@@ -30,8 +30,13 @@ README.md: README.Rmd
 		--TagRemovePreprocessor.remove_input_tags='hide-cell' \
 		--output $@
 
-test:
-	pytest
+test: test-most test-rsc
+
+test-most:
+	pytest pins -m "not fs_rsc" --workers 4 --tests-per-worker 1
+
+test-rsc:
+	pytest pins -m "fs_rsc"
 
 docs-build:
 	jb build --builder html docs

README.md

@@ -5,8 +5,8 @@
 The pins package publishes data, models, and other Python objects, making it
 easy to share them across projects and with your colleagues. You can pin
 objects to a variety of pin *boards*, including folders (to share on a
-networked drive or with services like DropBox), RStudio Connect, and Amazon
-S3.
+networked drive or with services like DropBox), RStudio Connect, Amazon S3,
+and Google Cloud Storage.
 Pins can be automatically versioned, making it straightforward to track changes,
 re-run analyses on historical data, and undo mistakes.
 
@@ -103,7 +103,7 @@ You can easily control who gets to access the data using the RStudio
 Connect permissions pane.
 
 The pins package also includes boards that allow you to share data on
-services like Amazon’s S3 (`board_s3()`), with plans to support other backends--
+services like Amazon’s S3 (`board_s3()`), and Google Cloud Storage (`board_gcs`), with plans to support other backends--
 such as Azure's blob storage.
 
 ## Development

docs/api/constructors.rst

@@ -10,5 +10,6 @@ Board Constructors
    ~board_local
    ~board_temp
    ~board_s3
+   ~board_gcs
    ~board_rsconnect
    ~board

docs/getting_started.Rmd

@@ -21,7 +21,8 @@ Getting Started
 ===============
 
 The pins package helps you publish data sets, models, and other Python objects, making it easy to share them across projects and with your colleagues.
-You can pin objects to a variety of "boards", including local folders (to share on a networked drive or with DropBox), RStudio connect, Amazon S3, and more.
+You can pin objects to a variety of "boards", including local folders (to share on a networked drive or with DropBox), RStudio connect, Amazon S3,
+Google Cloud Storage, and more.
 This vignette will introduce you to the basics of pins.
 
 ```{python}

pins/__init__.py

@@ -19,5 +19,6 @@
     board_urls,
     board_rsconnect,
     board_s3,
+    board_gcs,
     board,
 )

pins/boards.py

@@ -277,7 +277,8 @@ def pin_write(
             # move pin to destination ----
             # create pin version folder
             dst_pin_path = self.construct_path([pin_name])
-            dst_version_path = self.path_to_deploy_version(name, meta.version.version)
+            dst_version = meta.version.version
+            dst_version_path = self.path_to_deploy_version(name, dst_version)
 
             if not self.fs.exists(dst_pin_path):
                 # equivalent to mkdirp, want to fail quietly in case of race conditions
@@ -297,7 +298,9 @@ def pin_write(
                     "but that directory already exists."
                 )
 
-            inform(_log, f"Writing to pin {repr(pin_name)}")
+            inform(
+                _log, f"Writing pin:\nName: {repr(pin_name)}\nVersion: {dst_version}"
+            )
 
             res = self.fs.put(tmp_dir, dst_version_path, recursive=True)
 

pins/constructors.py

@@ -115,12 +115,13 @@ def board(
     ----
     Many fsspec implementations of filesystems cache the searching of files, which may
     cause you to not see pins saved by other people. Disable this on these file systems
-    with `storage_options = {"cache_timeout": 0}`.
+    with `storage_options = {"listings_expiry_time": 0}` on s3, or `{"cache_timeout": 0}`
+    on google cloud storage.
 
     """
 
     if storage_options is None:
-        storage_options = {"listings_expiry_time": 0}
+        storage_options = {}
 
     # TODO: at this point should just manually construct the rsc board directly
     # from board_rsconnect...
@@ -381,4 +382,31 @@ def board_s3(path, versioned=True, cache=DEFAULT, allow_pickle_read=None):
 
     """
     # TODO: user should be able to specify storage options here?
-    return board("s3", path, versioned, cache, allow_pickle_read)
+
+    opts = {"listings_expiry_time": 0}
+    return board("s3", path, versioned, cache, allow_pickle_read, storage_options=opts)
+
+
+def board_gcs(path, versioned=True, cache=DEFAULT, allow_pickle_read=None):
+    """Create a board to read and write pins from an AWS S3 bucket folder.
+
+    Parameters
+    ----------
+    path:
+        Path of form <bucket_name>/<optional>/<subdirectory>.
+    **kwargs:
+        Passed to the pins.board function.
+
+    Note
+    ----
+    The gcs board uses the fsspec library (gcsfs) to handle interacting with
+    google cloud storage. Currently, its default mode of authentication
+    is supported.
+
+    See https://gcsfs.readthedocs.io/en/latest/#credentials
+    """
+
+    # GCSFS uses a different name for listings_expiry_time, and then
+    # fixes it under the hood
+    opts = {"cache_timeout": 0}
+    return board("gcs", path, versioned, cache, allow_pickle_read, storage_options=opts)

pins/tests/conftest.py

@@ -14,12 +14,13 @@
 
 
 # Based on https://github.com/machow/siuba/blob/main/siuba/tests/helpers.py
-BACKEND_MARKS = ["fs_s3", "fs_file", "fs_rsc"]
+BACKEND_MARKS = ["fs_s3", "fs_file", "fs_gcs", "fs_rsc"]
 
 # parameters that can be used more than once per session
 params_safe = [
     pytest.param(lambda: BoardBuilder("file"), id="file", marks=m.fs_file),
     pytest.param(lambda: BoardBuilder("s3"), id="s3", marks=m.fs_s3),
+    pytest.param(lambda: BoardBuilder("gcs"), id="s3", marks=m.fs_gcs),
 ]
 
 # rsc should only be used once, because users are created at docker setup time