Refactor external storage to use fsspec for unified backend

- Add fsspec>=2023.1.0 as core dependency
- Add optional dependencies for cloud backends (s3fs, gcsfs, adlfs)
- Create new storage.py module with StorageBackend class
  - Unified interface for file, S3, GCS, and Azure storage
  - Methods: put_file, get_file, put_buffer, get_buffer, exists, remove
- Refactor ExternalTable to use StorageBackend instead of protocol-specific code
  - Replace _upload_file, _download_file, etc. with storage backend calls
  - Add storage property, deprecate s3 property
- Update settings.py to support GCS and Azure protocols
- Add deprecation warning to s3.py Folder class
  - Module kept for backward compatibility
  - Will be removed in future version

This lays the foundation for the new object type which will also use fsspec.

Author: claude

pyproject.toml

@@ -17,12 +17,14 @@ dependencies = [
   "networkx",
   "pydot",
   "minio>=7.0.0",
+  "fsspec>=2023.1.0",
   "matplotlib",
   "faker",
   "urllib3",
   "setuptools",
   "pydantic-settings>=2.0.0",
 ]
+
 requires-python = ">=3.10,<3.14"
 authors = [
   {name = "Dimitri Yatsenko", email = "[email protected]"},
@@ -90,6 +92,9 @@ test = [
 ]
 
 [project.optional-dependencies]
+s3 = ["s3fs>=2023.1.0"]
+gcs = ["gcsfs>=2023.1.0"]
+azure = ["adlfs>=2023.1.0"]
 dev = [
   "pre-commit",
   "ruff",

src/datajoint/external.py

@@ -1,15 +1,17 @@
 import logging
+import warnings
 from collections.abc import Mapping
 from pathlib import Path, PurePosixPath, PureWindowsPath
 
 from tqdm import tqdm
 
-from . import errors, s3
+from . import errors
 from .declare import EXTERNAL_TABLE_ROOT
 from .errors import DataJointError, MissingExternalFile
 from .hash import uuid_from_buffer, uuid_from_file
 from .heading import Heading
 from .settings import config
+from .storage import StorageBackend
 from .table import FreeTable, Table
 from .utils import safe_copy, safe_write
 
@@ -38,7 +40,7 @@ class ExternalTable(Table):
     def __init__(self, connection, store, database):
         self.store = store
         self.spec = config.get_store_spec(store)
-        self._s3 = None
+        self._storage = None
         self.database = database
         self._connection = connection
         self._heading = Heading(
@@ -52,9 +54,8 @@ def __init__(self, connection, store, database):
         self._support = [self.full_table_name]
         if not self.is_declared:
             self.declare()
-        self._s3 = None
-        if self.spec["protocol"] == "file" and not Path(self.spec["location"]).is_dir():
-            raise FileNotFoundError("Inaccessible local directory %s" % self.spec["location"]) from None
+        # Initialize storage backend (validates configuration)
+        _ = self.storage
 
     @property
     def definition(self):
@@ -73,17 +74,32 @@ def definition(self):
     def table_name(self):
         return f"{EXTERNAL_TABLE_ROOT}_{self.store}"
 
+    @property
+    def storage(self) -> StorageBackend:
+        """Get or create the storage backend instance."""
+        if self._storage is None:
+            self._storage = StorageBackend(self.spec)
+        return self._storage
+
     @property
     def s3(self):
-        if self._s3 is None:
-            self._s3 = s3.Folder(**self.spec)
-        return self._s3
+        """Deprecated: Use storage property instead."""
+        warnings.warn(
+            "ExternalTable.s3 is deprecated. Use ExternalTable.storage instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        # For backward compatibility, return a legacy s3.Folder if needed
+        from . import s3
+        if not hasattr(self, "_s3_legacy") or self._s3_legacy is None:
+            self._s3_legacy = s3.Folder(**self.spec)
+        return self._s3_legacy
 
     # - low-level operations - private
 
     def _make_external_filepath(self, relative_filepath):
         """resolve the complete external path based on the relative path"""
-        # Strip root
+        # Strip root for S3 paths
         if self.spec["protocol"] == "s3":
             posix_path = PurePosixPath(PureWindowsPath(self.spec["location"]))
             location_path = (
@@ -92,11 +108,13 @@ def _make_external_filepath(self, relative_filepath):
                 else Path(posix_path)
             )
             return PurePosixPath(location_path, relative_filepath)
-        # Preserve root
+        # Preserve root for local filesystem
         elif self.spec["protocol"] == "file":
             return PurePosixPath(Path(self.spec["location"]), relative_filepath)
         else:
-            assert False
+            # For other protocols (gcs, azure, etc.), treat like S3
+            location = self.spec.get("location", "")
+            return PurePosixPath(location, relative_filepath) if location else PurePosixPath(relative_filepath)
 
     def _make_uuid_path(self, uuid, suffix=""):
         """create external path based on the uuid hash"""
@@ -109,57 +127,32 @@ def _make_uuid_path(self, uuid, suffix=""):
         )
 
     def _upload_file(self, local_path, external_path, metadata=None):
-        if self.spec["protocol"] == "s3":
-            self.s3.fput(local_path, external_path, metadata)
-        elif self.spec["protocol"] == "file":
-            safe_copy(local_path, external_path, overwrite=True)
-        else:
-            assert False
+        """Upload a file to external storage using fsspec backend."""
+        self.storage.put_file(local_path, external_path, metadata)
 
     def _download_file(self, external_path, download_path):
-        if self.spec["protocol"] == "s3":
-            self.s3.fget(external_path, download_path)
-        elif self.spec["protocol"] == "file":
-            safe_copy(external_path, download_path)
-        else:
-            assert False
+        """Download a file from external storage using fsspec backend."""
+        self.storage.get_file(external_path, download_path)
 
     def _upload_buffer(self, buffer, external_path):
-        if self.spec["protocol"] == "s3":
-            self.s3.put(external_path, buffer)
-        elif self.spec["protocol"] == "file":
-            safe_write(external_path, buffer)
-        else:
-            assert False
+        """Upload bytes to external storage using fsspec backend."""
+        self.storage.put_buffer(buffer, external_path)
 
     def _download_buffer(self, external_path):
-        if self.spec["protocol"] == "s3":
-            return self.s3.get(external_path)
-        if self.spec["protocol"] == "file":
-            try:
-                return Path(external_path).read_bytes()
-            except FileNotFoundError:
-                raise errors.MissingExternalFile(f"Missing external file {external_path}") from None
-        assert False
+        """Download bytes from external storage using fsspec backend."""
+        return self.storage.get_buffer(external_path)
 
     def _remove_external_file(self, external_path):
-        if self.spec["protocol"] == "s3":
-            self.s3.remove_object(external_path)
-        elif self.spec["protocol"] == "file":
-            try:
-                Path(external_path).unlink()
-            except FileNotFoundError:
-                pass
+        """Remove a file from external storage using fsspec backend."""
+        self.storage.remove(external_path)
 
     def exists(self, external_filepath):
         """
+        Check if an external file is accessible using fsspec backend.
+
         :return: True if the external file is accessible
         """
-        if self.spec["protocol"] == "s3":
-            return self.s3.exists(external_filepath)
-        if self.spec["protocol"] == "file":
-            return Path(external_filepath).is_file()
-        assert False
+        return self.storage.exists(external_filepath)
 
     # --- BLOBS ----
 

src/datajoint/s3.py

@@ -1,9 +1,19 @@
 """
-AWS S3 operations
+AWS S3 operations using minio client.
+
+.. deprecated:: 0.15.0
+    This module is deprecated. Use :mod:`datajoint.storage` with fsspec backend instead.
+    The minio-based S3 client will be removed in a future version.
+
+    Migration guide:
+    - Instead of importing from datajoint.s3, use datajoint.storage.StorageBackend
+    - StorageBackend provides a unified interface for all storage protocols
+    - See datajoint.storage module for details
 """
 
 import logging
 import uuid
+import warnings
 from io import BytesIO
 from pathlib import Path
 
@@ -17,7 +27,10 @@
 
 class Folder:
     """
-    A Folder instance manipulates a flat folder of objects within an S3-compatible object store
+    A Folder instance manipulates a flat folder of objects within an S3-compatible object store.
+
+    .. deprecated:: 0.15.0
+        Use :class:`datajoint.storage.StorageBackend` instead.
     """
 
     def __init__(
@@ -31,6 +44,12 @@ def __init__(
         proxy_server=None,
         **_,
     ):
+        warnings.warn(
+            "datajoint.s3.Folder is deprecated and will be removed in a future version. "
+            "Use datajoint.storage.StorageBackend with fsspec instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         # from https://docs.min.io/docs/python-client-api-reference
         self.client = minio.Minio(
             endpoint,

src/datajoint/settings.py

@@ -275,13 +275,19 @@ def get_store_spec(self, store: str) -> dict[str, Any]:
 
         # Validate protocol
         protocol = spec.get("protocol", "").lower()
-        if protocol not in ("file", "s3"):
-            raise DataJointError(f'Missing or invalid protocol in config.stores["{store}"]')
+        supported_protocols = ("file", "s3", "gcs", "azure")
+        if protocol not in supported_protocols:
+            raise DataJointError(
+                f'Missing or invalid protocol in config.stores["{store}"]. '
+                f'Supported protocols: {", ".join(supported_protocols)}'
+            )
 
         # Define required and allowed keys by protocol
         required_keys: dict[str, tuple[str, ...]] = {
             "file": ("protocol", "location"),
             "s3": ("protocol", "endpoint", "bucket", "access_key", "secret_key", "location"),
+            "gcs": ("protocol", "bucket", "location"),
+            "azure": ("protocol", "container", "location"),
         }
         allowed_keys: dict[str, tuple[str, ...]] = {
             "file": ("protocol", "location", "subfolding", "stage"),
@@ -297,6 +303,25 @@ def get_store_spec(self, store: str) -> dict[str, Any]:
                 "stage",
                 "proxy_server",
             ),
+            "gcs": (
+                "protocol",
+                "bucket",
+                "location",
+                "token",
+                "project",
+                "subfolding",
+                "stage",
+            ),
+            "azure": (
+                "protocol",
+                "container",
+                "location",
+                "account_name",
+                "account_key",
+                "connection_string",
+                "subfolding",
+                "stage",
+            ),
         }
 
         # Check required keys