🪣 Support for OBject Storage (#1021)

Kezzsim · genematx · web-flow · commit c4b169315a51 · 2025-10-27T20:57:52.000-04:00
* 🎈 *writes data to get teh party started*

* 🌫️ *anxiously adds more cloud providers*

* Resolve mypy errors

* 👍️ Resolve minio https error preventing us from writing `zarr.json`

* 🚮 Experiment with writing (sloppy) data

* 🪲 DEBUG: problems with `write`

* 🕶️ Review : Add missing prefix

Co-authored-by: Eugene &lt;ymatviych@bnl.gov&gt;

* ✍️ Write regex helper function

* 🧽 refactor to clean up repeated code

* ✍️ Add Blobs to writing tests

* ✍️ Rewrite `get_storage` to be a router for buckets

* refactor ObjectStorage

* 🐋 Add minio container to CI for testing

* 🧪 Make `TILED_TEST_BUCKET` env var for advanced testing

* More refactoring of Storage

* FIX: look up registered storages instead of recreating them

* Simplify test config

* TST: fix test_writing + more refactoring

* MNT: add minio dependency for server

* ENH: generalize asset deletion

---------

Co-authored-by: Eugene &lt;ymatviych@bnl.gov&gt;
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -62,6 +62,12 @@ jobs:
       shell: bash -l {0}
       run: source continuous_integration/scripts/start_redis.sh
 
+    - name: Start Minio service in container.
+      #TODO: This product is leaving open-source container distribution
+      # Find a new image or product to use
+      # https://github.com/minio/minio/issues/21647#issuecomment-3418675115
+      shell: bash -l {0}
+      run: source continuous_integration/scripts/start_minio.sh
 
     - name: Ensure example data is migrated to current catalog database schema.
       # The example data is expected to be kept up to date to the latest Tiled
@@ -84,6 +90,7 @@ jobs:
         # Provide test suite with PostgreSQL and Redis databases to use.
         TILED_TEST_POSTGRESQL_URI: postgresql://postgres:secret@localhost:5432
         TILED_TEST_REDIS: redis://localhost:6379
+        TILED_TEST_BUCKET: http://minioadmin:minioadmin@localhost:9000/buck
         # TODO Reinstate after finding a new image to use
         # https://github.com/bluesky/tiled/issues/1109
         # # Opt in to LDAPAuthenticator tests.
diff --git a/continuous_integration/docker-configs/minio-docker-compose.yml b/continuous_integration/docker-configs/minio-docker-compose.yml
@@ -0,0 +1,29 @@
+version: "3.2"
+services:
+  minio:
+    image: minio/minio:latest
+    ports:
+      - 9000:9000
+      - 9001:9001
+    volumes:
+      - minio-data:/data
+    environment:
+      MINIO_ROOT_USER: "minioadmin"
+      MINIO_ROOT_PASSWORD: "minioadmin"
+    command: server /data --console-address :9001
+    restart: unless-stopped
+  create-bucket:
+    image: minio/mc:latest
+    environment:
+      MC_HOST_minio: http://minioadmin:minioadmin@minio:9000
+    entrypoint:
+      - sh
+      - -c
+      - |
+        until mc ls minio > /dev/null 2>&1; do
+          sleep 0.5
+        done
+
+        mc mb --ignore-existing minio/buck
+volumes:
+  minio-data:
diff --git a/continuous_integration/scripts/start_minio.sh b/continuous_integration/scripts/start_minio.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+# Start MinIO server in docker container
+docker pull minio/minio:latest
+docker compose -f continuous_integration/docker-configs/minio-docker-compose.yml up -d
+docker ps
diff --git a/example_configs/bucket_storage/bucket_storage.yml b/example_configs/bucket_storage/bucket_storage.yml
@@ -0,0 +1,17 @@
+authentication:
+  allow_anonymous_access: false
+trees:
+  - path: /
+    tree: catalog
+    args:
+      uri: "sqlite:///storage/catalog.db"
+      writable_storage:
+        - provider: s3
+          uri: "http://localhost:9000"
+          config:
+            access_key_id: "minioadmin"
+            secret_access_key: "minioadmin"
+            bucket: "buck"
+            virtual_hosted_style_request: False
+            client_options: {"allow_http": True}
+      init_if_not_exists: true
diff --git a/example_configs/bucket_storage/docker-compose.yml b/example_configs/bucket_storage/docker-compose.yml
@@ -0,0 +1,29 @@
+version: "3.2"
+services:
+  minio:
+    image: minio/minio:latest
+    ports:
+      - 9000:9000
+      - 9001:9001
+    volumes:
+      - minio-data:/data
+    environment:
+      MINIO_ROOT_USER: "minioadmin"
+      MINIO_ROOT_PASSWORD: "minioadmin"
+    command: server /data --console-address :9001
+    restart: unless-stopped
+  create-bucket:
+    image: minio/mc:latest
+    environment:
+      MC_HOST_minio: http://minioadmin:minioadmin@minio:9000
+    entrypoint:
+      - sh
+      - -c
+      - |
+        until mc ls minio > /dev/null 2>&1; do
+          sleep 0.5
+        done
+
+        mc mb --ignore-existing minio/buck
+volumes:
+  minio-data:
diff --git a/example_configs/bucket_storage/readme.md b/example_configs/bucket_storage/readme.md
@@ -0,0 +1,18 @@
+# Create a local bucket for testing access to BLOBS
+
+In this example there exists:
+- A `docker-compose.yml` file capable of instantiating and running a [Minio](https://min.io/) container.
+- A configuration yaml file `bucket_storage.yml` which contains information tiled needs to authenticate with the bucket storage system and write / read Binary Large Objects (BLOBS) through the Zaar adapter.
+
+## How to run this example:
+1. In one terminal window, navigate to the directory where the `docker-compose.yml` and `bucket_storage.yml` are.
+2. Run `docker compose up` with adequate permissions.
+3. Open another terminal window in the same location and run `tiled serve config bucket_storage.yml --api-key secret`
+4. You will need to create a `storage` directory in `/example_configs/bucket_storage` for the sqlite database.
+5. Create an `ipython` session and run the following commands to write array data as a BLOB in a bucket:
+```python
+from tiled.client import from_uri
+c = from_uri('http://localhost:8000', api_key='secret')
+c.write_array([1,2,3])
+```
+6. You will be able to see the written data in the bucket if you log in to the minio container, exposed on your machine at `http://localhost:9001/login`. </br> Use testing credentials `minioadmin` for both fields.
diff --git a/pixi.toml b/pixi.toml
@@ -54,6 +54,7 @@ flake8 = "*"
 ipython = "*"
 ldap3 = "*"
 matplotlib = "*"
+minio = "*"
 mistune = "*"
 myst-parser = "*"
 numpydoc = "*"
diff --git a/pyproject.toml b/pyproject.toml
@@ -80,10 +80,12 @@ all = [
     "jinja2",
     "jmespath",
     "lz4",
+    "minio",
     "ndindex",
     "numcodecs",
     "numpy",
     "numba >=0.59.0",  # indirect, pinned to assist uv solve
+    "obstore",
     "openpyxl",
     "packaging",
     "pandas",
@@ -156,6 +158,7 @@ dev = [
     "ldap3",
     "locust",
     "matplotlib",
+    "minio",
     "mistune",
     "myst-parser",
     "numpydoc",
@@ -242,10 +245,12 @@ server = [
     "jinja2",
     "jmespath",
     "lz4",
+    "minio",
     "ndindex",
     "numba >=0.59.0",  # indirect, pinned to assist uv solve
     "numcodecs",
     "numpy",
+    "obstore",
     "openpyxl",
     "packaging",
     "pandas",
diff --git a/tiled/_tests/conftest.py b/tiled/_tests/conftest.py
@@ -4,6 +4,7 @@
 import tempfile
 from pathlib import Path
 from typing import Any
+from urllib.parse import urlparse
 
 import asyncpg
 import pytest
@@ -313,6 +314,40 @@ def redis_uri():
         raise pytest.skip("No TILED_TEST_REDIS configured")
 
 
+@pytest.fixture
+def minio_uri():
+    if uri := os.getenv("TILED_TEST_BUCKET"):
+        from minio import Minio
+        from minio.deleteobjects import DeleteObject
+
+        # For convenience, we split the bucket from a string
+        url = urlparse(uri)
+        bucket = url.path.lstrip("/")
+        uri = url._replace(netloc="{}:{}".format(url.hostname, url.port), path="")
+
+        client = Minio(
+            uri.geturl(),
+            access_key=url.username,
+            secret_key=url.password,
+            secure=False,
+        )
+
+        # Reset the state of the bucket after each test.
+        if client.bucket_exists(bucket):
+            delete_object_list = map(
+                lambda x: DeleteObject(x.object_name),
+                client.list_objects(bucket, recursive=True),
+            )
+            errors = client.remove_objects(bucket, delete_object_list)
+            for error in errors:
+                print("error occurred when deleting object", error)
+        else:
+            client.make_bucket(bucket)
+
+    else:
+        raise pytest.skip("No TILED_TEST_BUCKET configured")
+
+
 @pytest.fixture(scope="function")
 def tiled_websocket_context(tmpdir, redis_uri):
     """Fixture that provides a Tiled context with websocket support."""
diff --git a/tiled/_tests/test_writing.py b/tiled/_tests/test_writing.py
@@ -5,9 +5,11 @@
 """
 
 import base64
+import os
 import threading
 import uuid
 from datetime import datetime
+from urllib.parse import urljoin, urlparse
 
 import awkward
 import dask.dataframe
@@ -17,6 +19,8 @@
 import pyarrow
 import pytest
 import sparse
+from minio import Minio
+from minio.error import S3Error
 from pandas.testing import assert_frame_equal
 from starlette.status import (
     HTTP_404_NOT_FOUND,
@@ -37,7 +41,7 @@
 from ..structures.data_source import DataSource
 from ..structures.sparse import COOStructure
 from ..structures.table import TableStructure
-from ..utils import APACHE_ARROW_FILE_MIME_TYPE, patch_mimetypes
+from ..utils import APACHE_ARROW_FILE_MIME_TYPE, patch_mimetypes, sanitize_uri
 from ..validation_registration import ValidationRegistry
 from .utils import fail_with_status_code
 
@@ -46,13 +50,55 @@
 
 
 @pytest.fixture
-def tree(tmpdir):
-    return in_memory(
-        writable_storage=[
-            f"file://localhost{str(tmpdir / 'data')}",
-            f"duckdb:///{tmpdir / 'data.duckdb'}",
-        ]
-    )
+def tmp_minio_bucket():
+    """Create a temporary MinIO bucket and clean it up after tests."""
+    if uri := os.getenv("TILED_TEST_BUCKET"):
+        clean_uri, username, password = sanitize_uri(uri)
+        minio_client = Minio(
+            urlparse(clean_uri).netloc,  # e.g. only "localhost:9000"
+            access_key=username or "minioadmin",
+            secret_key=password or "minioadmin",
+            secure=False,
+        )
+
+        bucket_name = f"test-{uuid.uuid4().hex}"
+        minio_client.make_bucket(bucket_name)
+
+        try:
+            yield urljoin(uri, "/" + bucket_name)  # full URI with credentials
+        finally:
+            # Cleanup: remove all objects and delete the bucket
+            try:
+                objects = minio_client.list_objects(bucket_name, recursive=True)
+                for obj in objects:
+                    minio_client.remove_object(bucket_name, obj.object_name)
+                minio_client.remove_bucket(bucket_name)
+            except S3Error as e:
+                print(f"Warning: failed to delete test bucket {bucket_name}: {e}")
+
+    else:
+        yield None
+
+
+@pytest.fixture
+def tree(tmpdir, tmp_minio_bucket):
+    writable_storage = [f"duckdb:///{tmpdir / 'data.duckdb'}"]
+
+    if tmp_minio_bucket:
+        writable_storage.append(
+            {
+                "provider": "s3",
+                "uri": tmp_minio_bucket,
+                "config": {
+                    "virtual_hosted_style_request": False,
+                    "client_options": {"allow_http": True},
+                },
+            }
+        )
+
+    writable_storage.append(f"file://localhost{str(tmpdir / 'data')}")
+
+    return in_memory(writable_storage=writable_storage)
 
 
 def test_write_array_full(tree):
diff --git a/tiled/adapters/zarr.py b/tiled/adapters/zarr.py
diff --git a/tiled/catalog/adapter.py b/tiled/catalog/adapter.py
diff --git a/tiled/storage.py b/tiled/storage.py
