Define reader usable by Xarray (#1)

* Define reader usable by Xarray

* Use uv in workflow

Author: maxrjones

.github/workflows/test.yml

@@ -28,24 +28,17 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0  # grab all branches and tags
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-        cache: 'pip'
-    - name: Install Hatch
-      run: |
-        python -m pip install --upgrade pip
-        pip install hatch
-    - name: Set Up Hatch Env
-      run: |
-        hatch env create upstream
-        hatch env run -e upstream list-env
     - name: Run Tests
-      env:
-        HYPOTHESIS_PROFILE: ci
       run: |
-        hatch env run --env upstream run-coverage
+        uv run --all-groups pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src
     - name: Upload coverage
       uses: codecov/codecov-action@v5
       with:

pyproject.toml

@@ -37,7 +37,7 @@ Documentation = "https://github.com/virtual-zarr/obspec-utils#readme"
 Issues = "https://github.com/virtual-zarr/obspec-utils/issues"
 Source = "https://github.com/virtual-zarr/obspec-utils"
 
-[project.optional-dependencies]
+[dependency-groups]
 test = [
     "coverage",
     "pytest",
@@ -47,12 +47,18 @@ test = [
     "rich",
     "mypy",
     "pytest-xdist",
+    "minio",
+]
+xarray = [
+  "xarray",
+  "h5netcdf",
+]
+fsspec = [
+  "s3fs",
+  "fsspec",
 ]
-
-[dependency-groups]
 dev = [
     "ipykernel>=6.29.5",
-    "pip>=25.0.1",
 ]
 
 [tool.hatch.metadata]
@@ -64,41 +70,6 @@ version.source = "vcs"
 [tool.hatch.build]
 hooks.vcs.version-file = "src/obspec_utils/_version.py"
 
-[tool.hatch.envs.types]
-extra-dependencies = [
-  "mypy>=1.0.0",
-]
-[tool.hatch.envs.types.scripts]
-check = "mypy --install-types --non-interactive {args:src/obspec_utils tests}"
-
-
-[tool.hatch.envs.test]
-features = ["test"]
-
-[tool.hatch.envs.test.scripts]
-run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy"
-run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src"
-run-pytest = "run-coverage --no-cov"
-run-verbose = "run-coverage --verbose"
-run-mypy = "mypy src"
-list-env = "pip list"
-
-[tool.hatch.envs.upstream]
-python = "3.13"
-dependencies = [
-    'obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore',
-    'obspec @ git+https://github.com/developmentseed/obspec',
-]
-features = ["test"]
-
-[tool.hatch.envs.upstream.scripts]
-run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy"
-run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src"
-run-pytest = "run-coverage --no-cov"
-run-verbose = "run-coverage --verbose"
-run-mypy = "mypy src"
-list-env = "pip list"
-
 [tool.coverage.run]
 source_pkgs = ["obspec_utils", "tests"]
 branch = true

src/obspec_utils/__init__.py

@@ -1,3 +1,4 @@
 from ._version import __version__
+from .file_handlers import ObstoreReader
 
-__all__ = ["__version__"]
+__all__ = ["__version__", "ObstoreReader"]

src/obspec_utils/file_handlers.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import obstore as obs
+
+if TYPE_CHECKING:
+    from obstore import ReadableFile
+    from obstore.store import ObjectStore
+
+
+class ObstoreReader:
+    _reader: ReadableFile
+
+    def __init__(self, store: ObjectStore, path: str) -> None:
+        """
+        Create an obstore file reader that implements the read, readall, seek, and tell methods, which
+        can be used in libraries that expect file-like objects.
+
+        This wrapper is necessary in order to return Python bytes types rather than obstore Bytes buffers.
+
+        Parameters
+        ----------
+        store
+            [ObjectStore][obstore.store.ObjectStore] for reading the file.
+        path
+            The path to the file within the store. This should not include the prefix.
+        """
+        self._reader = obs.open_reader(store, path)
+
+    def read(self, size: int, /) -> bytes:
+        return self._reader.read(size).to_bytes()
+
+    def readall(self) -> bytes:
+        return self._reader.read().to_bytes()
+
+    def seek(self, offset: int, whence: int = 0, /):
+        # TODO: Check on default for whence
+        return self._reader.seek(offset, whence)
+
+    def tell(self) -> int:
+        return self._reader.tell()

tests/conftest.py

@@ -0,0 +1,93 @@
+import xarray as xr
+
+import json
+import time
+
+import pytest
+from pathlib import Path
+
+
+@pytest.fixture(scope="session")
+def container():
+    import docker
+
+    client = docker.from_env()
+    port = 9000
+    minio_container = client.containers.run(
+        "quay.io/minio/minio",
+        "server /data",
+        detach=True,
+        ports={f"{port}/tcp": port},
+        environment={
+            "MINIO_ACCESS_KEY": "minioadmin",
+            "MINIO_SECRET_KEY": "minioadmin",
+        },
+    )
+    time.sleep(3)  # give it time to boot
+    # enter
+    yield {
+        "port": port,
+        "endpoint": f"http://localhost:{port}",
+        "username": "minioadmin",
+        "password": "minioadmin",
+    }
+    # exit
+    minio_container.stop()
+    minio_container.remove()
+
+
+@pytest.fixture(scope="session")
+def minio_bucket(container):
+    # Setup with guidance from https://medium.com/@sant1/using-minio-with-docker-and-python-cbbad397cb5d
+    from minio import Minio
+
+    bucket = "my-bucket"
+    filename = "test.nc"
+    # Initialize MinIO client
+    client = Minio(
+        "localhost:9000",
+        access_key=container["username"],
+        secret_key=container["password"],
+        secure=False,
+    )
+    client.make_bucket(bucket)
+    policy = {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Principal": {"AWS": "*"},
+                "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
+                "Resource": "arn:aws:s3:::my-bucket",
+            },
+            {
+                "Effect": "Allow",
+                "Principal": {"AWS": "*"},
+                "Action": [
+                    "s3:GetObject",
+                    "s3:GetObjectRetention",
+                    "s3:GetObjectLegalHold",
+                ],
+                "Resource": "arn:aws:s3:::my-bucket/*",
+            },
+        ],
+    }
+    client.set_bucket_policy(bucket, json.dumps(policy))
+    yield {
+        "port": container["port"],
+        "endpoint": container["endpoint"],
+        "username": container["username"],
+        "password": container["password"],
+        "bucket": bucket,
+        "file": filename,
+        "client": client,
+    }
+
+
+@pytest.fixture
+def local_netcdf4_file(tmp_path: Path) -> str:
+    """Create a NetCDF4 file with data in multiple groups."""
+    filepath = tmp_path / "test.nc"
+    ds1 = xr.DataArray([1, 2, 3], name="foo").to_dataset()
+    ds1.to_netcdf(filepath)
+    return str(filepath)

tests/test_xarray.py

@@ -0,0 +1,10 @@
+import xarray as xr
+from obspec_utils import ObstoreReader
+from obstore.store import LocalStore
+
+
+def test_local_reader(local_netcdf4_file) -> None:
+    ds_fsspec = xr.open_dataset(local_netcdf4_file, engine="h5netcdf")
+    reader = ObstoreReader(store=LocalStore(), path=local_netcdf4_file)
+    ds_obstore = xr.open_dataset(reader, engine="h5netcdf")
+    xr.testing.assert_allclose(ds_fsspec, ds_obstore)