diff --git a/.gitignore b/.gitignore index 959f966d..4cea06b9 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,4 @@ cdk.out/ node_modules cdk.context.json *.nc +.DS_Store diff --git a/README.md b/README.md index 72717068..4f1bd7db 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ Example of application built with `titiler.xarray` [package](https://development # It's recommended to install dependencies in a virtual environment uv sync --dev export TEST_ENVIRONMENT=true # set this when running locally to mock redis +#optional: Disable caching +#export TITILER_MULTIDIM_ENABLE_CACHE=false uv run uvicorn titiler.multidim.main:app --reload ``` diff --git a/pyproject.toml b/pyproject.toml index 47a2b9a9..af8e3714 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,8 +41,8 @@ dependencies = [ "requests", "rioxarray", "s3fs", - "xarray", - "zarr>=2,<3", + "xarray>2025.07.1", + "zarr>3.1.0", ] [project.optional-dependencies] @@ -56,6 +56,7 @@ lambda = [ [dependency-groups] dev = [ + "dask>=2025.9.1", "fakeredis>=2.23.5", "httpx", "ipykernel>=6.30.1", diff --git a/src/titiler/multidim/reader.py b/src/titiler/multidim/reader.py index 73477235..05b2cf42 100644 --- a/src/titiler/multidim/reader.py +++ b/src/titiler/multidim/reader.py @@ -34,6 +34,8 @@ def __attrs_post_init__(self): group=self.group, decode_times=self.decode_times, ) + print(f"DEBUG: Dataset id {id(self.ds)} opened from {self.src_path}") + print(f"DEBUG: {api_settings.enable_cache=}") if not ds and api_settings.enable_cache: # Serialize the dataset to bytes using pickle diff --git a/tests/conftest.py b/tests/conftest.py index 09a793f2..733ae255 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,15 +1,37 @@ -"""titiler.multidim tests configuration.""" +"""Auto-parametrized fixture that runs both cache configurations.""" +import sys import pytest from fastapi.testclient import TestClient -@pytest.fixture -def app(monkeypatch): - """App fixture.""" +# This fixture will automatically parametrize ALL tests that use it +@pytest.fixture( + params=[ + pytest.param({"cache": True}, id="with_cache"), + pytest.param({"cache": False}, id="without_cache"), + ] +) +def app(request, monkeypatch): + """Auto-parametrized app fixture that runs tests with both cache configurations.""" + config = request.param + enable_cache = config.get("cache", False) + + # Set environment variables using monkeypatch (auto-cleanup) monkeypatch.setenv("TITILER_MULTIDIM_DEBUG", "TRUE") monkeypatch.setenv("TEST_ENVIRONMENT", "1") + monkeypatch.setenv( + "TITILER_MULTIDIM_ENABLE_CACHE", "TRUE" if enable_cache else "FALSE" + ) + + # Clear module cache to ensure fresh import + modules_to_clear = [ + key for key in sys.modules.keys() if key.startswith("titiler.multidim") + ] + for module in modules_to_clear: + del sys.modules[module] + # Import and return the app from titiler.multidim.main import app with TestClient(app) as client: diff --git a/tests/fixtures/generate_test_zarr.py b/tests/fixtures/generate_test_zarr.py index 78900bee..66448207 100644 --- a/tests/fixtures/generate_test_zarr.py +++ b/tests/fixtures/generate_test_zarr.py @@ -1,4 +1,4 @@ -"""Create zarr fixture.""" +"""Create zarr fixtures for v2 and v3.""" import numpy as np import xarray as xr @@ -8,31 +8,32 @@ time_dim = 10 lat_dim = 36 lon_dim = 72 -chunk_size = (10, 10, 10) +chunk_size = {"time": 10, "lat": 10, "lon": 10} # Create coordinates time = np.arange(time_dim) -lat = np.linspace(-90 + res / 2, 90 - res / 2, lat_dim) -lon = np.linspace(-180 + res / 2, 180 - res / 2, lon_dim) +lat = np.linspace(-90.0 + res / 2, 90.0 - res / 2, lat_dim) +lon = np.linspace(-180.0 + res / 2, 180.0 - res / 2, lon_dim) +dtype = np.float64 # Initialize variables with random data CDD0 = xr.DataArray( - np.random.rand(time_dim, lat_dim, lon_dim).astype(np.uint8), + np.random.rand(time_dim, lat_dim, lon_dim).astype(dtype), dims=("time", "lat", "lon"), name="CDD0", ) DISPH = xr.DataArray( - np.random.rand(time_dim, lat_dim, lon_dim).astype(np.uint8), + np.random.rand(time_dim, lat_dim, lon_dim).astype(dtype), dims=("time", "lat", "lon"), name="DISPH", ) FROST_DAYS = xr.DataArray( - np.random.rand(time_dim, lat_dim, lon_dim).astype(np.uint8), + np.random.rand(time_dim, lat_dim, lon_dim).astype(dtype), dims=("time", "lat", "lon"), name="FROST_DAYS", ) GWETPROF = xr.DataArray( - np.random.rand(time_dim, lat_dim, lon_dim).astype(np.uint8), + np.random.rand(time_dim, lat_dim, lon_dim).astype(dtype), dims=("time", "lat", "lon"), name="GWETPROF", ) @@ -49,4 +50,17 @@ ) # Save dataset to a local Zarr store -ds.to_zarr("tests/fixtures/test_zarr_store.zarr", mode="w") +ds.to_zarr( + "tests/fixtures/zarr_store_v3.zarr", + mode="w", + zarr_format=3, + consolidated=False, +) + +# Save dataset to a local Zarr store +ds.to_zarr( + "tests/fixtures/zarr_store_v2.zarr", + mode="w", + zarr_format=2, + consolidated=True, +) diff --git a/tests/fixtures/responses/test_zarr_store_zarr_histogram.json b/tests/fixtures/responses/test_zarr_store_zarr_histogram.json deleted file mode 100644 index 384908e6..00000000 --- a/tests/fixtures/responses/test_zarr_store_zarr_histogram.json +++ /dev/null @@ -1,72 +0,0 @@ -[ - { - "bucket": [ - -0.5, - -0.4 - ], - "value": 0 - }, - { - "bucket": [ - -0.4, - -0.3 - ], - "value": 0 - }, - { - "bucket": [ - -0.3, - -0.19999999999999996 - ], - "value": 0 - }, - { - "bucket": [ - -0.19999999999999996, - -0.09999999999999998 - ], - "value": 0 - }, - { - "bucket": [ - -0.09999999999999998, - 0.0 - ], - "value": 0 - }, - { - "bucket": [ - 0.0, - 0.10000000000000009 - ], - "value": 2592 - }, - { - "bucket": [ - 0.10000000000000009, - 0.20000000000000007 - ], - "value": 0 - }, - { - "bucket": [ - 0.20000000000000007, - 0.30000000000000004 - ], - "value": 0 - }, - { - "bucket": [ - 0.30000000000000004, - 0.4 - ], - "value": 0 - }, - { - "bucket": [ - 0.4, - 0.5 - ], - "value": 0 - } -] \ No newline at end of file diff --git a/tests/fixtures/responses/test_zarr_store_zarr_info.json b/tests/fixtures/responses/test_zarr_store_zarr_info.json deleted file mode 100644 index b8be05e7..00000000 --- a/tests/fixtures/responses/test_zarr_store_zarr_info.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "bounds": [-180.0, -90.0, 180.0, 90.0], - "crs": "http://www.opengis.net/def/crs/EPSG/0/4326", - "band_metadata": [["b1", {}]], - "band_descriptions": [["b1", "0"]], - "dtype": "uint8", - "nodata_type": "None", - "name": "CDD0", - "count": 1, - "width": 72, - "height": 36, - "attrs": {}, - "dimensions": ["y", "x"] -} diff --git a/tests/fixtures/responses/test_zarr_store_info.json b/tests/fixtures/responses/zarr_store_v2_zarr_info.json similarity index 100% rename from tests/fixtures/responses/test_zarr_store_info.json rename to tests/fixtures/responses/zarr_store_v2_zarr_info.json diff --git a/tests/fixtures/responses/test_zarr_store_zarr_tilejson.json b/tests/fixtures/responses/zarr_store_v2_zarr_tilejson.json similarity index 74% rename from tests/fixtures/responses/test_zarr_store_zarr_tilejson.json rename to tests/fixtures/responses/zarr_store_v2_zarr_tilejson.json index 497b8ff6..bcbee613 100644 --- a/tests/fixtures/responses/test_zarr_store_zarr_tilejson.json +++ b/tests/fixtures/responses/zarr_store_v2_zarr_tilejson.json @@ -3,7 +3,7 @@ "version": "1.0.0", "scheme": "xyz", "tiles": [ - "http://testserver/tiles/WebMercatorQuad/{z}/{x}/{y}@1x?url=tests%2Ffixtures%2Ftest_zarr_store.zarr&variable=CDD0&decode_times=false&sel=time%3D0" + "http://testserver/tiles/WebMercatorQuad/{z}/{x}/{y}@1x?url=tests%2Ffixtures%2Fzarr_store_v2.zarr&variable=CDD0&decode_times=false&sel=time%3D0" ], "minzoom": 0, "maxzoom": 0, diff --git a/tests/fixtures/responses/zarr_store_v3_zarr_info.json b/tests/fixtures/responses/zarr_store_v3_zarr_info.json new file mode 100644 index 00000000..b8e44170 --- /dev/null +++ b/tests/fixtures/responses/zarr_store_v3_zarr_info.json @@ -0,0 +1,12 @@ +{ + "bounds": [-180, -90.0, 180.0, 90.0], + "band_metadata": [], + "band_descriptions": [], + "dtype": "float64", + "nodata_type": "None", + "height": 36, + "count": 1, + "width": 72, + "attrs": {}, + "name": "CDD0" +} diff --git a/tests/fixtures/responses/zarr_store_v3_zarr_tilejson.json b/tests/fixtures/responses/zarr_store_v3_zarr_tilejson.json new file mode 100644 index 00000000..ba29e301 --- /dev/null +++ b/tests/fixtures/responses/zarr_store_v3_zarr_tilejson.json @@ -0,0 +1,12 @@ +{ + "tilejson": "2.2.0", + "version": "1.0.0", + "scheme": "xyz", + "tiles": [ + "http://testserver/tiles/WebMercatorQuad/{z}/{x}/{y}@1x?url=tests%2Ffixtures%2Fzarr_store_v3.zarr&variable=CDD0&decode_times=false&sel=time%3D0" + ], + "minzoom": 0, + "maxzoom": 0, + "bounds": [-180.0, -90.0, 180.0, 90.0], + "center": [0.0, 0.0, 0] +} diff --git a/tests/fixtures/test_zarr_store.zarr/.zgroup b/tests/fixtures/test_zarr_store.zarr/.zgroup deleted file mode 100644 index 3b7daf22..00000000 --- a/tests/fixtures/test_zarr_store.zarr/.zgroup +++ /dev/null @@ -1,3 +0,0 @@ -{ - "zarr_format": 2 -} \ No newline at end of file diff --git a/tests/fixtures/test_zarr_store.zarr/.zmetadata b/tests/fixtures/test_zarr_store.zarr/.zmetadata deleted file mode 100644 index 5e3a9d2c..00000000 --- a/tests/fixtures/test_zarr_store.zarr/.zmetadata +++ /dev/null @@ -1,208 +0,0 @@ -{ - "metadata": { - ".zattrs": {}, - ".zgroup": { - "zarr_format": 2 - }, - "CDD0/.zarray": { - "chunks": [ - 10, - 10, - 10 - ], - "compressor": { - "blocksize": 0, - "clevel": 5, - "cname": "lz4", - "id": "blosc", - "shuffle": 1 - }, - "dtype": "|u1", - "fill_value": null, - "filters": null, - "order": "C", - "shape": [ - 10, - 36, - 72 - ], - "zarr_format": 2 - }, - "CDD0/.zattrs": { - "_ARRAY_DIMENSIONS": [ - "time", - "lat", - "lon" - ] - }, - "DISPH/.zarray": { - "chunks": [ - 10, - 10, - 10 - ], - "compressor": { - "blocksize": 0, - "clevel": 5, - "cname": "lz4", - "id": "blosc", - "shuffle": 1 - }, - "dtype": "|u1", - "fill_value": null, - "filters": null, - "order": "C", - "shape": [ - 10, - 36, - 72 - ], - "zarr_format": 2 - }, - "DISPH/.zattrs": { - "_ARRAY_DIMENSIONS": [ - "time", - "lat", - "lon" - ] - }, - "FROST_DAYS/.zarray": { - "chunks": [ - 10, - 10, - 10 - ], - "compressor": { - "blocksize": 0, - "clevel": 5, - "cname": "lz4", - "id": "blosc", - "shuffle": 1 - }, - "dtype": "|u1", - "fill_value": null, - "filters": null, - "order": "C", - "shape": [ - 10, - 36, - 72 - ], - "zarr_format": 2 - }, - "FROST_DAYS/.zattrs": { - "_ARRAY_DIMENSIONS": [ - "time", - "lat", - "lon" - ] - }, - "GWETPROF/.zarray": { - "chunks": [ - 10, - 10, - 10 - ], - "compressor": { - "blocksize": 0, - "clevel": 5, - "cname": "lz4", - "id": "blosc", - "shuffle": 1 - }, - "dtype": "|u1", - "fill_value": null, - "filters": null, - "order": "C", - "shape": [ - 10, - 36, - 72 - ], - "zarr_format": 2 - }, - "GWETPROF/.zattrs": { - "_ARRAY_DIMENSIONS": [ - "time", - "lat", - "lon" - ] - }, - "lat/.zarray": { - "chunks": [ - 36 - ], - "compressor": { - "blocksize": 0, - "clevel": 5, - "cname": "lz4", - "id": "blosc", - "shuffle": 1 - }, - "dtype": "