Zarr support (backend, in esmvalcore.preprocessor._io.py)

.github/workflows/run-tests.yml

@@ -21,6 +21,7 @@ on:
   push:
     branches:
       - main
+      - zarr_support
   # run the test only if the PR is to main
   # turn it on if required
   #pull_request:

environment.yml

@@ -5,6 +5,7 @@ channels:
   - nodefaults
 
 dependencies:
+  - aiohttp
   - cartopy
   - cf-units
   - cftime
@@ -18,6 +19,7 @@ dependencies:
   - fire
   - geopy
   - humanfriendly
+  - intake-esm
   - iris >=3.12.2  # https://github.com/SciTools/iris/issues/6417
   - iris-esmf-regrid >=0.11.0
   - iris-grib >=0.20.0  # github.com/ESMValGroup/ESMValCore/issues/2535
@@ -46,6 +48,7 @@ dependencies:
   - shapely >=2.0.0
   - xarray
   - yamale
+  - zarr >3
   # Python packages needed for building docs
   - autodocsumm >=0.2.2
   - ipython <9.0  # github.com/ESMValGroup/ESMValCore/issues/2680

esmvalcore/preprocessor/_io.py

@@ -9,6 +9,7 @@
 from itertools import groupby
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
+from urllib.parse import urlparse
 
 import iris
 import ncdata
@@ -83,6 +84,7 @@
     file:
         File to be loaded. If ``file`` is already a loaded dataset, return it
         as a :class:`~iris.cube.CubeList`.
+        File as ``Path`` object could be a Zarr store.
     ignore_warnings:
         Keyword arguments passed to :func:`warnings.filterwarnings` used to
         ignore warnings issued by :func:`iris.load_raw`. Each list element
@@ -102,7 +104,11 @@
 
     """
     if isinstance(file, (str, Path)):
-        cubes = _load_from_file(file, ignore_warnings=ignore_warnings)
+        if "zarr" not in str(file):
+            cubes = _load_from_file(file, ignore_warnings=ignore_warnings)
+        else:
+            zarr_xr = _load_zarr(file)
+            cubes = dataset_to_iris(zarr_xr, ignore_warnings=ignore_warnings)
     elif isinstance(file, Cube):
         cubes = CubeList([file])
     elif isinstance(file, CubeList):
@@ -134,6 +140,20 @@
     return cubes
 
 
+def _load_zarr(file):
+    if isinstance(file, Path):
+        zarr_xr = xr.open_zarr(file, consolidated=False)
+    elif urlparse(file):
+        zarr_xr = xr.open_dataset(
+            file,
+            consolidated=True,
+            use_cftime=True,
+            engine="zarr",
+        )
+
+    return zarr_xr
+
+
 def _load_from_file(
     file: str | Path,
     ignore_warnings: list[dict[str, Any]] | None = None,

pyproject.toml

@@ -32,6 +32,7 @@ dynamic = [
     "version",
 ]
 dependencies = [
+    "aiohttp",
     "cartopy",
     "cf-units",
     "dask[array,distributed]>=2025",  # Core/issues/2503
@@ -44,6 +45,7 @@ dependencies = [
     "fire",
     "geopy",
     "humanfriendly",
+    "intake-esm",
     "iris-grib>=0.20.0",  # github.com/ESMValGroup/ESMValCore/issues/2535
     "isodate>=0.7.0",
     "jinja2",
@@ -68,6 +70,7 @@ dependencies = [
     "stratify>=0.3",
     "xarray",
     "yamale",
+    "zarr>3",
 ]
 description = "A community tool for pre-processing data from Earth system models in CMIP and running analysis scripts"
 license = {text = "Apache License, Version 2.0"}

tests/integration/preprocessor/_io/test_load.py

@@ -4,6 +4,7 @@
 from importlib.resources import files as importlib_files
 from pathlib import Path
 
+import cf_units
 import iris
 import ncdata
 import numpy as np
@@ -120,6 +121,43 @@ def test_load_ncdata():
     assert not cube.coords()
 
 
+def test_load_zarr_local():
+    """Test loading a Zarr store as ncdata.NcData via Xarray."""
+    zarr_path = (
+        Path(importlib_files("tests"))
+        / "sample_data"
+        / "zarr-sample-data"
+        / "example_field_0.zarr2"
+    )
+
+    cubes = load(zarr_path)
+
+    assert len(cubes) == 1
+    cube = cubes[0]
+    assert cube.var_name == "q"
+    assert cube.standard_name == "specific_humidity"
+    assert cube.long_name is None
+    assert cube.units == cf_units.Unit("1")
+    coords = cube.coords()
+    coord_names = [coord.standard_name for coord in coords]
+    assert "longitude" in coord_names
+    assert "latitude" in coord_names
+
+
+def test_load_zarr_remote():
+    """Test loading a Zarr store from a https Object Store."""
+    zarr_path = (
+        "https://hackathon-o.s3-ext.jc.rl.ac.uk/sim-data/dev/v5/"
+        "glm.n2560_RAL3p3/um.PT1H.hp_z2.zarr"
+    )
+    cubes = load(zarr_path)
+    for cube in cubes:
+        if cube.standard_name == "air_temperature":
+            coords = cube.coords()
+            coord_names = [coord.standard_name for coord in coords]
+            assert "time" in coord_names
+
+
 def test_load_invalid_type_fail():
     """Test loading an invalid type."""
     with pytest.raises(TypeError):

tests/sample_data/zarr-sample-data/example_field_0.zarr2/.zattrs

@@ -0,0 +1,3 @@
+{
+    "Conventions": "CF-1.12"
+}

tests/sample_data/zarr-sample-data/example_field_0.zarr2/.zgroup

@@ -0,0 +1,3 @@
+{
+    "zarr_format": 2
+}

tests/sample_data/zarr-sample-data/example_field_0.zarr2/.zmetadata

@@ -0,0 +1,171 @@
+{
+    "metadata": {
+        ".zattrs": {
+            "Conventions": "CF-1.12"
+        },
+        ".zgroup": {
+            "zarr_format": 2
+        },
+        "lat/.zarray": {
+            "chunks": [
+                5
+            ],
+            "compressor": {
+                "blocksize": 0,
+                "clevel": 5,
+                "cname": "lz4",
+                "id": "blosc",
+                "shuffle": 1
+            },
+            "dtype": "<f8",
+            "fill_value": "NaN",
+            "filters": null,
+            "order": "C",
+            "shape": [
+                5
+            ],
+            "zarr_format": 2
+        },
+        "lat/.zattrs": {
+            "_ARRAY_DIMENSIONS": [
+                "lat"
+            ],
+            "bounds": "lat_bnds",
+            "standard_name": "latitude",
+            "units": "degrees_north"
+        },
+        "lat_bnds/.zarray": {
+            "chunks": [
+                3,
+                2
+            ],
+            "compressor": {
+                "blocksize": 0,
+                "clevel": 5,
+                "cname": "lz4",
+                "id": "blosc",
+                "shuffle": 1
+            },
+            "dtype": "<f8",
+            "fill_value": "NaN",
+            "filters": null,
+            "order": "C",
+            "shape": [
+                5,
+                2
+            ],
+            "zarr_format": 2
+        },
+        "lat_bnds/.zattrs": {
+            "_ARRAY_DIMENSIONS": [
+                "lat",
+                "bounds2"
+            ]
+        },
+        "lon/.zarray": {
+            "chunks": [
+                8
+            ],
+            "compressor": {
+                "blocksize": 0,
+                "clevel": 5,
+                "cname": "lz4",
+                "id": "blosc",
+                "shuffle": 1
+            },
+            "dtype": "<f8",
+            "fill_value": "NaN",
+            "filters": null,
+            "order": "C",
+            "shape": [
+                8
+            ],
+            "zarr_format": 2
+        },
+        "lon/.zattrs": {
+            "_ARRAY_DIMENSIONS": [
+                "lon"
+            ],
+            "bounds": "lon_bnds",
+            "standard_name": "longitude",
+            "units": "degrees_east"
+        },
+        "lon_bnds/.zarray": {
+            "chunks": [
+                4,
+                2
+            ],
+            "compressor": {
+                "blocksize": 0,
+                "clevel": 5,
+                "cname": "lz4",
+                "id": "blosc",
+                "shuffle": 1
+            },
+            "dtype": "<f8",
+            "fill_value": "NaN",
+            "filters": null,
+            "order": "C",
+            "shape": [
+                8,
+                2
+            ],
+            "zarr_format": 2
+        },
+        "lon_bnds/.zattrs": {
+            "_ARRAY_DIMENSIONS": [
+                "lon",
+                "bounds2"
+            ]
+        },
+        "q/.zarray": {
+            "chunks": [
+                3,
+                4
+            ],
+            "compressor": {
+                "blocksize": 0,
+                "clevel": 5,
+                "cname": "lz4",
+                "id": "blosc",
+                "shuffle": 1
+            },
+            "dtype": "<f8",
+            "fill_value": "NaN",
+            "filters": null,
+            "order": "C",
+            "shape": [
+                5,
+                8
+            ],
+            "zarr_format": 2
+        },
+        "q/.zattrs": {
+            "_ARRAY_DIMENSIONS": [
+                "lat",
+                "lon"
+            ],
+            "cell_methods": "area: mean",
+            "coordinates": "time",
+            "project": "research",
+            "standard_name": "specific_humidity",
+            "units": "1"
+        },
+        "time/.zarray": {
+            "chunks": [],
+            "compressor": null,
+            "dtype": "<f8",
+            "fill_value": "NaN",
+            "filters": null,
+            "order": "C",
+            "shape": [],
+            "zarr_format": 2
+        },
+        "time/.zattrs": {
+            "_ARRAY_DIMENSIONS": [],
+            "standard_name": "time",
+            "units": "days since 2018-12-01"
+        }
+    },
+    "zarr_consolidated_format": 1
+}

tests/sample_data/zarr-sample-data/example_field_0.zarr2/lat/.zarray

@@ -0,0 +1,20 @@
+{
+    "chunks": [
+        5
+    ],
+    "compressor": {
+        "blocksize": 0,
+        "clevel": 5,
+        "cname": "lz4",
+        "id": "blosc",
+        "shuffle": 1
+    },
+    "dtype": "<f8",
+    "fill_value": "NaN",
+    "filters": null,
+    "order": "C",
+    "shape": [
+        5
+    ],
+    "zarr_format": 2
+}

tests/sample_data/zarr-sample-data/example_field_0.zarr2/lat/.zattrs

@@ -0,0 +1,8 @@
+{
+    "_ARRAY_DIMENSIONS": [
+        "lat"
+    ],
+    "bounds": "lat_bnds",
+    "standard_name": "latitude",
+    "units": "degrees_north"
+}

tests/sample_data/zarr-sample-data/example_field_0.zarr2/lat_bnds/.zarray

@@ -0,0 +1,22 @@
+{
+    "chunks": [
+        3,
+        2
+    ],
+    "compressor": {
+        "blocksize": 0,
+        "clevel": 5,
+        "cname": "lz4",
+        "id": "blosc",
+        "shuffle": 1
+    },
+    "dtype": "<f8",
+    "fill_value": "NaN",
+    "filters": null,
+    "order": "C",
+    "shape": [
+        5,
+        2
+    ],
+    "zarr_format": 2
+}

tests/sample_data/zarr-sample-data/example_field_0.zarr2/lat_bnds/.zattrs

@@ -0,0 +1,6 @@
+{
+    "_ARRAY_DIMENSIONS": [
+        "lat",
+        "bounds2"
+    ]
+}