Merge branch 'main' into fix/zarrv2_str_fillval

Author: observingClouds

conftest.py

@@ -214,6 +214,17 @@ def netcdf4_file_with_data_in_multiple_groups(tmp_path: Path) -> str:
     return str(filepath)
 
 
+@pytest.fixture
+def netcdf4_file_with_data_in_sibling_groups(tmp_path: Path) -> str:
+    """Create a NetCDF4 file with data in sibling groups."""
+    filepath = tmp_path / "test.nc"
+    ds1 = xr.DataArray([1, 2, 3], name="foo").to_dataset()
+    ds1.to_netcdf(filepath, group="subgroup1")
+    ds2 = xr.DataArray([4, 5], coords={"x": [0, 1]}, dims="x", name="bar").to_dataset()
+    ds2.to_netcdf(filepath, group="subgroup2", mode="a")
+    return str(filepath)
+
+
 @pytest.fixture
 def netcdf4_files_factory(tmp_path: Path) -> Callable[[], tuple[str, str]]:
     """Factory fixture to create multiple NetCDF4 files."""

docs/api/virtualizarr.md

@@ -9,6 +9,9 @@ Users can use xarray for every step apart from reading and serializing virtual r
 
 ::: virtualizarr.open_virtual_mfdataset
 
+::: virtualizarr.open_virtual_datatree
+
+
 ## Information
 
 ::: virtualizarr.accessor.VirtualiZarrDatasetAccessor.nbytes

docs/releases.md

@@ -1,26 +1,37 @@
 # Release notes
 
-## Unreleased
+## unreleased
+
+### Bug fixes
+
+- Fix setting `fill_value` for Zarr V2 arrays if data type is a subtype of integer or float.
+  ([#845](https://github.com/zarr-developers/VirtualiZarr/pull/845)).
+  By [Hauke Schulz](https://github.com/observingClouds).
+
+## v2.3.0 (20th January 2026)
 
 ### New Features
 
+- Implement `open_virtual_datatree`.
+  ([838](https://github.com/zarr-developers/VirtualiZarr/pull/838)).
+  By [Max Jones](https://github.com/maxrjones).
 - Set `supports_consolidated_metadata` property on `ManifestStore` to `False`.
   ([809](https://github.com/zarr-developers/VirtualiZarr/pull/809)).
   By [Julia Signell](https://github.com/jsignell).
 
-### Bug fixes
+### Internal changes
 
-- Fix setting `fill_value` for Zarr V2 arrays if data type is a subtype of integer or float.
-  ([#845](https://github.com/zarr-developers/VirtualiZarr/pull/845)).
-  By [Hauke Schulz](https://github.com/observingClouds).
+- Remove the undocumented/unfunctional wrapper of Kerchunk's TIFF parser.
+  ([849](https://github.com/zarr-developers/VirtualiZarr/pull/849)).
+  By [Max Jones](https://github.com/maxrjones).
 
 ## v2.2.1 (17th November 2025)
 
 ### Bug fixes
 
 - Allow storing scalar arrays under 'c' key. ([#836](https://github.com/zarr-developers/VirtualiZarr/pull/836)).
   By [Max Jones](https://github.com/maxrjones)
-- Improve ManifestStore.list_dir for arrays and nested groups ([#837](https://github.com/zarr-developers/VirtualiZarr/pull/837))
+- Improve ManifestStore.list_dir for arrays and nested groups. ([#837](https://github.com/zarr-developers/VirtualiZarr/pull/837))
   By [Max Jones](https://github.com/maxrjones)
 
 ## v2.2.0 (12th November 2025)
@@ -67,6 +78,7 @@ Patch release with minor bug fixes for the DMRPParser and Icechunk writing behav
 - Support dtypes without an endianness ([#787](https://github.com/zarr-developers/VirtualiZarr/pull/787)). By [Justus Magin](https://github.com/keewis).
 
 ### Internal changes
+
 - Change default Icechunk writing behavior to not validate or write "empty" chunks ([#791](https://github.com/zarr-developers/VirtualiZarr/pull/791)). By [Sean Harkins](https://github.com/sharkinsspatial).
 
 ## v2.1.1 (14th August 2025)
@@ -151,10 +163,10 @@ Minor release to ensure compatibility with incoming changes to Icechunk.
 - Added [`open_virtual_mfdataset`][virtualizarr.open_virtual_mfdataset] function ([#345](https://github.com/zarr-developers/VirtualiZarr/issues/345), [#349](https://github.com/zarr-developers/VirtualiZarr/pull/349)).
   By [Tom Nicholas](https://github.com/TomNicholas).
 - Added `datatree_to_icechunk` function for writing an `xarray.DataTree` to
-  an Icechunk store ([#244](https://github.com/zarr-developers/VirtualiZarr/issues/244)).  By [Chuck Daniels](https://github.com/chuckwondo).
+  an Icechunk store ([#244](https://github.com/zarr-developers/VirtualiZarr/issues/244)). By [Chuck Daniels](https://github.com/chuckwondo).
 - Added a `.vz` custom accessor to `xarray.DataTree`, exposing the method
   `xarray.DataTree.vz.to_icechunk()` for writing an `xarray.DataTree`
-  to an Icechunk store ([#244](https://github.com/zarr-developers/VirtualiZarr/issues/244)).  By
+  to an Icechunk store ([#244](https://github.com/zarr-developers/VirtualiZarr/issues/244)). By
   [Chuck Daniels](https://github.com/chuckwondo).
 - Added a warning if you attempt to write an entirely non-virtual dataset to a virtual references format ([#657](https://github.com/zarr-developers/VirtualiZarr/pull/657)).
   By [Tom Nicholas](https://github.com/TomNicholas).
@@ -213,16 +225,16 @@ Minor release to ensure compatibility with incoming changes to Icechunk.
 ### Internal Changes
 
 - `ManifestArrays` now internally use [zarr.core.metadata.v3.ArrayV3Metadata](https://github.com/zarr-developers/zarr-python/blob/v3.0.2/src/zarr/core/metadata/v3.py). This replaces the `ZArray` class that was previously used to store metadata about manifest arrays. ([#429](https://github.com/zarr-developers/VirtualiZarr/pull/429)) By [Aimee Barciauskas](https://github.com/abarciauskas-bgse). Notable internal changes:
-    - Make zarr-python a required dependency with a minimum version `>=3.0.2`.
-    - Specify a minimum numcodecs version of `>=0.15.1`.
-    - When creating a `ManifestArray`, the `metadata` property should be an `zarr.core.metadata.v3.ArrayV3Metadata` object. There is a helper function `create_v3_array_metadata` which should be used, as it has some useful defaults and includes `convert_to_codec_pipeline` (see next bullet).
-    - The function `convert_to_codec_pipeline` ensures the codec pipeline passed to `ArrayV3Metadata` has valid codecs in the expected order (`ArrayArrayCodec`s, `ArrayBytesCodec`, `BytesBytesCodec`s) and includes the required `ArrayBytesCodec` using the default for the data type.
-      - Note: `convert_to_codec_pipeline` uses the zarr-python function `get_codec_class` to convert codec configurations (i.e. `dict`s with a name and configuration key, see [parse_named_configuration](https://github.com/zarr-developers/zarr-python/blob/v3.0.2/src/zarr/core/common.py#L116-L130)) to valid Zarr V3 codec classes.
-    - Parser changes are minimal.
-    - Writer changes:
-      - Kerchunk uses Zarr version format 2 so we convert `ArrayV3Metadata` to `ArrayV2Metadata` using the `convert_v3_to_v2_metadata` function. This means the `to_kerchunk_json` function is now a bit more complex because we're converting `ArrayV2Metadata` filters and compressor to serializable objects.
-    - zarr-python 3.0 does not yet support the big endian data type. This means that FITS and NetCDF-3 are not currently supported ([zarr-python issue #2324](https://github.com/zarr-developers/zarr-python/issues/2324)).
-    - zarr-python 3.0 does not yet support datetime and timedelta data types ([zarr-python issue #2616](https://github.com/zarr-developers/zarr-python/issues/2616)).
+  - Make zarr-python a required dependency with a minimum version `>=3.0.2`.
+  - Specify a minimum numcodecs version of `>=0.15.1`.
+  - When creating a `ManifestArray`, the `metadata` property should be an `zarr.core.metadata.v3.ArrayV3Metadata` object. There is a helper function `create_v3_array_metadata` which should be used, as it has some useful defaults and includes `convert_to_codec_pipeline` (see next bullet).
+  - The function `convert_to_codec_pipeline` ensures the codec pipeline passed to `ArrayV3Metadata` has valid codecs in the expected order (`ArrayArrayCodec`s, `ArrayBytesCodec`, `BytesBytesCodec`s) and includes the required `ArrayBytesCodec` using the default for the data type.
+    - Note: `convert_to_codec_pipeline` uses the zarr-python function `get_codec_class` to convert codec configurations (i.e. `dict`s with a name and configuration key, see [parse_named_configuration](https://github.com/zarr-developers/zarr-python/blob/v3.0.2/src/zarr/core/common.py#L116-L130)) to valid Zarr V3 codec classes.
+  - Parser changes are minimal.
+  - Writer changes:
+    - Kerchunk uses Zarr version format 2 so we convert `ArrayV3Metadata` to `ArrayV2Metadata` using the `convert_v3_to_v2_metadata` function. This means the `to_kerchunk_json` function is now a bit more complex because we're converting `ArrayV2Metadata` filters and compressor to serializable objects.
+  - zarr-python 3.0 does not yet support the big endian data type. This means that FITS and NetCDF-3 are not currently supported ([zarr-python issue #2324](https://github.com/zarr-developers/zarr-python/issues/2324)).
+  - zarr-python 3.0 does not yet support datetime and timedelta data types ([zarr-python issue #2616](https://github.com/zarr-developers/zarr-python/issues/2616)).
 - The continuous integration workflows and developer environment now use [pixi](https://pixi.sh/latest/) ([#407](https://github.com/zarr-developers/VirtualiZarr/pull/407)).
 - Added `loadable_variables` kwarg to `ManifestStore.to_virtual_dataset`.
   ([#543](https://github.com/zarr-developers/VirtualiZarr/pull/543)) By [Tom Nicholas](https://github.com/TomNicholas).
@@ -284,14 +296,14 @@ It also fixes a number of bugs, adds minor features, changes the default reader
 - Added a `.nbytes` accessor method which displays the bytes needed to hold the virtual references in memory.
   ([#167](https://github.com/zarr-developers/VirtualiZarr/issues/167), [#227](https://github.com/zarr-developers/VirtualiZarr/pull/227)) By [Tom Nicholas](https://github.com/TomNicholas).
 - Upgrade icechunk dependency to `>=0.1.0a12`. ([#406](https://github.com/zarr-developers/VirtualiZarr/pull/406)) By [Julia Signell](https://github.com/jsignell).
-- Sync with Icechunk v0.1.0a8  ([#368](https://github.com/zarr-developers/VirtualiZarr/pull/368)) By [Matthew Iannucci](https://github.com/mpiannucci). This also adds support
+- Sync with Icechunk v0.1.0a8 ([#368](https://github.com/zarr-developers/VirtualiZarr/pull/368)) By [Matthew Iannucci](https://github.com/mpiannucci). This also adds support
   for the `to_icechunk` method to add timestamps as checksums when writing virtual references to an icechunk store. This
   is useful for ensuring that virtual references are not stale when reading from an icechunk store, which can happen if the
   underlying data has changed since the virtual references were written.
 - Add `group=None` keyword-only parameter to the
   `VirtualiZarrDatasetAccessor.to_icechunk` method to allow writing to a nested group
   at a specified group path (rather than defaulting to the root group, when no group is
-  specified).  ([#341](https://github.com/zarr-developers/VirtualiZarr/issues/341)) By [Chuck Daniels](https://github.com/chuckwondo).
+  specified). ([#341](https://github.com/zarr-developers/VirtualiZarr/issues/341)) By [Chuck Daniels](https://github.com/chuckwondo).
 
 ### Breaking changes
 
@@ -304,8 +316,8 @@ It also fixes a number of bugs, adds minor features, changes the default reader
   ([#18](https://github.com/zarr-developers/VirtualiZarr/issues/18), [#357](https://github.com/zarr-developers/VirtualiZarr/pull/357), [#358](https://github.com/zarr-developers/VirtualiZarr/pull/358)) By [Tom Nicholas](https://github.com/TomNicholas).
 - The `append_dim` and `last_updated_at` parameters of the
   `VirtualiZarrDatasetAccessor.to_icechunk` method are now keyword-only parameters,
-  rather than positional or keyword.  This change is breaking _only_ where arguments for
-  these parameters are currently given positionally.  ([#341](https://github.com/zarr-developers/VirtualiZarr/issues/341)) By
+  rather than positional or keyword. This change is breaking _only_ where arguments for
+  these parameters are currently given positionally. ([#341](https://github.com/zarr-developers/VirtualiZarr/issues/341)) By
   [Chuck Daniels](https://github.com/chuckwondo).
 - The default backend for netCDF4 and HDF5 is now the custom `HDFVirtualBackend` replacing
   the previous default which was a wrapper around the kerchunk backend.
@@ -322,7 +334,7 @@ It also fixes a number of bugs, adds minor features, changes the default reader
   ([#336](https://github.com/zarr-developers/VirtualiZarr/issues/336), [#338](https://github.com/zarr-developers/VirtualiZarr/pull/338)) By [Tom Nicholas](https://github.com/TomNicholas).
 - Fix bug in HDF reader where dimension names of dimensions in a subgroup would be incorrect.
   ([#364](https://github.com/zarr-developers/VirtualiZarr/issues/364), [#366](https://github.com/zarr-developers/VirtualiZarr/pull/366)) By [Tom Nicholas](https://github.com/TomNicholas).
-- Fix bug in dmrpp reader so _FillValue is included in variables' encodings.
+- Fix bug in dmrpp reader so \_FillValue is included in variables' encodings.
   ([#369](https://github.com/zarr-developers/VirtualiZarr/pull/369)) By [Aimee Barciauskas](https://github.com/abarciauskas-bgse).
 - Fix bug passing arguments to FITS reader, and test it on Hubble Space Telescope data.
   ([#363](https://github.com/zarr-developers/VirtualiZarr/pull/363)) By [Tom Nicholas](https://github.com/TomNicholas).

mkdocs.yml

@@ -130,6 +130,7 @@ plugins:
             - https://icechunk.io/en/stable/objects.inv
             - https://lithops-cloud.github.io/docs/objects.inv
             - https://docs.dask.org/en/stable/objects.inv
+            - https://virtual-tiff.readthedocs.io/en/latest/objects.inv
 # https://github.com/developmentseed/titiler/blob/50934c929cca2fa8d3c408d239015f8da429c6a8/docs/mkdocs.yml#L115-L140
 markdown_extensions:
   - admonition

pyproject.toml

@@ -101,6 +101,7 @@ upstream = [
     's3fs @ git+https://github.com/fsspec/s3fs',
     'kerchunk @ git+https://github.com/fsspec/kerchunk',
     'icechunk @ git+https://github.com/earth-mover/icechunk#subdirectory=icechunk-python',
+    'virtual_tiff @ git+https://github.com/virtual-zarr/virtual-tiff',
 ]
 docs = [
     "mkdocs-material[imaging]>=9.6.14",

virtualizarr/__init__.py

@@ -4,7 +4,11 @@
     VirtualiZarrDatasetAccessor,
     VirtualiZarrDataTreeAccessor,
 )
-from virtualizarr.xarray import open_virtual_dataset, open_virtual_mfdataset
+from virtualizarr.xarray import (
+    open_virtual_dataset,
+    open_virtual_datatree,
+    open_virtual_mfdataset,
+)
 
 try:
     __version__ = _version("virtualizarr")
@@ -18,4 +22,5 @@
     "VirtualiZarrDataTreeAccessor",
     "open_virtual_dataset",
     "open_virtual_mfdataset",
+    "open_virtual_datatree",
 ]

virtualizarr/manifests/group.py

@@ -125,3 +125,36 @@ def to_virtual_dataset(self) -> xr.Dataset:
             coord_names=coord_names,
             attrs=attributes,
         )
+
+    def to_virtual_datasets(self) -> Mapping[str, xr.Dataset]:
+        """
+        Create a dictionary containing virtual datasets for all the sub-groups of a ManifestGroup. All the
+        variables in the datasets will be "virtual", i.e., they will wrap ManifestArray objects.
+
+        It is convenient to have a separate `to_virtual_datasets` function from `to_virtual_datatree` so that
+        it can be called recursively without needing to use `DataTree.to_dict() and `.from_dict()` repeatedly.
+        """
+        result = {"": self.to_virtual_dataset()}
+
+        # Recursively process all subgroups
+        for group_name, subgroup in self.groups.items():
+            subgroup_datasets = subgroup.to_virtual_datasets()
+
+            # Add the subgroup's datasets with proper path prefixes
+            for subpath, dataset in subgroup_datasets.items():
+                if subpath == "":
+                    # Direct child group
+                    full_path = group_name
+                else:
+                    # Nested subgroup
+                    full_path = f"{group_name}/{subpath}"
+                result[full_path] = dataset
+        return result
+
+    def to_virtual_datatree(self) -> xr.DataTree:
+        """
+        Create a "virtual" [xarray.DataTree][] containing the contents of one zarr group.
+
+        All variables in the returned DataTree will be "virtual", i.e. they will wrap ManifestArray objects.
+        """
+        return xr.DataTree.from_dict(self.to_virtual_datasets())

virtualizarr/manifests/store.py

@@ -332,6 +332,42 @@ def to_virtual_dataset(
             decode_times=decode_times,
         )
 
+    def to_virtual_datatree(
+        self,
+        group="",
+        *,
+        loadable_variables: Iterable[str] | None = None,
+        decode_times: bool | None = None,
+    ) -> "xr.DataTree":
+        """
+        Create a "virtual" [xarray.DataTree][] containing the contents of a zarr group. Default is the root group and all sub-groups.
+
+        Will ignore the contents of any other groups in the store.
+
+        Requires xarray.
+
+        Parameters
+        ----------
+        group : Group to convert to a virtual DataTree
+        loadable_variables
+            Variables in the data source to load as Dask/NumPy arrays instead of as virtual arrays.
+        decode_times
+            Bool that is passed into [xarray.open_dataset][]. Allows time to be decoded into a datetime object.
+
+        Returns
+        -------
+        vdt : xarray.DataTree
+        """
+
+        from virtualizarr.xarray import construct_virtual_datatree
+
+        return construct_virtual_datatree(
+            manifest_store=self,
+            group=group,
+            loadable_variables=loadable_variables,
+            decode_times=decode_times,
+        )
+
 
 def _transform_byte_range(
     byte_range: ByteRequest | None, *, chunk_start: int, chunk_end_exclusive: int

virtualizarr/tests/test_parsers/test_tiff.py

@@ -1,8 +1,8 @@
 import pytest
 from obstore.store import S3Store
-from xarray import Dataset
+from xarray import Dataset, DataTree
 
-from virtualizarr import open_virtual_dataset
+from virtualizarr import open_virtual_dataset, open_virtual_datatree
 from virtualizarr.registry import ObjectStoreRegistry
 from virtualizarr.tests import requires_network, requires_tiff
 
@@ -11,7 +11,25 @@
 
 @requires_tiff
 @requires_network
-def test_virtual_tiff() -> None:
+def test_virtual_tiff_datatree() -> None:
+    store = S3Store("sentinel-cogs", region="us-west-2", skip_signature=True)
+    registry = ObjectStoreRegistry({"s3://sentinel-cogs/": store})
+    url = "s3://sentinel-cogs/sentinel-s2-l2a-cogs/12/S/UF/2022/6/S2B_12SUF_20220609_0_L2A/B04.tif"
+    parser = virtual_tiff.VirtualTIFF(ifd_layout="nested")
+    with open_virtual_datatree(url=url, parser=parser, registry=registry) as vdt:
+        assert isinstance(vdt, DataTree)
+        assert list(vdt["0"].ds.variables) == ["0"]
+        var = vdt["0"].ds["0"].variable
+        assert var.sizes == {"y": 10980, "x": 10980}
+        assert var.dtype == "<u2"
+        var = vdt["1"].ds["1"].variable
+        assert var.sizes == {"y": 5490, "x": 5490}
+        assert var.dtype == "<u2"
+
+
+@requires_tiff
+@requires_network
+def test_virtual_tiff_dataset() -> None:
     store = S3Store("sentinel-cogs", region="us-west-2", skip_signature=True)
     registry = ObjectStoreRegistry({"s3://sentinel-cogs/": store})
     url = "s3://sentinel-cogs/sentinel-s2-l2a-cogs/12/S/UF/2022/6/S2B_12SUF_20220609_0_L2A/B04.tif"