Opening 0D scalars (zarr-developers#102)

* regression test

* test creating scalars from kerchunk refs

* fix bug by removing faulty check

* refcator to remove duplicated code path

* use temporary directory

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: TomNicholas

virtualizarr/kerchunk.py

@@ -151,7 +151,7 @@ def parse_array_refs(
     arr_refs: KerchunkArrRefs,
 ) -> Tuple[dict, ZArray, ZAttrs]:
     zarray = ZArray.from_kerchunk_refs(arr_refs.pop(".zarray"))
-    zattrs = arr_refs.pop(".zattrs")
+    zattrs = arr_refs.pop(".zattrs", {})
     chunk_dict = arr_refs
 
     return chunk_dict, zarray, zattrs

virtualizarr/manifests/array.py

@@ -1,4 +1,3 @@
-import re
 import warnings
 from typing import Any, Tuple, Union
 
@@ -7,7 +6,7 @@
 from ..kerchunk import KerchunkArrRefs
 from ..zarr import ZArray
 from .array_api import MANIFESTARRAY_HANDLED_ARRAY_FUNCTIONS
-from .manifest import _CHUNK_KEY, ChunkManifest
+from .manifest import ChunkManifest
 
 
 class ManifestArray:
@@ -54,30 +53,20 @@ def __init__(
                 f"chunkmanifest arg must be of type ChunkManifest, but got type {type(chunkmanifest)}"
             )
 
-        # Check that the chunk grid implied by zarray info is consistent with shape implied by chunk keys in manifest
-        if _zarray.shape_chunk_grid != _chunkmanifest.shape_chunk_grid:
-            raise ValueError(
-                f"Inconsistent chunk grid shape between zarray info and manifest: {_zarray.shape_chunk_grid} vs {_chunkmanifest.shape_chunk_grid}"
-            )
-
         self._zarray = _zarray
         self._manifest = _chunkmanifest
 
     @classmethod
     def _from_kerchunk_refs(cls, arr_refs: KerchunkArrRefs) -> "ManifestArray":
-        from virtualizarr.kerchunk import fully_decode_arr_refs
+        from virtualizarr.kerchunk import fully_decode_arr_refs, parse_array_refs
 
         decoded_arr_refs = fully_decode_arr_refs(arr_refs)
 
-        zarray = ZArray.from_kerchunk_refs(decoded_arr_refs[".zarray"])
-
-        kerchunk_chunk_dict = {
-            k: v for k, v in decoded_arr_refs.items() if re.match(_CHUNK_KEY, k)
-        }
-        chunkmanifest = ChunkManifest._from_kerchunk_chunk_dict(kerchunk_chunk_dict)
+        chunk_dict, zarray, _zattrs = parse_array_refs(decoded_arr_refs)
+        manifest = ChunkManifest._from_kerchunk_chunk_dict(chunk_dict)
 
         obj = object.__new__(cls)
-        obj._manifest = chunkmanifest
+        obj._manifest = manifest
         obj._zarray = zarray
 
         return obj

virtualizarr/tests/test_integration.py

@@ -0,0 +1,13 @@
+import xarray as xr
+
+from virtualizarr import open_virtual_dataset
+
+
+def test_open_scalar_variable(tmpdir):
+    # regression test for GH issue #100
+
+    ds = xr.Dataset(data_vars={"a": 0})
+    ds.to_netcdf(f"{tmpdir}/scalar.nc")
+
+    vds = open_virtual_dataset(f"{tmpdir}/scalar.nc")
+    assert vds["a"].shape == ()

virtualizarr/tests/test_manifests/test_array.py

@@ -34,27 +34,6 @@ def test_create_manifestarray(self):
         assert marr.size == 5 * 2 * 20
         assert marr.ndim == 3
 
-    def test_create_invalid_manifestarray(self):
-        chunks_dict = {
-            "0.0.0": {"path": "foo.nc", "offset": 100, "length": 100},
-        }
-        manifest = ChunkManifest(entries=chunks_dict)
-        chunks = (5, 1, 10)
-        shape = (5, 2, 20)
-        zarray = ZArray(
-            chunks=chunks,
-            compressor="zlib",
-            dtype=np.dtype("int32"),
-            fill_value=0.0,
-            filters=None,
-            order="C",
-            shape=shape,
-            zarr_format=2,
-        )
-
-        with pytest.raises(ValueError, match="Inconsistent chunk grid shape"):
-            ManifestArray(zarray=zarray, chunkmanifest=manifest)
-
     def test_create_manifestarray_from_kerchunk_refs(self):
         arr_refs = {
             ".zarray": '{"chunks":[2,3],"compressor":null,"dtype":"<i8","fill_value":null,"filters":null,"order":"C","shape":[2,3],"zarr_format":2}',
@@ -70,6 +49,16 @@ def test_create_manifestarray_from_kerchunk_refs(self):
         assert marr.zarray.filters is None
         assert marr.zarray.order == "C"
 
+    def test_create_scalar_manifestarray_from_kerchunk_refs(self):
+        arr_refs = {
+            ".zarray": '{"chunks":[],"compressor":null,"dtype":"<i8","fill_value":null,"filters":null,"order":"C","shape":[],"zarr_format":2}',
+            "0": ["test1.nc", 6144, 48],
+        }
+        marr = ManifestArray._from_kerchunk_refs(arr_refs)
+
+        assert marr.shape == ()
+        assert marr.chunks == ()
+
 
 class TestEquals:
     def test_equals(self):

virtualizarr/zarr.py

@@ -73,15 +73,6 @@ def codec(self) -> Codec:
         """For comparison against other arrays."""
         return Codec(compressor=self.compressor, filters=self.filters)
 
-    @property
-    def shape_chunk_grid(self) -> Tuple[int, ...]:
-        """Shape of the chunk grid implied by the array shape and chunk shape."""
-        chunk_grid_shape = tuple(
-            ceildiv(array_side_length, chunk_length)
-            for array_side_length, chunk_length in zip(self.shape, self.chunks)
-        )
-        return chunk_grid_shape
-
     def __repr__(self) -> str:
         return f"ZArray(shape={self.shape}, chunks={self.chunks}, dtype={self.dtype}, compressor={self.compressor}, filters={self.filters}, fill_value={self.fill_value})"