Use ManifesStore to_virtual_variable directly in HDFVirtualBackend. (#542)

* Use ManifesStore to_virtual_variable directory in HDFVirtualBackend.

* Update HDFVirtualBackend tests to reflect changes to internals.

* Simplify _FillValue encoding logic.

* Add default_object_store internal func

* Remove TODO because lack of duplicate keys are useful

* Also return prefix

* Remove config complexity

* Make _find_matching_store a method

* Test default store creation with HDF5 reader

* Improve typing

* Protect against duplicate config options

* Mark minio tests

* Fix test

* Specify that other schemes aren't supported

* Make mypy pass

* Revise codecov config

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add http support for default_object_store.

* Re-include relative path normalization.

* Typing fixes.

* Include obstore as hdf backend dependency.

* Pin obstore minimum version.

Co-authored-by: Max Jones <[email protected]>

* Undo test parallelization changes.

Co-authored-by: Max Jones <[email protected]>

* Spellcheck.

Co-authored-by: Max Jones <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Spellcheck.

Co-authored-by: Max Jones <[email protected]>

* Include comment about current big endian dtype support.

* Pass indexes arg through to virtual dataset creation.

---------

Co-authored-by: Max Jones <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: 3 people

pyproject.toml

@@ -50,6 +50,7 @@ hdf = [
     "hdf5plugin",
     "imagecodecs",
     "imagecodecs-numcodecs==2024.6.1",
+    "obstore>=0.5.1",
 ]
 
 # kerchunk-based readers

virtualizarr/manifests/store.py

@@ -150,7 +150,9 @@ def default_object_store(filepath: str) -> ObjectStore:
             virtual_hosted_style_request=False,
             region=_find_bucket_region(bucket),
         )
-
+    if parsed.scheme in ["http", "https"]:
+        base_url = f"{parsed.scheme}://{parsed.netloc}"
+        return obs.store.HTTPStore.from_url(base_url)
     raise NotImplementedError(f"{parsed.scheme} is not yet supported")
 
 

virtualizarr/readers/hdf/hdf.py

@@ -4,9 +4,6 @@
 from pathlib import Path
 from typing import (
     TYPE_CHECKING,
-    Any,
-    Dict,
-    Hashable,
     Iterable,
     List,
     Mapping,
@@ -31,13 +28,9 @@
 from virtualizarr.manifests.store import ObjectStoreRegistry, default_object_store
 from virtualizarr.manifests.utils import create_v3_array_metadata
 from virtualizarr.readers.api import VirtualBackend
-from virtualizarr.readers.hdf.filters import cfcodec_from_dataset, codecs_from_dataset
+from virtualizarr.readers.hdf.filters import codecs_from_dataset
 from virtualizarr.types import ChunkKey
-from virtualizarr.utils import _FsspecFSFromFilepath, soft_import
-from virtualizarr.xarray import (
-    construct_fully_virtual_dataset,
-    construct_virtual_dataset,
-)
+from virtualizarr.utils import soft_import
 
 h5py = soft_import("h5py", "For reading hdf files", strict=False)
 
@@ -88,6 +81,22 @@ def _construct_manifest_array(
         attrs = HDFVirtualBackend._extract_attrs(dataset)
         dtype = dataset.dtype
 
+        # Temporarily disable use CF->Codecs - TODO re-enable in subsequent PR.
+        # cfcodec = cfcodec_from_dataset(dataset)
+        # if cfcodec:
+        # codecs.insert(0, cfcodec["codec"])
+        # dtype = cfcodec["target_dtype"]
+        # attrs.pop("scale_factor", None)
+        # attrs.pop("add_offset", None)
+        # else:
+        # dtype = dataset.dtype
+
+        if "_FillValue" in attrs:
+            encoded_cf_fill_value = HDFVirtualBackend._encode_cf_fill_value(
+                attrs["_FillValue"], dtype
+            )
+            attrs["_FillValue"] = encoded_cf_fill_value
+
         codec_configs = [
             numcodec_config_to_configurable(codec.get_config()) for codec in codecs
         ]
@@ -113,7 +122,7 @@ def _construct_manifest_group(
         filepath: str,
         *,
         group: str | None = None,
-        drop_variables: Optional[List[str]] = None,
+        drop_variables: Optional[Iterable[str]] = None,
     ) -> ManifestGroup:
         """
         Construct a virtual Group from a HDF dataset.
@@ -136,11 +145,16 @@ def _construct_manifest_group(
             group_name = "/"
 
         manifest_dict = {}
-        non_coordinate_dimesion_vars = HDFVirtualBackend._find_non_coord_dimension_vars(
-            group=g
+        # Several of our test fixtures which use xr.tutorial data have
+        # non coord dimensions serialized using big endian dtypes which are not
+        # yet supported in zarr-python v3.  We'll drop these variables for the
+        # moment until big endian support is included upstream.)
+
+        non_coordinate_dimension_vars = (
+            HDFVirtualBackend._find_non_coord_dimension_vars(group=g)
         )
-        drop_variables = list(set(drop_variables + non_coordinate_dimesion_vars))
-        attrs: dict[str, Any] = {}
+        drop_variables = list(set(list(drop_variables) + non_coordinate_dimension_vars))
+        attrs = HDFVirtualBackend._extract_attrs(g)
         for key in g.keys():
             if key not in drop_variables:
                 if isinstance(g[key], h5py.Dataset):
@@ -159,12 +173,16 @@ def _create_manifest_store(
         *,
         store: ObjectStore | None = None,
         group: str | None = None,
+        drop_variables: Iterable[str] | None = None,
     ) -> ManifestStore:
         # Create a group containing dataset level metadata and all the manifest arrays
         if not store:
             store = default_object_store(filepath)  # type: ignore
         manifest_group = HDFVirtualBackend._construct_manifest_group(
-            store=store, filepath=filepath, group=group
+            store=store,
+            filepath=filepath,
+            group=group,
+            drop_variables=drop_variables,
         )
         registry = ObjectStoreRegistry({filepath: store})
         # Convert to a manifest store
@@ -192,40 +210,21 @@ def open_virtual_dataset(
             filepath, fs_root=Path.cwd().as_uri()
         )
 
-        _drop_vars: list[Hashable] = (
+        _drop_vars: Iterable[str] = (
             [] if drop_variables is None else list(drop_variables)
         )
 
-        # TODO provide a way to drop a variable _before_ h5py attempts to inspect it?
-        virtual_vars = HDFVirtualBackend._virtual_vars_from_hdf(
-            path=filepath,
-            group=group,
-            reader_options=reader_options,
-        )
-
-        attrs = HDFVirtualBackend._get_group_attrs(
-            path=filepath, reader_options=reader_options, group=group
-        )
-        coordinates_attr = attrs.pop("coordinates", "")
-        coord_names = coordinates_attr.split()
-
-        fully_virtual_dataset = construct_fully_virtual_dataset(
-            virtual_vars=virtual_vars,
-            coord_names=coord_names,
-            attrs=attrs,
-        )
-
-        vds = construct_virtual_dataset(
-            fully_virtual_ds=fully_virtual_dataset,
+        manifest_store = HDFVirtualBackend._create_manifest_store(
             filepath=filepath,
+            drop_variables=_drop_vars,
             group=group,
+        )
+        ds = manifest_store.to_virtual_dataset(
             loadable_variables=loadable_variables,
-            reader_options=reader_options,
-            indexes=indexes,
             decode_times=decode_times,
+            indexes=indexes,
         )
-
-        return vds.drop_vars(_drop_vars)
+        return ds
 
     @staticmethod
     def _dataset_chunk_manifest(
@@ -346,29 +345,6 @@ def _dataset_dims(dataset: H5Dataset, group: str = "") -> List[str]:
 
         return [dim.removeprefix(group) for dim in dims]
 
-    @staticmethod
-    def _extract_cf_fill_value(
-        h5obj: Union[H5Dataset, H5Group],
-    ) -> Optional[FillValueType]:
-        """
-        Convert the _FillValue attribute from an HDF5 group or dataset into
-        encoding.
-
-        Parameters
-        ----------
-        h5obj : h5py.Group or h5py.Dataset
-            An h5py group or dataset.
-        """
-        fillvalue = None
-        for n, v in h5obj.attrs.items():
-            if n == "_FillValue":
-                if isinstance(v, np.ndarray) and v.size == 1:
-                    fillvalue = v.item()
-                else:
-                    fillvalue = v
-                fillvalue = FillValueCoder.encode(fillvalue, h5obj.dtype)  # type: ignore[arg-type]
-        return fillvalue
-
     @staticmethod
     def _extract_attrs(h5obj: Union[H5Dataset, H5Group]):
         """
@@ -394,7 +370,7 @@ def _extract_attrs(h5obj: Union[H5Dataset, H5Group]):
             if n in _HIDDEN_ATTRS:
                 continue
             if n == "_FillValue":
-                continue
+                v = v
             # Fix some attribute values to avoid JSON encoding exceptions...
             if isinstance(v, bytes):
                 v = v.decode("utf-8") or " "
@@ -414,148 +390,6 @@ def _extract_attrs(h5obj: Union[H5Dataset, H5Group]):
             attrs[n] = v
         return attrs
 
-    @staticmethod
-    def _dataset_to_variable(
-        path: str,
-        dataset: H5Dataset,
-        group: str,
-    ) -> Optional[xr.Variable]:
-        """
-        Extract an xarray Variable with ManifestArray data from an h5py dataset
-
-        Parameters
-        ----------
-        dataset : h5py.Dataset
-            An h5py dataset.
-        group : str
-            Name of the group containing this h5py.Dataset.
-
-        Returns
-        -------
-        list: xarray.Variable
-            A list of xarray variables.
-        """
-        chunks = dataset.chunks if dataset.chunks else dataset.shape
-        codecs = codecs_from_dataset(dataset)
-        cfcodec = cfcodec_from_dataset(dataset)
-        attrs = HDFVirtualBackend._extract_attrs(dataset)
-        cf_fill_value = HDFVirtualBackend._extract_cf_fill_value(dataset)
-        attrs.pop("_FillValue", None)
-
-        if cfcodec:
-            codecs.insert(0, cfcodec["codec"])
-            dtype = cfcodec["target_dtype"]
-            attrs.pop("scale_factor", None)
-            attrs.pop("add_offset", None)
-        else:
-            dtype = dataset.dtype
-
-        codec_configs = [
-            numcodec_config_to_configurable(codec.get_config()) for codec in codecs
-        ]
-
-        fill_value = dataset.fillvalue.item()
-        metadata = create_v3_array_metadata(
-            shape=dataset.shape,
-            data_type=dtype,
-            chunk_shape=chunks,
-            fill_value=fill_value,
-            codecs=codec_configs,
-        )
-        dims = HDFVirtualBackend._dataset_dims(dataset, group=group)
-        manifest = HDFVirtualBackend._dataset_chunk_manifest(path, dataset)
-        if manifest:
-            marray = ManifestArray(metadata=metadata, chunkmanifest=manifest)
-            variable = xr.Variable(data=marray, dims=dims, attrs=attrs)
-        else:
-            variable = xr.Variable(data=np.empty(dataset.shape), dims=dims, attrs=attrs)
-        if cf_fill_value is not None:
-            variable.encoding["_FillValue"] = cf_fill_value
-        return variable
-
-    @staticmethod
-    def _virtual_vars_from_hdf(
-        path: str,
-        group: Optional[str] = None,
-        drop_variables: Optional[List[str]] = None,
-        reader_options: Optional[dict] = {
-            "storage_options": {"key": "", "secret": "", "anon": True}
-        },
-    ) -> Dict[str, xr.Variable]:
-        """
-        Extract xarray Variables with ManifestArray data from an HDF file or group
-
-        Parameters
-        ----------
-        path: str
-            The path of the hdf5 file.
-        group: str, optional
-            The name of the group for which to extract variables. None refers to the root group.
-        drop_variables: list of str
-            A list of variable names to skip extracting.
-        reader_options: dict
-            A dictionary of reader options passed to fsspec when opening the file.
-
-        Returns
-        -------
-        dict
-            A dictionary of Xarray Variables with the variable names as keys.
-        """
-        if drop_variables is None:
-            drop_variables = []
-
-        open_file = _FsspecFSFromFilepath(
-            filepath=path, reader_options=reader_options
-        ).open_file()
-        f = h5py.File(open_file, mode="r")
-
-        if group is not None and group != "":
-            g = f[group]
-            group_name = group
-            if not isinstance(g, h5py.Group):
-                raise ValueError("The provided group is not an HDF group")
-        else:
-            g = f["/"]
-            group_name = "/"
-
-        variables = {}
-        non_coordinate_dimesion_vars = HDFVirtualBackend._find_non_coord_dimension_vars(
-            group=g
-        )
-        drop_variables = list(set(drop_variables + non_coordinate_dimesion_vars))
-        for key in g.keys():
-            if key not in drop_variables:
-                if isinstance(g[key], h5py.Dataset):
-                    variable = HDFVirtualBackend._dataset_to_variable(
-                        path=path,
-                        dataset=g[key],
-                        group=group_name,
-                    )
-                    if variable is not None:
-                        variables[key] = variable
-        return variables
-
-    @staticmethod
-    def _get_group_attrs(
-        path: str,
-        group: Optional[str] = None,
-        reader_options: Optional[dict] = {
-            "storage_options": {"key": "", "secret": "", "anon": True}
-        },
-    ):
-        open_file = _FsspecFSFromFilepath(
-            filepath=path, reader_options=reader_options
-        ).open_file()
-        f = h5py.File(open_file, mode="r")
-        if group:
-            g = f[group]
-            if not isinstance(g, h5py.Group):
-                raise ValueError("The provided group is not an HDF group")
-        else:
-            g = f
-        attrs = HDFVirtualBackend._extract_attrs(g)
-        return attrs
-
     @staticmethod
     def _find_non_coord_dimension_vars(group: H5Group) -> List[str]:
         dimension_names = []
@@ -569,3 +403,28 @@ def _find_non_coord_dimension_vars(group: H5Group) -> List[str]:
                     non_coordinate_dimension_variables.append(name)
 
         return non_coordinate_dimension_variables
+
+    @staticmethod
+    def _encode_cf_fill_value(
+        fill_value: Union[np.ndarray, np.generic],
+        target_dtype: np.dtype,
+    ) -> FillValueType:
+        """
+        Convert the _FillValue attribute from an HDF5 group or dataset into
+        one properly encoded for the target dtype.
+
+        Parameters
+        ----------
+        fill_value
+            An ndarray or value.
+        target_dtype
+            The target dtype of the ManifestArray that will use the _FillValue
+        """
+        if isinstance(fill_value, (np.ndarray, np.generic)):
+            if isinstance(fill_value, np.ndarray) and fill_value.size > 1:
+                raise ValueError("Expected a scalar")
+            fillvalue = fill_value.item()
+        else:
+            fillvalue = fill_value
+        encoded_fillvalue = FillValueCoder.encode(fillvalue, target_dtype)
+        return encoded_fillvalue

virtualizarr/tests/test_manifests/test_store.py

@@ -140,6 +140,18 @@ def test_default_object_store_s3(minio_bucket):
     assert isinstance(store, S3Store)
 
 
+@requires_obstore
+@requires_minio
+def test_default_object_store_http(minio_bucket):
+    from obstore.store import HTTPStore
+
+    filepath = minio_bucket["endpoint"]
+    store = default_object_store(
+        filepath,
+    )
+    assert isinstance(store, HTTPStore)
+
+
 @requires_obstore
 def test_default_object_store_local(tmpdir):
     from obstore.store import LocalStore