Non-kerchunk backend for HDF5/netcdf4 files.

ci/environment.yml

@@ -13,6 +13,8 @@ dependencies:
   - ujson
   - packaging
   - universal_pathlib
+  - hdf5plugin
+  - numcodecs
   # Testing
   - codecov
   - pre-commit
@@ -27,7 +29,11 @@ dependencies:
   - fsspec
   - s3fs
   - fastparquet
+  - imagecodecs>=2024.6.1
   # for opening tiff files
   - tifffile
   # for opening FITS files
   - astropy
+  - pip
+  - pip:
+    - imagecodecs-numcodecs==2024.6.1

ci/min-deps.yml

@@ -3,7 +3,6 @@ channels:
   - conda-forge
   - nodefaults
 dependencies:
-  - h5netcdf
   - h5py
   - hdf5
   - netcdf4

ci/upstream.yml

@@ -12,6 +12,9 @@ dependencies:
   - packaging
   - ujson
   - universal_pathlib
+  - hdf5plugin
+  - numcodecs
+  - imagecodecs>=2024.6.1
   # Testing
   - codecov
   - pre-commit
@@ -27,3 +30,4 @@ dependencies:
   - pip:
     - icechunk # Installs zarr v3 as dependency
     # - git+https://github.com/fsspec/kerchunk@main  # kerchunk is currently incompatible with zarr-python v3 (https://github.com/fsspec/kerchunk/pull/516)
+    - imagecodecs-numcodecs==2024.6.1

pyproject.toml

@@ -30,15 +30,23 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
+hdf_reader = [
+    "fsspec",
+    "h5py",
+    "hdf5plugin",
+    "imagecodecs",
+    "imagecodecs-numcodecs==2024.6.1",
+    "numcodecs"
+]
 test = [
     "codecov",
     "fastparquet",
     "fsspec",
-    "h5netcdf",
     "h5py",
     "kerchunk>=0.2.5",
     "mypy",
     "netcdf4",
+    "numcodecs",
     "pandas-stubs",
     "pooch",
     "pre-commit",
@@ -48,6 +56,7 @@ test = [
     "ruff",
     "s3fs",
     "scipy",
+    "virtualizarr[hdf_reader]"
 ]
 
 

virtualizarr/backend.py

@@ -16,8 +16,10 @@
     HDF5VirtualBackend,
     KerchunkVirtualBackend,
     NetCDF3VirtualBackend,
+    TIFFVirtualBackend,
     ZarrV3VirtualBackend,
 )
+from virtualizarr.readers.common import VirtualBackend
 from virtualizarr.utils import _FsspecFSFromFilepath, check_for_collisions
 
 # TODO add entrypoint to allow external libraries to add to this mapping
@@ -26,10 +28,10 @@
     "zarr_v3": ZarrV3VirtualBackend,
     "dmrpp": DMRPPVirtualBackend,
     # all the below call one of the kerchunk backends internally (https://fsspec.github.io/kerchunk/reference.html#file-format-backends)
-    "netcdf3": NetCDF3VirtualBackend,
     "hdf5": HDF5VirtualBackend,
     "netcdf4": HDF5VirtualBackend,  # note this is the same as for hdf5
-    # "tiff": TIFFVirtualBackend,
+    "netcdf3": NetCDF3VirtualBackend,
+    "tiff": TIFFVirtualBackend,
     "fits": FITSVirtualBackend,
 }
 
@@ -112,6 +114,7 @@ def open_virtual_dataset(
     indexes: Mapping[str, Index] | None = None,
     virtual_array_class=ManifestArray,
     reader_options: Optional[dict] = None,
+    backend: Optional[VirtualBackend] = None,
 ) -> Dataset:
     """
     Open a file or store as an xarray Dataset wrapping virtualized zarr arrays.
@@ -173,15 +176,20 @@ def open_virtual_dataset(
     if reader_options is None:
         reader_options = {}
 
+    if backend and filetype:
+        raise ValueError("Cannot pass both a filetype and an explicit VirtualBackend")
+
     if filetype is not None:
         # if filetype is user defined, convert to FileType
         filetype = FileType(filetype)
     else:
         filetype = automatically_determine_filetype(
             filepath=filepath, reader_options=reader_options
         )
-
-    backend_cls = VIRTUAL_BACKENDS.get(filetype.name.lower())
+    if backend:
+        backend_cls = backend
+    else:
+        backend_cls = VIRTUAL_BACKENDS.get(filetype.name.lower())  # type: ignore
 
     if backend_cls is None:
         raise NotImplementedError(f"Unsupported file type: {filetype.name}")

virtualizarr/readers/__init__.py

@@ -1,5 +1,6 @@
 from virtualizarr.readers.dmrpp import DMRPPVirtualBackend
 from virtualizarr.readers.fits import FITSVirtualBackend
+from virtualizarr.readers.hdf import HDFVirtualBackend
 from virtualizarr.readers.hdf5 import HDF5VirtualBackend
 from virtualizarr.readers.kerchunk import KerchunkVirtualBackend
 from virtualizarr.readers.netcdf3 import NetCDF3VirtualBackend
@@ -9,6 +10,7 @@
 __all__ = [
     "DMRPPVirtualBackend",
     "FITSVirtualBackend",
+    "HDFVirtualBackend",
     "HDF5VirtualBackend",
     "KerchunkVirtualBackend",
     "NetCDF3VirtualBackend",

virtualizarr/readers/hdf/__init__.py

@@ -0,0 +1,11 @@
+from .hdf import (
+    HDFVirtualBackend,
+    construct_virtual_dataset,
+    open_loadable_vars_and_indexes,
+)
+
+__all__ = [
+    "HDFVirtualBackend",
+    "construct_virtual_dataset",
+    "open_loadable_vars_and_indexes",
+]

virtualizarr/readers/hdf/filters.py

@@ -0,0 +1,195 @@
+import dataclasses
+from typing import TYPE_CHECKING, List, Tuple, TypedDict, Union
+
+import numcodecs.registry as registry
+import numpy as np
+from numcodecs.abc import Codec
+from numcodecs.fixedscaleoffset import FixedScaleOffset
+from xarray.coding.variables import _choose_float_dtype
+
+from virtualizarr.utils import soft_import
+
+if TYPE_CHECKING:
+    import h5py  # type: ignore
+    from h5py import Dataset, Group  # type: ignore
+
+h5py = soft_import("h5py", "For reading hdf files", strict=False)
+if h5py:
+    Dataset = h5py.Dataset
+    Group = h5py.Group
+else:
+    Dataset = dict()
+    Group = dict()
+
+hdf5plugin = soft_import(
+    "hdf5plugin", "For reading hdf files with filters", strict=False
+)
+imagecodecs = soft_import(
+    "imagecodecs", "For reading hdf files with filters", strict=False
+)
+
+
+_non_standard_filters = {
+    "gzip": "zlib",
+    "lzf": "imagecodecs_lzf",
+}
+
+_hdf5plugin_imagecodecs = {"lz4": "imagecodecs_lz4h5", "bzip2": "imagecodecs_bz2"}
+
+
+@dataclasses.dataclass
+class BloscProperties:
+    blocksize: int
+    clevel: int
+    shuffle: int
+    cname: str
+
+    def __post_init__(self):
+        blosc_compressor_codes = {
+            value: key
+            for key, value in hdf5plugin._filters.Blosc._Blosc__COMPRESSIONS.items()
+        }
+        self.cname = blosc_compressor_codes[self.cname]
+
+
+@dataclasses.dataclass
+class ZstdProperties:
+    level: int
+
+
+@dataclasses.dataclass
+class ShuffleProperties:
+    elementsize: int
+
+
+@dataclasses.dataclass
+class ZlibProperties:
+    level: int
+
+
+class CFCodec(TypedDict):
+    target_dtype: np.dtype
+    codec: Codec
+
+
+def _filter_to_codec(
+    filter_id: str, filter_properties: Union[int, None, Tuple] = None
+) -> Codec:
+    """
+    Convert an h5py filter to an equivalent numcodec
+
+    Parameters
+    ----------
+    filter_id: str
+        An h5py filter id code.
+    filter_properties : int or None or Tuple
+        A single or Tuple of h5py filter configuration codes.
+
+    Returns
+    -------
+        A numcodec codec
+    """
+    id_int = None
+    id_str = None
+    try:
+        id_int = int(filter_id)
+    except ValueError:
+        id_str = filter_id
+    conf = {}
+    if id_str:
+        if id_str in _non_standard_filters.keys():
+            id = _non_standard_filters[id_str]
+        else:
+            id = id_str
+        if id == "zlib":
+            zlib_props = ZlibProperties(level=filter_properties)  # type: ignore
+            conf = dataclasses.asdict(zlib_props)
+        if id == "shuffle" and isinstance(filter_properties, tuple):
+            shuffle_props = ShuffleProperties(elementsize=filter_properties[0])
+            conf = dataclasses.asdict(shuffle_props)
+        conf["id"] = id  # type: ignore[assignment]
+    if id_int:
+        filter = hdf5plugin.get_filters(id_int)[0]
+        id = filter.filter_name
+        if id in _hdf5plugin_imagecodecs.keys():
+            id = _hdf5plugin_imagecodecs[id]
+        if id == "blosc" and isinstance(filter_properties, tuple):
+            blosc_fields = [field.name for field in dataclasses.fields(BloscProperties)]
+            blosc_props = BloscProperties(
+                **{k: v for k, v in zip(blosc_fields, filter_properties[-4:])}
+            )
+            conf = dataclasses.asdict(blosc_props)
+        if id == "zstd" and isinstance(filter_properties, tuple):
+            zstd_props = ZstdProperties(level=filter_properties[0])
+            conf = dataclasses.asdict(zstd_props)
+        conf["id"] = id
+    codec = registry.get_codec(conf)
+    return codec
+
+
+def cfcodec_from_dataset(dataset: Dataset) -> Codec | None:
+    """
+    Converts select h5py dataset CF convention attrs to CFCodec
+
+    Parameters
+    ----------
+    dataset: h5py.Dataset
+       An h5py dataset.
+
+    Returns
+    -------
+    CFCodec
+        A CFCodec.
+    """
+    attributes = {attr: dataset.attrs[attr] for attr in dataset.attrs}
+    mapping = {}
+    if "scale_factor" in attributes:
+        try:
+            scale_factor = attributes["scale_factor"][0]
+        except IndexError:
+            scale_factor = attributes["scale_factor"]
+        mapping["scale_factor"] = float(1 / scale_factor)
+    else:
+        mapping["scale_factor"] = 1
+    if "add_offset" in attributes:
+        try:
+            offset = attributes["add_offset"][0]
+        except IndexError:
+            offset = attributes["add_offset"]
+        mapping["add_offset"] = float(offset)
+    else:
+        mapping["add_offset"] = 0
+    if mapping["scale_factor"] != 1 or mapping["add_offset"] != 0:
+        float_dtype = _choose_float_dtype(dtype=dataset.dtype, mapping=mapping)
+        target_dtype = np.dtype(float_dtype)
+        codec = FixedScaleOffset(
+            offset=mapping["add_offset"],
+            scale=mapping["scale_factor"],
+            dtype=target_dtype,
+            astype=dataset.dtype,
+        )
+        cfcodec = CFCodec(target_dtype=target_dtype, codec=codec)
+        return cfcodec
+    else:
+        return None
+
+
+def codecs_from_dataset(dataset: Dataset) -> List[Codec]:
+    """
+    Extracts a list of numcodecs from an h5py dataset
+
+    Parameters
+    ----------
+    dataset: h5py.Dataset
+       An h5py dataset.
+
+    Returns
+    -------
+    list
+        A list of numcodecs codecs.
+    """
+    codecs = []
+    for filter_id, filter_properties in dataset._filters.items():
+        codec = _filter_to_codec(filter_id, filter_properties)
+        codecs.append(codec)
+    return codecs