HDF5 support (#7690)

* initial hdf5 support

* handle zero dims

* add tests

* refactor type inference

* refactor vlen, drop ragged, add complex/compound

* update tests

* explicit h5py dependency

* allow mismatched lengths if ignored

* Update setup.py

* Sequence -> List

* Sequence -> List cont.

* Fix features.List and typing.List conflict

* Use `Features(dict)` for complex and compound

* Hardcode .hdf5 and .h5 extensions to point to hdf5

* Update type hints from Any to Features

* Unsized List in zero dim case

Author: klamike

setup.py

@@ -166,6 +166,7 @@
     "aiohttp",
     "elasticsearch>=7.17.12,<8.0.0",  # 8.0 asks users to provide hosts or cloud_id when instantiating ElasticSearch(); 7.9.1 has legacy numpy.float_ which was fixed in https://github.com/elastic/elasticsearch-py/pull/2551.
     "faiss-cpu>=1.8.0.post1",  # Pins numpy < 2
+    "h5py",
     "jax>=0.3.14; sys_platform != 'win32'",
     "jaxlib>=0.3.14; sys_platform != 'win32'",
     "lz4",

src/datasets/packaged_modules/__init__.py

@@ -8,6 +8,7 @@
 from .audiofolder import audiofolder
 from .cache import cache
 from .csv import csv
+from .hdf5 import hdf5
 from .imagefolder import imagefolder
 from .json import json
 from .pandas import pandas
@@ -47,6 +48,7 @@ def _hash_python_lines(lines: list[str]) -> str:
     "pdffolder": (pdffolder.__name__, _hash_python_lines(inspect.getsource(pdffolder).splitlines())),
     "webdataset": (webdataset.__name__, _hash_python_lines(inspect.getsource(webdataset).splitlines())),
     "xml": (xml.__name__, _hash_python_lines(inspect.getsource(xml).splitlines())),
+    "hdf5": (hdf5.__name__, _hash_python_lines(inspect.getsource(hdf5).splitlines())),
 }
 
 # get importable module names and hash for caching
@@ -76,6 +78,8 @@ def _hash_python_lines(lines: list[str]) -> str:
     ".txt": ("text", {}),
     ".tar": ("webdataset", {}),
     ".xml": ("xml", {}),
+    ".hdf5": ("hdf5", {}),
+    ".h5": ("hdf5", {}),
 }
 _EXTENSION_TO_MODULE.update({ext: ("imagefolder", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
 _EXTENSION_TO_MODULE.update({ext.upper(): ("imagefolder", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})