Vendor kerchunk netCDF3 reader (#397)

Author: TomNicholas

docs/releases.rst

@@ -60,6 +60,8 @@ Documentation
 Internal Changes
 ~~~~~~~~~~~~~~~~
 
+- Vendor netCDF3 reader from kerchunk. (:pull:`397`) By `Tom Nicholas <https://github.com/TomNicholas>`_.
+
 .. _v1.2.0:
 
 v1.2.0 (5th Dec 2024)

virtualizarr/readers/netcdf3.py

@@ -26,7 +26,7 @@ def open_virtual_dataset(
         virtual_backend_kwargs: Optional[dict] = None,
         reader_options: Optional[dict] = None,
     ) -> Dataset:
-        from kerchunk.netCDF3 import NetCDF3ToZarr
+        from virtualizarr.vendor.kerchunk.netCDF3 import NetCDF3ToZarr
 
         if virtual_backend_kwargs:
             raise NotImplementedError(
@@ -38,6 +38,9 @@ def open_virtual_dataset(
             loadable_variables,
         )
 
+        if reader_options is None:
+            reader_options = {}
+
         refs = NetCDF3ToZarr(filepath, inline_threshold=0, **reader_options).translate()
 
         # both group=None and group='' mean to read root group

virtualizarr/tests/test_readers/conftest.py

@@ -1,3 +1,4 @@
+import pathlib
 import warnings
 
 import h5py  # type: ignore
@@ -322,3 +323,13 @@ def root_coordinates_hdf5_file(tmpdir, np_uncompressed_int16):
     f.create_dataset(name="lon", data=data)
     f.attrs.create(name="coordinates", data="lat lon")
     return filepath
+
+
+@pytest.fixture
+def netcdf3_file(tmp_path: pathlib.Path) -> pathlib.Path:
+    ds = xr.Dataset({"foo": ("x", np.array([1, 2, 3]))})
+
+    filepath = tmp_path / "file.nc"
+    ds.to_netcdf(filepath, format="NETCDF3_CLASSIC")
+
+    return filepath

virtualizarr/tests/test_readers/test_netcdf3.py

@@ -0,0 +1,30 @@
+import numpy as np
+import xarray as xr
+import xarray.testing as xrt
+
+from virtualizarr import open_virtual_dataset
+from virtualizarr.manifests import ChunkManifest, ManifestArray
+from virtualizarr.tests import requires_scipy
+from virtualizarr.zarr import ZArray
+
+
+@requires_scipy
+def test_read_netcdf3(netcdf3_file):
+    filepath = str(netcdf3_file)
+    vds = open_virtual_dataset(filepath)
+
+    assert isinstance(vds, xr.Dataset)
+    assert list(vds.variables.keys()) == ["foo"]
+    assert isinstance(vds["foo"].data, ManifestArray)
+
+    expected_manifest = ChunkManifest(
+        entries={"0": {"path": filepath, "offset": 80, "length": 12}}
+    )
+    expected_zarray = ZArray(dtype=np.dtype(">i4"), shape=(3,), chunks=(3,))
+    expected_ma = ManifestArray(chunkmanifest=expected_manifest, zarray=expected_zarray)
+    expected_vds = xr.Dataset({"foo": xr.Variable(data=expected_ma, dims=["x"])})
+
+    xrt.assert_identical(vds, expected_vds)
+
+
+# TODO test loading data against xarray backend, see issue #394 for context

virtualizarr/vendor/kerchunk/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Intake
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.