Merge pull request #927 from davidhassell/h5netcdf-write-from-parallel-read

New 'h5netcdf-h5py' netCDF backend in `cf.write`, and 'h5py_options' parameter

Author: davidhassell

Changelog.rst

@@ -3,6 +3,10 @@ Version NEXTVERSION
 
 **2026-??-??**
 
+* New default backend for netCDF-4 in `cf.write`: ``h5netcdf-h5py``,
+  that allows control of the internal file metadata via the new
+  ``h5py_options`` parameter
+  (https://github.com/NCAS-CMS/cf-python/issues/924)
 * New default backend for netCDF-4 in `cf.read` that allows parallel
   reading: ``h5netcdf-pyfive``
   (https://github.com/NCAS-CMS/cf-python/issues/912)
@@ -13,8 +17,6 @@ Version NEXTVERSION
 * Fix for subspacing with cyclic `cf.wi` and `cf.wo` arguments
   (https://github.com/NCAS-CMS/cf-python/issues/887)
 
-----
-
 Version 3.19.0
 --------------
 

README.md

@@ -88,7 +88,7 @@ of its array manipulation and can:
   choice of netCDF backends,and in local, http, and s3 locations,
 * create new field constructs in memory,
 * write and append field and domain constructs to netCDF and Zarr v3
-  datasets on disk,
+  datasets on disk, with control over HDF5 internal file metadata,
 * read, create, and manipulate UGRID mesh topologies,
 * read, write, and create coordinates defined by geometry cells,
 * read netCDF and CDL datasets containing hierarchical groups,

cf/test/test_quantization.py

@@ -2,6 +2,7 @@
 import datetime
 import faulthandler
 import os
+import shutil
 import tempfile
 import unittest
 
@@ -20,6 +21,13 @@
 ]
 [tmpfile1, tmpfile2] = tmpfiles
 
+# Set up temporary directories
+tmpdirs = [
+    tempfile.mkdtemp("_test_quantization.zarr", dir=os.getcwd())
+    for i in range(1)
+]
+[tmpdir] = tmpdirs
+
 
 def _remove_tmpfiles():
     """Remove temporary files created during tests."""
@@ -29,6 +37,13 @@ def _remove_tmpfiles():
         except OSError:
             pass
 
+    for d in tmpdirs:
+        try:
+            shutil.rmtree(d)
+            os.rmdir(d)
+        except OSError:
+            pass
+
 
 atexit.register(_remove_tmpfiles)
 
@@ -63,7 +78,7 @@ def test_quantization_read_write(self):
         f.set_quantize_on_write(q0)
 
         # Write the field and read it back in
-        cf.write(f, tmpfile1)
+        cf.write(f, tmpfile1, netcdf_backend="netCDF4")
         g = cf.read(tmpfile1)[0]
 
         # Check that f and g have different data (i.e. that
@@ -174,7 +189,7 @@ def test_quantization_write_exceptions(self):
         # digit_round
         f.set_quantize_on_write(algorithm="digitround", quantization_nsd=2)
         with self.assertRaises(ValueError):
-            cf.write(f, tmpfile1)
+            cf.write(f, tmpfile1, netcdf_backend="netCDF4")
 
         # NetCDF3 formats
         for fmt in self.netcdf3_fmts:
@@ -184,29 +199,29 @@ def test_quantization_write_exceptions(self):
         # Integer data type
         f.data.dtype = int
         with self.assertRaises(ValueError):
-            cf.write(f, tmpfile1)
+            cf.write(f, tmpfile1, netcdf_backend="netCDF4")
 
         # Out-of-range quantization_nsd
         f.data.dtype = "float32"
         f.set_quantize_on_write(algorithm="bitgroom", quantization_nsd=8)
         with self.assertRaises(ValueError):
-            cf.write(f, tmpfile1)
+            cf.write(f, tmpfile1, netcdf_backend="netCDF4")
 
         f.data.dtype = "float64"
         f.set_quantize_on_write(algorithm="bitgroom", quantization_nsd=16)
         with self.assertRaises(ValueError):
-            cf.write(f, tmpfile1)
+            cf.write(f, tmpfile1, netcdf_backend="netCDF4")
 
         # Out-of-range quantization_nsb
         f.data.dtype = "float32"
         f.set_quantize_on_write(algorithm="bitround", quantization_nsb=24)
         with self.assertRaises(ValueError):
-            cf.write(f, tmpfile1)
+            cf.write(f, tmpfile1, netcdf_backend="netCDF4")
 
         f.data.dtype = "float64"
         f.set_quantize_on_write(algorithm="bitround", quantization_nsb=53)
         with self.assertRaises(ValueError):
-            cf.write(f, tmpfile1)
+            cf.write(f, tmpfile1, netcdf_backend="netCDF4")
 
     def test_quantization_copy(self):
         """Test that quantization information gets copied."""
@@ -222,6 +237,24 @@ def test_quantization_copy(self):
         g = f.copy()
         self.assertTrue(g.get_quantization().equals(q))
 
+    def test_quantization_backends(self):
+        """Test that quantization-on-write with different backends."""
+        f = self.f1.copy()
+        f.set_quantize_on_write(
+            algorithm="granular_bitround", quantization_nsd=8
+        )
+
+        # Backends that allow quantisation-on-write
+        for backend in ("netCDF4",):
+            cf.write(f, tmpfile1, netcdf_backend=backend)
+
+        # Backends that do not allow quantisation-on-write
+        with self.assertRaises(NotImplementedError):
+            cf.write(f, tmpdir, fmt="ZARR3", netcdf_backend="zarr")
+
+        with self.assertRaises(NotImplementedError):
+            cf.write(f, tmpfile1, netcdf_backend="h5netcdf-h5py")
+
 
 if __name__ == "__main__":
     print("Run date:", datetime.datetime.now())

cf/test/test_read_write.py

@@ -352,7 +352,13 @@ def test_write_netcdf_mode(self):
                 if ex_field_n in (8, 9, 10):
                     continue
 
-                cf.write(ex_field, tmpfile, fmt=fmt, mode="a")
+                cf.write(
+                    ex_field,
+                    tmpfile,
+                    fmt=fmt,
+                    mode="a",
+                    netcdf_backend="netCDF4",
+                )
                 f = cf.read(tmpfile)
 
                 if ex_field_n == 5:  # another special case
@@ -433,15 +439,25 @@ def test_write_netcdf_mode(self):
             # field n=5 which aggregates to one with n=2] => + 1 - 1 = + 0:
             overall_length = len(append_ex_fields)
             cf.write(
-                append_ex_fields, tmpfile, fmt=fmt, mode="a"
+                append_ex_fields,
+                tmpfile,
+                fmt=fmt,
+                mode="a",
+                netcdf_backend="netCDF4",
             )  # 2. now append
             f = cf.read(tmpfile)
             self.assertEqual(len(f), overall_length)
 
             # Also test the mode="r+" alias for mode="a".
-            cf.write(g, tmpfile, fmt=fmt, mode="w")  # 1. overwrite to wipe
             cf.write(
-                append_ex_fields, tmpfile, fmt=fmt, mode="r+"
+                g, tmpfile, fmt=fmt, mode="w", netcdf_backend="netCDF4"
+            )  # 1. overwrite to wipe
+            cf.write(
+                append_ex_fields,
+                tmpfile,
+                fmt=fmt,
+                mode="r+",
+                netcdf_backend="netCDF4",
             )  # 2. now append
             f = cf.read(tmpfile)
             self.assertEqual(len(f), overall_length)
@@ -543,8 +559,12 @@ def test_write_netcdf_mode(self):
             #     )
 
             # Check behaviour when append identical fields, as an edge case:
-            cf.write(g, tmpfile, fmt=fmt, mode="w")  # 1. overwrite to wipe
-            cf.write(g_copy, tmpfile, fmt=fmt, mode="a")  # 2. now append
+            cf.write(
+                g, tmpfile, fmt=fmt, mode="w", netcdf_backend="netCDF4"
+            )  # 1. overwrite to wipe
+            cf.write(
+                g_copy, tmpfile, fmt=fmt, mode="a", netcdf_backend="netCDF4"
+            )  # 2. now append
             f = cf.read(tmpfile)
             self.assertEqual(len(f), 2)
             self.assertTrue(
@@ -849,7 +869,7 @@ def test_write_omit_data(self):
         f = self.f1
         cf.write(f, tmpfile)
 
-        cf.write(f, tmpfile, omit_data="all")
+        cf.write(f, tmpfile, omit_data="all", netcdf_backend="netCDF4")
         g = cf.read(tmpfile)
         self.assertEqual(len(g), 1)
         g = g[0]
@@ -861,7 +881,12 @@ def test_write_omit_data(self):
         # Check that a dump works
         g.dump(display=False)
 
-        cf.write(f, tmpfile, omit_data=("field", "dimension_coordinate"))
+        cf.write(
+            f,
+            tmpfile,
+            omit_data=("field", "dimension_coordinate"),
+            netcdf_backend="netCDF4",
+        )
         g = cf.read(tmpfile)[0]
 
         # Check that only the field and dimension coordinate data are
@@ -870,7 +895,7 @@ def test_write_omit_data(self):
         self.assertFalse(np.ma.count(g.construct("grid_latitude").array))
         self.assertTrue(np.ma.count(g.construct("latitude").array))
 
-        cf.write(f, tmpfile, omit_data="field")
+        cf.write(f, tmpfile, omit_data="field", netcdf_backend="netCDF4")
         g = cf.read(tmpfile)[0]
 
         # Check that only the field data are missing
@@ -956,6 +981,73 @@ def test_read_zarr(self):
             z = cf.read(zarr_dataset, dataset_type="Zarr")
             self.assertEqual(len(z), 1)
 
+    def test_write_netcdf_backend(self):
+        """Test cf.write with different netCDF backends."""
+        f = self.f0
+
+        cf.write(f, tmpfile0, netcdf_backend="h5netcdf-h5py")
+        cf.write(f, tmpfile1, netcdf_backend="netCDF4")
+        f0 = cf.read(tmpfile0)[0]
+        f1 = cf.read(tmpfile1)[0]
+        self.assertTrue(f1.equals(f0))
+
+        f = cf.read(filename)
+        cf.write(f, tmpfile0, netcdf_backend="h5netcdf-h5py")
+        cf.write(f, tmpfile1, netcdf_backend="netCDF4")
+        f0 = cf.read(tmpfile0)[0]
+        f1 = cf.read(tmpfile1)[0]
+        self.assertTrue(f1.equals(f0))
+
+        # Bad fmt/backend combinations
+        for backend in ("netCDF4", "h5netcdf-h5py"):
+            with self.assertRaises(ValueError):
+                cf.write(f, tmpfile, fmt="ZARR3", netcdf_backend=backend)
+
+        for backend in ("zarr", "h5netcdf-h5py"):
+            with self.assertRaises(ValueError):
+                cf.write(
+                    f, tmpfile, fmt="NETCDF3_CLASSIC", netcdf_backend=backend
+                )
+
+        for backend in ("zarr",):
+            with self.assertRaises(ValueError):
+                cf.write(f, tmpfile, fmt="NETCDF4", netcdf_backend=backend)
+
+    def test_write_h5py_options(self):
+        """Test cf.write with h5py_options."""
+        f = self.f0
+        h5py_options = dict(
+            fs_strategy="page", fs_page_size=2**20, meta_block_size=500000
+        )
+
+        cf.write(f, tmpfile0, h5py_options=None)
+        size = os.path.getsize(tmpfile0)
+
+        cf.write(
+            f,
+            tmpfile1,
+            netcdf_backend="h5netcdf-h5py",
+            h5py_options=h5py_options,
+        )
+        self.assertTrue(os.path.getsize(tmpfile1) > size)
+
+        f0 = cf.read(tmpfile0)[0]
+        f1 = cf.read(tmpfile1)[0]
+        self.assertTrue(f1.equals(f0))
+
+        with self.assertRaises(ValueError):
+            cf.write(
+                f,
+                tmpfile0,
+                netcdf_backend="netCDF4",
+                h5py_options=h5py_options,
+            )
+
+        with self.assertRaises(ValueError):
+            cf.write(
+                f, tmpfile0, fmt="NETCDF3_CLASSIC", h5py_options=h5py_options
+            )
+
     def test_read_netcdf_file(self):
         """Test cf.read for differing the netcdf_file backend."""
         f = self.f0

docs/source/introduction.rst

@@ -77,7 +77,7 @@ The `cf` package can:
 * create new field constructs in memory,
 
 * write and append field and domain constructs to netCDF and Zarr
-  datasets on disk,
+  datasets on disk, with control over HDF5 internal file metadata,
 
 * read, write, and manipulate UGRID mesh topologies,
 

setup.py

@@ -184,7 +184,7 @@ def compile():
 
 * create new field constructs in memory,
 
-* write and append field and domain constructs to netCDF and Zarr v3 datasets on disk,
+* write and append field and domain constructs to netCDF and Zarr v3 datasets on disk, with control over HDF5 internal file metadata,
 
 * read, write, and create coordinates defined by geometry cells,