Merge pull request #873 from davidhassell/quantization-init

Implement lossy compression via quantization

Author: davidhassell

Changelog.rst

@@ -3,6 +3,14 @@ Version NEXTVERSION
 
 **2025-??-??**
 
+* Implement lossy compression via quantization
+  (https://github.com/NCAS-CMS/cf-python/issues/870)
+* New quantization class: `cf.Quantization`
+  (https://github.com/NCAS-CMS/cf-python/issues/870)
+* New quantization methods: `cf.Field.get_quantization`,
+  `cf.Field.get_quantize_on_write`, `cf.Field.set_quantize_on_write`,
+  `cf.Field.del_quantize_on_write`
+  (https://github.com/NCAS-CMS/cf-python/issues/870)
 * New keyword parameter to `cf.write`: ``chunk_cache``
   (https://github.com/NCAS-CMS/cf-python/issues/871)
 * Read Zarr datasets with `cf.read`

README.md

@@ -116,6 +116,8 @@ of its array manipulation and can:
 * create running means from field constructs,
 * apply differential operators to field constructs,
 * create derived quantities (such as relative vorticity).
+* read and write that data that are quantized to eliminate false
+  precision.
 
 Visualization
 =============

RELEASE.md

@@ -17,10 +17,11 @@
 - [ ] Change the version and date in `cf/__init__.py` (`__version__` and
   `__date__` variables)
 
-- [ ] Ensure that the requirements on dependencies & their versions are
-  up-to-date and consistent in both the `requirements.txt` and in
-  `docs/source/installation.rst`; and in the `_requires` list and
-  `Version` checks in `cf/__init__.py`.
+- [ ] Ensure that the requirements on dependencies & their versions
+  are up-to-date and consistent in both the `requirements.txt` and in
+  `docs/source/installation.rst` (paying particular attention to
+  `cfdm`); and in the `_requires` list and `Version` checks in
+  `cf/__init__.py`.
 
 - [ ] Make sure that `README.md` is up to date.
 

cf/__init__.py

@@ -241,6 +241,7 @@
 from .nodecountproperties import NodeCountProperties
 from .partnodecountproperties import PartNodeCountProperties
 from .interiorring import InteriorRing
+from .quantization import Quantization
 from .tiepointindex import TiePointIndex
 
 from .bounds import Bounds

cf/cfimplementation.py

@@ -23,6 +23,7 @@
     List,
     NodeCountProperties,
     PartNodeCountProperties,
+    Quantization,
     TiePointIndex,
 )
 from .data import Data
@@ -147,6 +148,7 @@ def set_construct(self, parent, construct, axes=None, copy=True, **kwargs):
     H5netcdfArray=H5netcdfArray,
     NetCDF4Array=NetCDF4Array,
     PointTopologyArray=PointTopologyArray,
+    Quantization=Quantization,
     RaggedContiguousArray=RaggedContiguousArray,
     RaggedIndexedArray=RaggedIndexedArray,
     RaggedIndexedContiguousArray=RaggedIndexedContiguousArray,
@@ -203,6 +205,7 @@ def implementation():
      'H5netcdfArray': cf.data.array.h5netcdfarray.H5netcdfArray,
      'NetCDF4Array': cf.data.array.netcdf4array.NetCDF4Array,
      'PointTopologyArray': <class 'cf.data.array.pointtopologyarray.PointTopologyArray'>,
+     'Quantization': cf.quantization.Quantization,
      'RaggedContiguousArray': cf.data.array.raggedcontiguousarray.RaggedContiguousArray,
      'RaggedIndexedArray': cf.data.array.raggedindexedarray.RaggedIndexedArray,
      'RaggedIndexedContiguousArray': cf.data.array.raggedindexedcontiguousarray.RaggedIndexedContiguousArray,

cf/field.py

@@ -22,6 +22,7 @@
     Flags,
     Index,
     List,
+    Quantization,
     mixin,
 )
 from .constants import masked as cf_masked
@@ -280,7 +281,7 @@ def __new__(cls, *args, **kwargs):
         instance._Domain = Domain
         instance._DomainAncillary = DomainAncillary
         instance._DomainAxis = DomainAxis
-        #        instance._Data = Data
+        instance._Quantization = Quantization
         instance._RaggedContiguousArray = RaggedContiguousArray
         instance._RaggedIndexedArray = RaggedIndexedArray
         instance._RaggedIndexedContiguousArray = RaggedIndexedContiguousArray

cf/fieldancillary.py

@@ -1,7 +1,42 @@
 import cfdm
 
-from . import mixin
+from . import Quantization, mixin
 
 
 class FieldAncillary(mixin.PropertiesData, cfdm.FieldAncillary):
-    pass
+    """A field ancillary construct of the CF data model.
+
+    The field ancillary construct provides metadata which are
+    distributed over the same sampling domain as the field itself. For
+    example, if a data variable holds a variable retrieved from a
+    satellite instrument, a related ancillary data variable might
+    provide the uncertainty estimates for those retrievals (varying
+    over the same spatiotemporal domain).
+
+    The field ancillary construct consists of an array of the
+    ancillary data, which is zero-dimensional or which depends on one
+    or more of the domain axes, and properties to describe the
+    data. It is assumed that the data do not depend on axes of the
+    domain which are not spanned by the array, along which the values
+    are implicitly propagated. CF-netCDF ancillary data variables
+    correspond to field ancillary constructs. Note that a field
+    ancillary construct is constrained by the domain definition of the
+    parent field construct but does not contribute to the domain's
+    definition, unlike, for instance, an auxiliary coordinate
+    construct or domain ancillary construct.
+
+    **NetCDF interface**
+
+    {{netCDF variable}}
+
+    {{netCDF dataset chunks}}
+
+    .. versionadded:: 2.0
+
+    """
+
+    def __new__(cls, *args, **kwargs):
+        """Store component classes."""
+        instance = super().__new__(cls)
+        instance._Quantization = Quantization
+        return instance

cf/quantization.py

@@ -0,0 +1,59 @@
+import cfdm
+
+
+class Quantization(cfdm.Quantization):
+    """A quantization variable.
+
+    A quantization variable describes a quantization algorithm via a
+    collection of parameters.
+
+    The ``algorithm`` parameter names a specific quantization
+    algorithm via one of the keys in the `algorithm_parameters`
+    dictionary.
+
+    The ``implementation`` parameter contains unstandardised text that
+    concisely conveys the algorithm provenance including the name of
+    the library or client that performed the quantization, the
+    software version, and any other information required to
+    disambiguate the source of the algorithm employed. The text must
+    take the form ``software-name version version-string
+    [(optional-information)]``.
+
+    The retained precision of the algorithm is defined with either the
+    ``quantization_nsb`` or ``quantization_nsd`` parameter.
+
+    For instance, the following parameters describe quantization via
+    the BitRound algorithm, retaining 6 significant bits, and
+    implemented by libnetcdf::
+
+       >>> q = {{package}}.{{class}}(
+       ...         parameters={'algorithm': 'bitround',
+       ...                     'quantization_nsb': 6,
+       ...                     'implementation': 'libnetcdf version 4.9.4'}
+       ... )
+       >>> q.parameters()
+       {'algorithm': 'bitround',
+        'quantization_nsb': 6,
+        'implementation': 'libnetcdf version 4.9.4'}
+
+    See CF section 8.4. "Lossy Compression via Quantization".
+
+    **NetCDF interface**
+
+    {{netCDF variable}}
+
+    {{netCDF group attributes}}
+
+    .. versionadded:: NEXTVERSION
+
+    """
+
+    def __repr__(self):
+        """Called by the `repr` built-in function.
+
+        x.__repr__() <==> repr(x)
+
+        .. versionadded:: NEXTVERSION
+
+        """
+        return super().__repr__().replace("<", "<CF ", 1)

cf/test/test_Quantization.py

@@ -0,0 +1,42 @@
+import datetime
+import unittest
+
+import cf
+
+
+class QuantizationTest(unittest.TestCase):
+    """Unit test for the Quantization class."""
+
+    q = cf.Quantization({"quantization_nsd": 4, "algorithm": "bitgroom"})
+
+    def test_Quantization_algorithm_parameters(self):
+        """Test Quantization.algorithm_parameters."""
+        self.assertEqual(
+            cf.Quantization().algorithm_parameters(),
+            {
+                "bitgroom": "quantization_nsd",
+                "bitround": "quantization_nsb",
+                "digitround": "quantization_nsd",
+                "granular_bitround": "quantization_nsd",
+            },
+        )
+
+    def test_Quantization__str__(self):
+        """Test Quantization.__str__."""
+        self.assertEqual(str(self.q), "algorithm=bitgroom, quantization_nsd=4")
+
+    def test_Quantization_dump(self):
+        """Test Quantization.dump."""
+        self.assertEqual(
+            self.q.dump(display=False),
+            "Quantization: \n"
+            "    algorithm = 'bitgroom'\n"
+            "    quantization_nsd = 4",
+        )
+
+
+if __name__ == "__main__":
+    print("Run date:", datetime.datetime.now())
+    cf.environment()
+    print("")
+    unittest.main(verbosity=2)