Merge branch 'main' into merge_dataless

Author: pp-mo

.github/workflows/ci-manifest.yml

@@ -23,4 +23,4 @@ concurrency:
 jobs:
   manifest:
     name: "check-manifest"
-    uses: scitools/workflows/.github/workflows/ci-manifest.yml@2025.09.6
+    uses: scitools/workflows/.github/workflows/ci-manifest.yml@2025.10.3

.github/workflows/ci-template-check.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   prompt-share:
-    uses: scitools/workflows/.github/workflows/ci-template-check.yml@2025.09.6
+    uses: scitools/workflows/.github/workflows/ci-template-check.yml@2025.10.3
     secrets: inherit
     with:
       pr_number: ${{ github.event.pull_request.number }}

.github/workflows/refresh-lockfiles.yml

@@ -14,5 +14,5 @@ on:
 
 jobs:
   refresh_lockfiles:
-    uses: scitools/workflows/.github/workflows/refresh-lockfiles.yml@2025.09.6
+    uses: scitools/workflows/.github/workflows/refresh-lockfiles.yml@2025.10.3
     secrets: inherit

.pre-commit-config.yaml

@@ -29,7 +29,7 @@ repos:
     -   id: no-commit-to-branch
 
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.13.2"
+    rev: "v0.14.0"
     hooks:
     -   id: ruff
         types: [file, python]

docs/src/whatsnew/latest.rst

@@ -45,6 +45,10 @@ This document explains the changes made to Iris for this release
    Also added a new documentation section on dataless cubes.
    (:issue:`5770`, :pull:`6581`)
 
+#. `@ukmo-ccbunney`_ added a new :class:`~iris.util.CMLSettings` class to control
+   the formatting of Cube CML output via a context manager.
+   (:issue:`6244`, :pull:`6743`)
+
 
 🐛 Bugs Fixed
 =============
@@ -70,10 +74,18 @@ This document explains the changes made to Iris for this release
 #. N/A
 
 
-🚀 Performance Enhancements
-===========================
+🚀 Performance
+==============
 
-#. N/A
+#. `@trexfeathers`_ investigated a significant performance regression in NetCDF
+   loading and saving, caused by ``libnetcdf`` version ``4.9.3``.
+   The regression is equal to several milliseconds per chunk
+   of parallel operation; so a dataset containing ~100 chunks could be around
+   0.5 seconds slower to load or save. This regression will NOT be fixed within
+   Iris - doing so would introduce unacceptable complexity and potential
+   concurrency problems. The regession has been reported to the NetCDF team; it
+   is hoped that a future ``libnetcdf`` release will recover the original
+   performance. See `netcdf-c#3183`_ for more details. (:pull:`6747`)
 
 
 🔥 Deprecations
@@ -106,9 +118,12 @@ This document explains the changes made to Iris for this release
 #. `@melissaKG`_ upgraded Iris' tests to no longer use the deprecated
    ``git whatchanged`` command. (:pull:`6672`)
 
-#. `@ukmo-ccbunney` merged functionality of ``assert_CML_approx_data`` into
+#. `@ukmo-ccbunney`_ merged functionality of ``assert_CML_approx_data`` into
    ``assert_CML`` via the use of a new ``approx_data`` keyword. (:pull:`6713`)
 
+#. `@ukmo-ccbunney`_ ``assert_CML`` now uses stricter array formatting to avoid
+   changes in tests due to Numpy version changes. (:pull:`6743`)
+
 
 .. comment
     Whatsnew author names (@github name) in alphabetical order. Note that,
@@ -119,4 +134,6 @@ This document explains the changes made to Iris for this release
 
 
 .. comment
-    Whatsnew resources in alphabetical order:
+    Whatsnew resources in alphabetical order:
+
+.. _netcdf-c#3183: https://github.com/Unidata/netcdf-c/issues/3183

lib/iris/coords.py

@@ -32,6 +32,7 @@
 import iris.exceptions
 import iris.time
 import iris.util
+from iris.util import CML_SETTINGS
 import iris.warnings
 
 #: The default value for ignore_axis which controls guess_coord_axis' behaviour
@@ -853,10 +854,45 @@ def xml_element(self, doc):
             if self.coord_system:
                 element.appendChild(self.coord_system.xml_element(doc))
 
+        is_masked_array = np.ma.isMaskedArray(self._values)
+
         # Add the values
         element.setAttribute("value_type", str(self._value_type_name()))
         element.setAttribute("shape", str(self.shape))
 
+        # data checksum
+        if CML_SETTINGS.coord_checksum:
+            crc = iris.util.array_checksum(self._values)
+            element.setAttribute("checksum", crc)
+
+            if is_masked_array:
+                # Add the number of masked elements
+                if np.ma.is_masked(self._values):
+                    crc = iris.util.array_checksum(self._values.mask)
+                else:
+                    crc = "no-masked-elements"
+                element.setAttribute("mask_checksum", crc)
+
+        # array ordering:
+        def _order(array):
+            order = ""
+            if array.flags["C_CONTIGUOUS"]:
+                order = "C"
+            elif array.flags["F_CONTIGUOUS"]:
+                order = "F"
+            return order
+
+        if CML_SETTINGS.coord_order:
+            element.setAttribute("order", _order(self._values))
+            if is_masked_array:
+                element.setAttribute("mask_order", _order(self._values.mask))
+
+        # masked element count:
+        if CML_SETTINGS.masked_value_count and is_masked_array:
+            element.setAttribute(
+                "masked_count", str(np.count_nonzero(self._values.mask))
+            )
+
         # The values are referred to "points" of a coordinate and "data"
         # otherwise.
         if isinstance(self, Coord):
@@ -865,7 +901,31 @@ def xml_element(self, doc):
             values_term = "indices"
         else:
             values_term = "data"
-        element.setAttribute(values_term, self._xml_array_repr(self._values))
+        element.setAttribute(
+            values_term,
+            self._xml_array_repr(self._values),
+        )
+
+        if iris.util.CML_SETTINGS.coord_data_array_stats and len(self._values) > 1:
+            data = self._values
+
+            if np.issubdtype(data.dtype.type, np.number):
+                data_min = data.min()
+                data_max = data.max()
+                if data_min == data_max:
+                    # When data is constant, std() is too sensitive.
+                    data_std = 0
+                else:
+                    data_std = data.std()
+
+                stats_xml_element = doc.createElement("stats")
+                stats_xml_element.setAttribute("std", str(data_std))
+                stats_xml_element.setAttribute("min", str(data_min))
+                stats_xml_element.setAttribute("max", str(data_max))
+                stats_xml_element.setAttribute("masked", str(ma.is_masked(data)))
+                stats_xml_element.setAttribute("mean", str(data.mean()))
+
+                element.appendChild(stats_xml_element)
 
         return element
 
@@ -896,7 +956,11 @@ def _xml_array_repr(data):
         if hasattr(data, "to_xml_attr"):
             result = data._values.to_xml_attr()
         else:
-            result = iris.util.format_array(data)
+            edgeitems = CML_SETTINGS.array_edgeitems
+            if CML_SETTINGS.numpy_formatting:
+                result = iris.util.format_array(data, edgeitems=edgeitems)
+            else:
+                result = iris.util.array_summary(data, edgeitems=edgeitems)
         return result
 
     def _value_type_name(self):
@@ -2565,7 +2629,10 @@ def xml_element(self, doc):
 
         # Add bounds, points are handled by the parent class.
         if self.has_bounds():
-            element.setAttribute("bounds", self._xml_array_repr(self.bounds))
+            element.setAttribute(
+                "bounds",
+                self._xml_array_repr(self.bounds),
+            )
 
         return element
 

lib/iris/cube.py

@@ -22,7 +22,6 @@
 from typing import TYPE_CHECKING, Any, Optional, TypeGuard
 import warnings
 from xml.dom.minidom import Document
-import zlib
 
 from cf_units import Unit
 import dask.array as da
@@ -56,6 +55,7 @@
     from iris.mesh import MeshCoord
 import iris.exceptions
 import iris.util
+from iris.util import CML_SETTINGS
 import iris.warnings
 
 __all__ = ["Cube", "CubeAttrsDict", "CubeList"]
@@ -171,7 +171,10 @@ def insert(self, index, cube):
         super(CubeList, self).insert(index, cube)
 
     def xml(self, checksum=False, order=True, byteorder=True):
-        """Return a string of the XML that this list of cubes represents."""
+        """Return a string of the XML that this list of cubes represents.
+
+        See :func:`iris.util.CML_SETTINGS.set` for controlling the XML output formatting.
+        """
         with np.printoptions(legacy=NP_PRINTOPTIONS_LEGACY):
             doc = Document()
             cubes_xml_element = doc.createElement("cubes")
@@ -3902,12 +3905,29 @@ def xml(
         order: bool = True,
         byteorder: bool = True,
     ) -> str:
-        """Return a fully valid CubeML string representation of the Cube."""
+        """Return a fully valid CubeML string representation of the Cube.
+
+        The format of the generated XML can be controlled using the
+        ``iris.util.CML_SETTINGS.set`` method as a context manager.
+
+        For example, to include array statistics for the coordinate data:
+
+        .. code-block:: python
+
+            with CML_SETTINGS.set(coord_data_array_stats=True):
+                print(cube.xml())
+
+        See :func:`iris.util.CML_SETTINGS.set` for more details.
+
+        """
         with np.printoptions(legacy=NP_PRINTOPTIONS_LEGACY):
             doc = Document()
 
             cube_xml_element = self._xml_element(
-                doc, checksum=checksum, order=order, byteorder=byteorder
+                doc,
+                checksum=checksum,
+                order=order,
+                byteorder=byteorder,
             )
             cube_xml_element.setAttribute("xmlns", XML_NAMESPACE_URI)
             doc.appendChild(cube_xml_element)
@@ -3916,7 +3936,13 @@ def xml(
             doc = self._sort_xml_attrs(doc)
             return iris.util._print_xml(doc)
 
-    def _xml_element(self, doc, checksum=False, order=True, byteorder=True):
+    def _xml_element(
+        self,
+        doc,
+        checksum=False,
+        order=True,
+        byteorder=True,
+    ):
         cube_xml_element = doc.createElement("cube")
 
         if self.standard_name:
@@ -4006,39 +4032,46 @@ def dimmeta_xml_element(element, typename, dimscall):
         data_xml_element = doc.createElement("data")
         data_xml_element.setAttribute("shape", str(self.shape))
 
-        # NB. Getting a checksum triggers any deferred loading,
+        # NB. Getting a checksum or data stats triggers any deferred loading,
         # in which case it also has the side-effect of forcing the
         # byte order to be native.
+
         if checksum:
             data = self.data
-
-            # Ensure consistent memory layout for checksums.
-            def normalise(data):
-                data = np.ascontiguousarray(data)
-                if data.dtype.newbyteorder("<") != data.dtype:
-                    data = data.byteswap(False)
-                    data.dtype = data.dtype.newbyteorder("<")
-                return data
-
+            crc = iris.util.array_checksum(data)
+            data_xml_element.setAttribute("checksum", crc)
             if ma.isMaskedArray(data):
-                # Fill in masked values to avoid the checksum being
-                # sensitive to unused numbers. Use a fixed value so
-                # a change in fill_value doesn't affect the
-                # checksum.
-                crc = "0x%08x" % (zlib.crc32(normalise(data.filled(0))) & 0xFFFFFFFF,)
-                data_xml_element.setAttribute("checksum", crc)
                 if ma.is_masked(data):
-                    crc = "0x%08x" % (zlib.crc32(normalise(data.mask)) & 0xFFFFFFFF,)
+                    crc = iris.util.array_checksum(data.mask)
                 else:
                     crc = "no-masked-elements"
                 data_xml_element.setAttribute("mask_checksum", crc)
+
+        if CML_SETTINGS.data_array_stats:
+            data = self.data
+            data_min = data.min()
+            data_max = data.max()
+            if data_min == data_max:
+                # When data is constant, std() is too sensitive.
+                data_std = 0
             else:
-                crc = "0x%08x" % (zlib.crc32(normalise(data)) & 0xFFFFFFFF,)
-                data_xml_element.setAttribute("checksum", crc)
-        elif self.has_lazy_data():
-            data_xml_element.setAttribute("state", "deferred")
-        else:
-            data_xml_element.setAttribute("state", "loaded")
+                data_std = data.std()
+
+            stats_xml_element = doc.createElement("stats")
+            stats_xml_element.setAttribute("std", str(data_std))
+            stats_xml_element.setAttribute("min", str(data_min))
+            stats_xml_element.setAttribute("max", str(data_max))
+            stats_xml_element.setAttribute("masked", str(ma.is_masked(data)))
+            stats_xml_element.setAttribute("mean", str(data.mean()))
+
+            data_xml_element.appendChild(stats_xml_element)
+
+        # We only print the "state" if we have not output checksum or data stats:
+        if not (checksum or CML_SETTINGS.data_array_stats):
+            if self.has_lazy_data():
+                data_xml_element.setAttribute("state", "deferred")
+            else:
+                data_xml_element.setAttribute("state", "loaded")
 
         # Add the dtype, and also the array and mask orders if the
         # data is loaded.
@@ -4065,8 +4098,14 @@ def _order(array):
                 if array_byteorder is not None:
                     data_xml_element.setAttribute("byteorder", array_byteorder)
 
-            if order and ma.isMaskedArray(data):
-                data_xml_element.setAttribute("mask_order", _order(data.mask))
+            if ma.isMaskedArray(data):
+                if CML_SETTINGS.masked_value_count:
+                    data_xml_element.setAttribute(
+                        "masked_count", str(np.count_nonzero(data.mask))
+                    )
+                if order:
+                    data_xml_element.setAttribute("mask_order", _order(data.mask))
+
         else:
             dtype = self.lazy_data().dtype
         data_xml_element.setAttribute("dtype", dtype.name)