Merge pull request #842 from davidhassell/cfa-in-cfdm

Import CFA from cfdm

Author: davidhassell

Changelog.rst

@@ -1,10 +1,16 @@
 version 3.17.0
 --------------
 
-**2025-02-??**
+**2025-??-??**
 
+* Replace dataset aggregation functionality (CFA) with that imported
+  from `cfdm` (https://github.com/NCAS-CMS/cf-python/issues/841)
 * New keyword parameter to `cf.Field.compute_vertical_coordinates`:
   ``key`` (https://github.com/NCAS-CMS/cf-python/issues/802)
+* Changed dependency: ``1.12.0.0<=cfdm<1.12.1.0``
+* Changed dependency: ``h5py>=3.12.0``
+
+----
 
 
 version 3.16.3

cf/__init__.py

@@ -105,7 +105,6 @@
     raise ImportError(_error0 + str(error1))
 
 __cf_version__ = cfdm.core.__cf_version__
-__cfa_version__ = "0.6.2"
 
 from packaging.version import Version
 import importlib.util
@@ -207,13 +206,10 @@
     )
 
 # Check the version of cfdm
-_minimum_vn = "1.11.2.0"
-_maximum_vn = "1.11.3.0"
-if (
-    not Version(_minimum_vn)
-    <= Version(cfdm.__version__)
-    < Version(_maximum_vn)
-):
+_minimum_vn = "1.12.0.0"
+_maximum_vn = "1.12.1.0"
+_cfdm_version = Version(cfdm.__version__)
+if not Version(_minimum_vn) <= _cfdm_version < Version(_maximum_vn):
     raise RuntimeError(
         f"Bad cfdm version: cf requires {_minimum_vn}<=cfdm<{_maximum_vn}. "
         f"Got {cfdm.__version__} at {cfdm.__file__}"
@@ -291,10 +287,9 @@
 from .field import Field
 from .data import Data
 from .data.array import (
+    AggregatedArray,
     BoundsFromNodesArray,
     CellConnectivityArray,
-    CFAH5netcdfArray,
-    CFANetCDF4Array,
     FullArray,
     GatheredArray,
     H5netcdfArray,
@@ -308,12 +303,6 @@
     UMArray,
 )
 
-from .data.fragment import (
-    FullFragmentArray,
-    NetCDFFragmentArray,
-    UMFragmentArray,
-)
-
 from .aggregate import aggregate, climatology_cells
 from .query import (
     Query,

cf/aggregate.py

@@ -426,6 +426,7 @@ def __init__(
         # Promote selected properties to field ancillaries that span
         # the same domain axes as the field
         # ------------------------------------------------------------
+        self.promoted_field_ancillaries = []
         if field_ancillaries:
             f = self.promote_to_field_ancillary(field_ancillaries)
 
@@ -2088,13 +2089,6 @@ def promote_to_field_ancillary(self, properties):
         ancillary construct that spans the entire domain, with the
         constant value of the property.
 
-        The `Data` of any new field ancillary construct is marked
-        as a CFA term, meaning that it will only be written to disk if
-        the parent field construct is written as a CFA aggregation
-        variable, and in that case the field ancillary is written as a
-        non-standard CFA aggregation instruction variable, rather than
-        a CF-netCDF ancillary variable.
-
         If a domain construct is being aggregated then it is always
         returned unchanged.
 
@@ -2125,7 +2119,6 @@ def promote_to_field_ancillary(self, properties):
             data = Data(
                 FullArray(value, shape=f.shape, dtype=np.array(value).dtype)
             )
-            data._cfa_set_term(True)
 
             field_anc = FieldAncillary(
                 data=data, properties={"long_name": prop}, copy=False
@@ -2137,9 +2130,15 @@ def promote_to_field_ancillary(self, properties):
                 f = f.copy()
                 copy = False
 
-            f.set_construct(field_anc, axes=f.get_data_axes(), copy=False)
+            key = f.set_construct(
+                field_anc, axes=f.get_data_axes(), copy=False
+            )
             f.del_property(prop)
 
+            # Record that this field ancillary is derived from a
+            # promotion
+            self.promoted_field_ancillaries.append(key)
+
         self.field = f
         return f
 
@@ -2434,9 +2433,9 @@ def aggregate(
             Create new field ancillary constructs for each input field
             which has one or more of the given properties. For each
             input field, each property is converted to a field
-            ancillary construct that spans the entire domain, with the
-            constant value of the property, and the property itself is
-            deleted.
+            ancillary construct that spans the aggregation axes with
+            the constant value of the property, and the property
+            itself is deleted.
 
             .. versionadded:: 3.15.0
 
@@ -3039,6 +3038,9 @@ def aggregate(
 
         unaggregatable = False
 
+        # Record the names of the axes that are actually aggregated
+        axes_aggregated = []
+
         for axis in aggregating_axes:
             number_of_fields = len(meta)
             if number_of_fields == 1:
@@ -3251,6 +3253,7 @@ def aggregate(
             # the aggregated fields as a single list ready for
             # aggregation along the next axis.
             # --------------------------------------------------------
+            axes_aggregated.append(axis)
             meta = [m for gm in grouped_meta for m in gm]
 
         # Add fields to the output list
@@ -3267,6 +3270,10 @@ def aggregate(
     if cells:
         _set_cell_conditions(output_meta)
 
+    # Remove non-aggregated axes from promoted field ancillaries
+    if field_ancillaries:
+        _fix_promoted_field_ancillaries(output_meta, axes_aggregated)
+
     output_constructs = [m.field for m in output_meta]
 
     aggregate.status = status
@@ -4724,6 +4731,14 @@ def _aggregate_2_fields(
             hash_value1 = anc1["hash_value"]
             anc0["hash_value"] = hash_value0 + hash_value1
 
+            # The result of aggregating a promoted amd non-promoted
+            # field ancillary is a non-promoted field ancillary
+            if (
+                key0 in m0.promoted_field_ancillaries
+                and key1 not in m1.promoted_field_ancillaries
+            ):
+                m0.promoted_field_ancillaries.remove(key0)
+
     # Domain ancillaries
     for identity in m0.domain_anc:
         anc0 = m0.domain_anc[identity]
@@ -4745,9 +4760,9 @@ def _aggregate_2_fields(
             anc0["hash_value"] = hash_value0 + hash_value1
 
     # ----------------------------------------------------------------
-    # For each matching pair of coordinates, cell measures, field and
-    # domain ancillaries which span the aggregating axis, insert the
-    # one from parent1 into the one from parent0
+    # For each matching pair of coordinates, cell measures, and field
+    # and domain ancillaries which span the aggregating axis, insert
+    # the one from parent1 into the one from parent0
     # ----------------------------------------------------------------
     for key0, key1, construct0, construct1 in spanning_variables:
         construct_axes0 = parent0.get_data_axes(key0)
@@ -4909,7 +4924,7 @@ def _aggregate_2_fields(
                     actual_range = parent0.del_property("actual_range", None)
                     if actual_range is not None and is_log_level_info(logger):
                         logger.info(
-                            "Deleted 'actual_range' attribute due to being "
+                            "Deleted 'actual_range' attribute due to it being "
                             "outside of 'valid_range' attribute limits."
                         )
 
@@ -4919,7 +4934,6 @@ def _aggregate_2_fields(
 
     # Make a note that the parent construct in this _Meta object has
     # already been aggregated
-
     m0.aggregated_field = True
 
     # ----------------------------------------------------------------
@@ -4986,3 +5000,53 @@ def dsg_feature_type_axis(meta, axis):
     # cf_role property
     cf_role = coords["cf_role"]
     return cf_role.count(None) != len(cf_role)
+
+
+def _fix_promoted_field_ancillaries(output_meta, axes_aggregated):
+    """Remove non-aggregated axes from promoted field ancillaries.
+
+    .. versionadded:: NEXTVERSION
+
+    :Parameters:
+
+        output_meta: `list`
+            The list of `_Meta` objects. If any include promoted field
+            ancillaries then these will be updated in-place.
+
+    :Returns:
+
+        `None`
+
+    """
+    for m in output_meta:
+        for value in m.field_anc.values():
+            index = []
+            squeeze = []
+
+            key = value["key"]
+            if key not in m.promoted_field_ancillaries:
+                continue
+
+            # Remove the non-aggregated axes from the promoted field
+            # ancillary
+            for i, axis in enumerate(value["axes"]):
+                if axis in axes_aggregated:
+                    index.append(slice(None))
+                else:
+                    index.append(0)
+                    squeeze.append(i)
+
+            if not squeeze:
+                continue
+
+            fa_axes = m.field.get_data_axes(key)
+            fa = m.field.del_construct(key)
+            fa = fa[tuple(index)]
+            fa.squeeze(squeeze, inplace=True)
+            fa_axes = [a for i, a in enumerate(fa_axes) if i not in squeeze]
+
+            # Record the field ancillary as being able to be written
+            # as a CF-netCDF aggregation 'value' variable
+            fa.data._nc_set_aggregation_fragment_type("value")
+
+            m.field.set_construct(fa, axes=fa_axes, copy=False)

cf/cfimplementation.py

@@ -27,10 +27,9 @@
 )
 from .data import Data
 from .data.array import (
+    AggregatedArray,
     BoundsFromNodesArray,
     CellConnectivityArray,
-    CFAH5netcdfArray,
-    CFANetCDF4Array,
     GatheredArray,
     H5netcdfArray,
     NetCDF4Array,
@@ -114,49 +113,14 @@ def set_construct(self, parent, construct, axes=None, copy=True, **kwargs):
             parent, construct, axes=axes, copy=copy, **kwargs
         )
 
-    def initialise_CFANetCDF4Array(self, **kwargs):
-        """Return a `CFANetCDF4Array` instance.
-
-        :Parameters:
-
-            kwargs: optional
-                Initialisation parameters to pass to the new instance.
-
-        :Returns:
-
-            `CFANetCDF4Array`
-
-        """
-        cls = self.get_class("CFANetCDF4Array")
-        return cls(**kwargs)
-
-    def initialise_CFAH5netcdfArray(self, **kwargs):
-        """Return a `CFAH5netcdfArray` instance.
-
-        .. versionadded:: 1.11.2.0
-
-        :Parameters:
-
-            kwargs: optional
-                Initialisation parameters to pass to the new instance.
-
-        :Returns:
-
-            `CFAH5netcdfArray`
-
-        """
-        cls = self.get_class("CFAH5netcdfArray")
-        return cls(**kwargs)
-
 
 _implementation = CFImplementation(
     cf_version=CF(),
+    AggregatedArray=AggregatedArray,
     AuxiliaryCoordinate=AuxiliaryCoordinate,
     CellConnectivity=CellConnectivity,
     CellMeasure=CellMeasure,
     CellMethod=CellMethod,
-    CFAH5netcdfArray=CFAH5netcdfArray,
-    CFANetCDF4Array=CFANetCDF4Array,
     CoordinateReference=CoordinateReference,
     DimensionCoordinate=DimensionCoordinate,
     Domain=Domain,
@@ -214,8 +178,6 @@ def implementation():
      'CellConnectivityArray': cf.data.array.cellconnectivityarray.CellConnectivityArray,
      'CellMeasure': cf.cellmeasure.CellMeasure,
      'CellMethod': cf.cellmethod.CellMethod,
-     'CFAH5netcdfArray': cf.data.array.cfah5netcdfarray.CFAH5netcdfArray,
-     'CFANetCDF4Array': cf.data.array.cfanetcdf4array.CFANetCDF4Array,
      'CoordinateReference': cf.coordinatereference.CoordinateReference,
      'DimensionCoordinate': cf.dimensioncoordinate.DimensionCoordinate,
      'Domain': cf.domain.Domain,

cf/data/array/__init__.py

@@ -1,7 +1,6 @@
+from .aggregatedarray import AggregatedArray
 from .boundsfromnodesarray import BoundsFromNodesArray
 from .cellconnectivityarray import CellConnectivityArray
-from .cfah5netcdfarray import CFAH5netcdfArray
-from .cfanetcdf4array import CFANetCDF4Array
 from .fullarray import FullArray
 from .gatheredarray import GatheredArray
 from .h5netcdfarray import H5netcdfArray

cf/data/array/abstract/__init__.py

@@ -1,2 +1 @@
 from .array import Array
-from .filearray import FileArray

cf/data/array/abstract/array.py

@@ -1,10 +1,9 @@
 import cfdm
 
 from ....mixin_container import Container
-from ..mixin import ArrayMixin
 
 
-class Array(ArrayMixin, Container, cfdm.Array):
+class Array(Container, cfdm.Array):
     """Abstract base class for a container of an underlying array.
 
     The form of the array is defined by the initialisation parameters