Implement dataless concatenation (#6860)

* made concat work datalessly

* Apply suggestions from code review

Co-authored-by: Chris Bunney <[email protected]>

* added whatsnew

* Added missing comma

---------

Co-authored-by: Chris Bunney <[email protected]>

Author: ESadek-MO

docs/src/whatsnew/latest.rst

@@ -44,9 +44,9 @@ This document explains the changes made to Iris for this release
 
    (:issue:`5819`, :pull:`6854`)
 
-#. `@ESadek-MO`_ added functionality to allow :func:`~iris.cube.Cube.rolling_window` and
-   :func:`~iris.cube.Cube.intersection` to work with dataless cubes. (:pull:`6757`)
-
+#. `@ESadek-MO`_ added functionality to allow :func:`~iris.cube.Cube.concatenate`,
+   :func:`~iris.cube.Cube.rolling_window` and :func:`~iris.cube.Cube.intersection`
+   to work with dataless cubes. (:pull:`6860`, :pull:`6757`)
 
 🐛 Bugs Fixed
 =============

lib/iris/_concatenate.py

@@ -239,7 +239,15 @@ def name(self):
         return self.defn.name()
 
 
-class _SkeletonCube(namedtuple("SkeletonCube", ["signature", "data"])):
+class _SkeletonCube(
+    namedtuple(
+        "SkeletonCube",
+        ["signature", "data", "shape"],
+        defaults=[
+            None,
+        ],
+    )
+):
     """Basis of a source-cube.
 
     Basis of a source-cube, containing the associated coordinate metadata,
@@ -575,8 +583,6 @@ def concatenate(
     """
     cube_signatures = []
     for cube in cubes:
-        if cube.is_dataless():
-            raise iris.exceptions.DatalessError("concatenate")
         cube_signatures.append(_CubeSignature(cube))
 
     proto_cubes: list[_ProtoCube] = []
@@ -869,8 +875,13 @@ def match(self, other, error_on_mismatch):
             msgs.append(
                 msg_template.format("Data dimensions", "", self.ndim, other.ndim)
             )
-        # Check data type.
-        if self.data_type != other.data_type:
+        if (
+            self.data_type is not None
+            and other.data_type is not None
+            and self.data_type != other.data_type
+        ):
+            # N.B. allow "None" to match any other dtype: this means that dataless
+            # cubes can merge with 'dataful' ones.
             msgs.append(
                 msg_template.format("Data types", "", self.data_type, other.data_type)
             )
@@ -1026,7 +1037,9 @@ def __init__(self, cube_signature):
 
         # The list of source-cubes relevant to this proto-cube.
         self._skeletons = []
-        self._add_skeleton(self._coord_signature, self._cube.lazy_data())
+        self._add_skeleton(
+            self._coord_signature, self._cube.lazy_data(), shape=self._cube.shape
+        )
 
         # The nominated axis of concatenation.
         self._axis = None
@@ -1090,6 +1103,10 @@ def concatenate(self):
 
             # Concatenate the new data payload.
             data = self._build_data()
+            if data is None:
+                shape = [coord.shape[0] for coord, _dim in dim_coords_and_dims]
+            else:
+                shape = None
 
             # Build the new cube.
             all_aux_coords_and_dims = aux_coords_and_dims + [
@@ -1098,6 +1115,7 @@ def concatenate(self):
             kwargs = cube_signature.defn._asdict()
             cube = iris.cube.Cube(
                 data,
+                shape=shape,
                 dim_coords_and_dims=dim_coords_and_dims,
                 aux_coords_and_dims=all_aux_coords_and_dims,
                 cell_measures_and_dims=cell_measures_and_dims,
@@ -1268,7 +1286,11 @@ def check_coord_match(coord_type: str) -> tuple[bool, str]:
 
         if match:
             # Register the cube as a source-cube for this proto-cube.
-            self._add_skeleton(coord_signature, cube_signature.src_cube.lazy_data())
+            self._add_skeleton(
+                coord_signature,
+                cube_signature.src_cube.lazy_data(),
+                shape=cube_signature.src_cube.shape,
+            )
             # Declare the nominated axis of concatenation.
             self._axis = candidate_axis
             # If the protocube dimension order is constant (indicating it was
@@ -1284,7 +1306,7 @@ def check_coord_match(coord_type: str) -> tuple[bool, str]:
 
         return match, mismatch_error_msg
 
-    def _add_skeleton(self, coord_signature, data):
+    def _add_skeleton(self, coord_signature, data, shape=None):
         """Create and add the source-cube skeleton to the :class:`_ProtoCube`.
 
         Parameters
@@ -1298,7 +1320,7 @@ def _add_skeleton(self, coord_signature, data):
             source-cube.
 
         """
-        skeleton = _SkeletonCube(coord_signature, data)
+        skeleton = _SkeletonCube(coord_signature, data, shape)
         self._skeletons.append(skeleton)
 
     def _build_aux_coordinates(self):
@@ -1534,9 +1556,22 @@ def _build_data(self):
 
         """
         skeletons = self._skeletons
-        data = [skeleton.data for skeleton in skeletons]
 
-        data = concatenate_arrays(data, self.axis)
+        if all(skeleton.data is None for skeleton in skeletons):
+            data = None
+        else:
+            data = []
+            for skeleton in skeletons:
+                if skeleton.data is None:
+                    skeleton_data = da.ma.masked_array(
+                        data=da.zeros(skeleton.shape, dtype=np.int8),
+                        mask=da.ones(skeleton.shape),
+                    )
+                else:
+                    skeleton_data = skeleton.data
+                data.append(skeleton_data)
+
+            data = concatenate_arrays(data, self.axis)
 
         return data
 

lib/iris/tests/unit/cube/test_CubeList.py

@@ -44,21 +44,44 @@ def test_fail(self):
 
 
 class Test_concatenate_cube:
-    @pytest.fixture(autouse=True)
-    def _setup(self):
+    @pytest.fixture(autouse=True, params=[True, False], ids=["dataless", "with_data"])
+    def _setup(self, request):
         self.units = Unit("days since 1970-01-01 00:00:00", calendar="standard")
-        self.cube1 = Cube([1, 2, 3], "air_temperature", units="K")
+        if request.param:
+            self.cube1 = Cube(shape=(3,), standard_name="air_temperature", units="K")
+        else:
+            self.cube1 = Cube(
+                data=[1, 2, 3], standard_name="air_temperature", units="K"
+            )
         self.cube1.add_dim_coord(DimCoord([0, 1, 2], "time", units=self.units), 0)
 
-    def test_pass(self):
-        self.cube2 = Cube([1, 2, 3], "air_temperature", units="K")
-        self.cube2.add_dim_coord(DimCoord([3, 4, 5], "time", units=self.units), 0)
-        result = CubeList([self.cube1, self.cube2]).concatenate_cube()
+    @pytest.mark.parametrize(
+        "dataless_c2", [True, False], ids=["and-dataless", "and-with_data"]
+    )
+    def test_pass(self, dataless_c2):
+        if dataless_c2:
+            data = None
+            shape = (3,)
+        else:
+            data = [1, 2, 3]
+            shape = None
+        cube2 = Cube(data=data, shape=shape, standard_name="air_temperature", units="K")
+        cube2.add_dim_coord(DimCoord([3, 4, 5], "time", units=self.units), 0)
+        result = CubeList([self.cube1, cube2]).concatenate_cube()
         assert isinstance(result, Cube)
 
-    def test_fail(self):
+    @pytest.mark.parametrize(
+        "dataless_c2", [True, False], ids=["and-dataless", "and-with_data"]
+    )
+    def test_fail(self, dataless_c2):
         units = Unit("days since 1970-01-02 00:00:00", calendar="standard")
-        cube2 = Cube([1, 2, 3], "air_temperature", units="K")
+        if dataless_c2:
+            data = None
+            shape = (3,)
+        else:
+            data = [1, 2, 3]
+            shape = None
+        cube2 = Cube(data=data, shape=shape, standard_name="air_temperature", units="K")
         cube2.add_dim_coord(DimCoord([0, 1, 2], "time", units=units), 0)
         with pytest.raises(iris.exceptions.ConcatenateError):
             CubeList([self.cube1, cube2]).concatenate_cube()