test: replace MockField with a real ekd field in tests (#236)

## Description
To avoid the problem of "mock skew" (i.e. the behaviour of a mock
differing from the real object), the `MockField` in the tests has been
replaced with a real earthkit data field (in memory)

---------

Co-authored-by: Harrison Cook <Harrison.cook@ecmwf.int>

Author: aaron-hopkinson

tests/test_fields.py

@@ -13,14 +13,7 @@
 from anemoi.transform.fields import new_field_with_metadata
 from src.anemoi.transform.fields import FieldSelection
 
-
-class MockField:
-    def __init__(self, **metadata):
-        self._metadata = metadata
-
-    # FieldSelection only sends the metadata message to objects
-    def metadata(self, key):
-        return self._metadata[key]
+from .utils import mock_field
 
 
 @pytest.fixture
@@ -74,7 +67,7 @@ def test_field_adding_metadata_updates_keys(sample_field):
 
 def test_fieldselection_match_all():
     """Test FieldSelection with no arguments matches all fields."""
-    field = MockField(invalid_key="any_value")
+    field = mock_field(invalid_key="any_value")
     selection = FieldSelection()
     assert selection.match(field)
 
@@ -87,55 +80,55 @@ def test_fieldselection_invalid_key():
 
 def test_fieldselection_match_fail_different_param():
     """Test FieldSelection match fails when param is different."""
-    field = MockField(param="2t")
+    field = mock_field(param="2t")
     selection = FieldSelection(param="2z")
     assert not selection.match(field)
 
 
 def test_fieldselection_match_same_param():
     """Test FieldSelection match succeeds when param is the same."""
-    field = MockField(param="2t")
+    field = mock_field(param="2t")
     selection = FieldSelection(param="2t")
     assert selection.match(field)
 
 
 def test_fieldselection_match_fail_missing_key():
     """Test FieldSelection match fails when a selection key is missing on the field."""
-    field = MockField(param="t")
+    field = mock_field(param="t")
     selection = FieldSelection(param="t", levelist=850)
     assert not selection.match(field)
 
 
 def test_fieldselection_match_field_with_extra_metadata():
     """Test FieldSelection match succeeds when the field has extra metadata."""
-    field = MockField(param="t", levelist=850)
+    field = mock_field(param="t", levelist=850)
     selection = FieldSelection(param="t")
     assert selection.match(field)
 
 
 def test_fieldselection_match_fail_same_param_different_level():
     """Test FieldSelection match fails when param is the same but the levelist is different."""
-    field = MockField(param="t", levelist=100)
+    field = mock_field(param="t", levelist=100)
     selection = FieldSelection(param="t", levelist=850)
     assert not selection.match(field)
 
 
 def test_fieldselection_match_same_param_same_level():
     """Test FieldSelection match succeeds when param and level are the same."""
-    field = MockField(param="t", levelist=850)
+    field = mock_field(param="t", levelist=850)
     selection = FieldSelection(param="t", levelist=850)
     assert selection.match(field)
 
 
 def test_fieldselection_match_is_subset():
     """Test FieldSelection match succeeds when the field is a subset of the selection."""
-    field = MockField(param="t", levelist=850)
+    field = mock_field(param="t", levelist=850)
     selection = FieldSelection(param=["t", "q"], levelist=[850, 950])
     assert selection.match(field)
 
 
 def test_fieldselection_match_fail_different_param_same_level():
     """Test FieldSelection match fails when the is on the same level but a different param."""
-    field = MockField(param="t", levelist=850)
+    field = mock_field(param="t", levelist=850)
     selection = FieldSelection(param="q", levelist=[850, 950])
     assert not selection.match(field)

tests/test_grouping.py

@@ -1,41 +1,9 @@
+import earthkit.data as ekd
 import pytest
 
 from anemoi.transform.grouping import GroupByParam
 
-
-class MockField:
-    def __init__(self, **metadata):
-        self._metadata = metadata
-
-    def metadata(self, key=None, namespace=None, **kwargs):
-        MARS_KEYS = [
-            "domain",
-            "levtype",
-            "levelist",
-            "date",
-            "time",
-            "step",
-            "param",
-            "class",
-            "type",
-            "stream",
-            "expver",
-        ]
-        if namespace and (key or kwargs):
-            raise ValueError("Cannot specify both namespace and key, or namespace and kwargs")
-        if key and kwargs:
-            raise ValueError("Cannot specify both key and kwargs")
-
-        if namespace == "mars":
-            return {k: self._metadata[k] for k in MARS_KEYS if k in self._metadata}
-        elif namespace:
-            raise ValueError(f"Unknown namespace {namespace}")
-        if key:
-            return self._metadata[key]
-        return {k: self._metadata[k] for k in kwargs}
-
-    def __repr__(self):
-        return f"MockField({self._metadata})"
+from .utils import mock_field
 
 
 def field_generator(**metadata_values):
@@ -58,7 +26,7 @@ def field_generator(**metadata_values):
     combinations = itertools.product(*metadata_values.values())
     for values in combinations:
         metadata = MOCK_MARS_METADATA | dict(zip(metadata_values.keys(), values))
-        fields.append(MockField(**metadata))
+        fields.append(mock_field(**metadata))
     return fields
 
 
@@ -112,7 +80,7 @@ def test_group_by_param_vertical(sample_fields_vertical):
     from anemoi.transform.grouping import GroupByParamVertical
 
     def get_param(f):
-        if isinstance(f, MockField):
+        if isinstance(f, ekd.Field):
             f = [f]
 
         param = [x.metadata("param") for x in f]
@@ -131,7 +99,7 @@ def get_param(f):
         assert [get_param(field) for field in group] == match_params
         metadata = []
         for fields in group:
-            if isinstance(fields, MockField):
+            if isinstance(fields, ekd.Field):
                 fields = [fields]
             for field in fields:
                 num_matching += 1

tests/test_matching.py

@@ -10,28 +10,13 @@
 
 from collections.abc import Iterator
 
-import numpy as np
+import earthkit.data as ekd
 import pytest
 
 from anemoi.transform.filters.matching import MatchingFieldsFilter
 from anemoi.transform.filters.matching import matching
 
-
-class MockField:
-    def __init__(self, param, **meta):
-        self._param = param
-        self._meta = meta
-        self.values = np.array([1.0])  # dummy data
-
-    def metadata(self, namespace=None):
-        if namespace == "mars":
-            return dict(self._meta, param=self._param)
-        return self._param
-
-
-class MockFieldList(list):
-    def metadata(self, name):
-        return [getattr(f, "metadata")("mars")[name] for f in self]
+from .utils import mock_field
 
 
 class AddFields(MatchingFieldsFilter):
@@ -41,15 +26,13 @@ def __init__(self, a, b, return_inputs="none"):
         self.b = b
         self.return_inputs = return_inputs
 
-    def forward_transform(self, a: MockField, b: MockField) -> Iterator[MockField]:
+    def forward_transform(self, a: ekd.Field, b: ekd.Field) -> Iterator[ekd.Field]:
         result = a.values + b.values
         yield self.new_field_from_numpy(result, template=a, param="c")
 
     def new_field_from_numpy(self, array, *, template, param):
-        return MockField(param, **template._meta)
-
-    def new_fieldlist_from_list(self, fields):
-        return MockFieldList(fields)
+        metadata = dict(template.metadata()) | {"param": param}
+        return mock_field(**metadata)
 
 
 def test_matching_decorator_initializes_correctly():
@@ -59,43 +42,43 @@ def test_matching_decorator_initializes_correctly():
 
 
 def test_forward_transform_adds_fields():
-    a = MockField("a", step=0, level=850)
-    b = MockField("b", step=0, level=850)
-    data = MockFieldList([a, b])
+    a = mock_field(param="a", step=0, level=850)
+    b = mock_field(param="b", step=0, level=850)
+    data = ekd.SimpleFieldList([a, b])
 
     f = AddFields(a="a", b="b")
     result = f.forward(data)
     assert len(result) == 1
-    assert isinstance(result[0], MockField)
-    assert result[0]._param == "c"
+    assert isinstance(result[0], ekd.Field)
+    assert result[0].metadata("param") == "c"
 
 
 def test_return_inputs():
-    a = MockField("a", step=0, level=850)
-    b = MockField("b", step=0, level=850)
-    data = MockFieldList([a, b])
+    a = mock_field(param="a", step=0, level=850)
+    b = mock_field(param="b", step=0, level=850)
+    data = ekd.SimpleFieldList([a, b])
 
     f = AddFields(a="a", b="b", return_inputs="all")
     result = f.forward(data)
     assert len(result) == 3
     for i in range(3):
-        assert isinstance(result[i], MockField)
-    assert {result[i]._param for i in range(2)} == {"a", "b"}
-    assert result[2]._param == "c"
+        assert isinstance(result[i], ekd.Field)
+    assert {result[i].metadata("param") for i in range(2)} == {"a", "b"}
+    assert result[2].metadata("param") == "c"
 
     f = AddFields(a="a", b="b", return_inputs=["a"])
     result = f.forward(data)
     assert len(result) == 2
     for i in range(2):
-        assert isinstance(result[i], MockField)
-    assert result[0]._param == "a"
-    assert result[1]._param == "c"
+        assert isinstance(result[i], ekd.Field)
+    assert result[0].metadata("param") == "a"
+    assert result[1].metadata("param") == "c"
 
 
 def test_missing_component_raises():
-    a = MockField("a", step=0, level=850)
+    a = mock_field(param="a", step=0, level=850)
     # Missing 'b'
-    data = MockFieldList([a])
+    data = ekd.SimpleFieldList([a])
     f = AddFields(a="a", b="b")
 
     with pytest.raises(ValueError):
@@ -113,9 +96,9 @@ def forward_transform(self, *args):
 
 
 def test_metadata_mismatch_warning(caplog):
-    c = MockField("c", step=0, level=850)
-    d = MockField("d", step=0, level=850)
-    data = MockFieldList([c, d])
+    c = mock_field(param="c", step=0, level=850)
+    d = mock_field(param="d", step=0, level=850)
+    data = ekd.SimpleFieldList([c, d])
 
     f = AddFields(a="a", b="b")
 

tests/utils/__init__.py

@@ -8,6 +8,7 @@
 # nor does it submit to any jurisdiction.
 from collections import defaultdict
 
+import earthkit.data as ekd
 import numpy as np
 
 from anemoi.transform.fields import new_fieldlist_from_list
@@ -104,5 +105,15 @@ def compare_npz_files(file1, file2):
         assert (data1[key] == data2[key]).all(), f"Data for key {key} does not match between {file1} and {file2}"
 
 
+def mock_field(**metadata):
+    class MetadataOverride(ekd.core.metadata.RawMetadata):
+        def as_namespace(self, namespace):
+            if namespace != "mars":
+                raise ValueError(f"Unknown namespace {namespace}")
+            return dict(self)
+
+    return ekd.ArrayField(array=[1], metadata=MetadataOverride(**metadata))
+
+
 def create_tabular_filter(name, **kwargs):
     return create_filter(name, **kwargs)