Make use of RuleTest.expected

CHANGES.md

@@ -11,9 +11,6 @@
 - Now supporting xcube multi-level datasets `*.levels`:
   - Added xcube plugin processor `"xcube/multi-level-dataset"` that is used
     inside the predefined xcube configurations "all" and "recommended".
-- Directories that are recognized by file patterns associated with a non-empty 
-  configuration object are no longer recursively
-  traversed.
 - Introduced method `Plugin.define_config` which defines a named plugin
   configuration. It takes a name and a configuration object or list of 
   configuration objects.
@@ -24,11 +21,13 @@
   - The returned value should be a list of values that can be 
     converted into configuration objects: mixed `Config` instances,
     dictionary, or a name that refers to a named configuration of a plugin.
-- Node path names now contain the dataset index if a file path 
-  has been opened by a processor produced multiple 
-  datasets to validate.
 
 - Other changes:
+  - Directories that are recognized by file patterns associated with a non-empty 
+    configuration object are no longer recursively traversed.
+  - Node path names now contain the dataset index if a file path 
+    has been opened by a processor produced multiple 
+    datasets to validate.
   - Changed type of `Plugin.configs` from `dict[str, Config]` to 
     `dict[str, list[Config]]`.
   - Inbuilt plugin rules now import their `plugin` instance from
@@ -37,7 +36,8 @@
     serializes property values that are also default values.
   - Pinned zarr dependency to be >=2.18, <3 until test
     `tests.plugins.xcube.processors.test_mldataset.MultiLevelDatasetProcessorTest`
-    is adjusted or fsspec's memory filesystem is updated.
+    is adjusted or `fsspec`'s memory filesystem is updated.
+  - Now making use of the `expected` property of `RuleTest`.
 
 ## Version 0.3.0 (from 2025-01-20)
 

docs/todo.md

@@ -16,11 +16,10 @@
 - project logo
 - add some more tests so we reach 99% coverage
 - apply rule op args/kwargs validation schema 
-- support `RuleTest.expected`, it is currently unused
 - provide core rule that checks for configurable list of std attributes
 - measure time it takes to open a dataset and pass time into rule context 
   so we can write a configurable rule that checks the opening time
-- allow outputting suggestions that me be included in rule results
+- allow outputting suggestions, if any, that emitted by some rules
 
 ## Nice to have
 

examples/rule_testing.py

@@ -36,7 +36,7 @@ def dataset(self, ctx: RuleContext, node: DatasetNode):
     "good-title",
     GoodTitle,
     valid=[RuleTest(dataset=valid_dataset)],
-    invalid=[RuleTest(dataset=invalid_dataset)],
+    invalid=[RuleTest(dataset=invalid_dataset, expected=1)],
 )
 
 # or define a test class derived from unitest.TestCase

tests/plugins/core/rules/test_coords_for_dims.py

@@ -20,6 +20,6 @@
         RuleTest(dataset=valid_dataset_2),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_2),
+        RuleTest(dataset=invalid_dataset_2, expected=1),
     ],
 )

tests/plugins/core/rules/test_dataset_title_attr.py

@@ -17,7 +17,7 @@
         RuleTest(dataset=valid_dataset_2),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_1),
-        RuleTest(dataset=invalid_dataset_2),
+        RuleTest(dataset=invalid_dataset_1, expected=1),
+        RuleTest(dataset=invalid_dataset_2, expected=1),
     ],
 )

tests/plugins/core/rules/test_flags.py

@@ -83,13 +83,13 @@
         RuleTest(dataset=valid_dataset_3),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_0),
-        RuleTest(dataset=invalid_dataset_1),
-        RuleTest(dataset=invalid_dataset_2),
-        RuleTest(dataset=invalid_dataset_3),
-        RuleTest(dataset=invalid_dataset_4),
-        RuleTest(dataset=invalid_dataset_5),
-        RuleTest(dataset=invalid_dataset_6),
-        RuleTest(dataset=invalid_dataset_7),
+        RuleTest(dataset=invalid_dataset_0, expected=2),
+        RuleTest(dataset=invalid_dataset_1, expected=1),
+        RuleTest(dataset=invalid_dataset_2, expected=1),
+        RuleTest(dataset=invalid_dataset_3, expected=1),
+        RuleTest(dataset=invalid_dataset_4, expected=1),
+        RuleTest(dataset=invalid_dataset_5, expected=1),
+        RuleTest(dataset=invalid_dataset_6, expected=1),
+        RuleTest(dataset=invalid_dataset_7, expected=1),
     ],
 )

tests/plugins/core/rules/test_grid_mappings.py

@@ -60,9 +60,9 @@ def make_dataset():
         RuleTest(dataset=valid_dataset_2),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_1),
-        RuleTest(dataset=invalid_dataset_2),
-        RuleTest(dataset=invalid_dataset_3),
-        RuleTest(dataset=invalid_dataset_4),
+        RuleTest(dataset=invalid_dataset_1, expected=1),
+        RuleTest(dataset=invalid_dataset_2, expected=1),
+        RuleTest(dataset=invalid_dataset_3, expected=1),
+        RuleTest(dataset=invalid_dataset_4, expected=1),
     ],
 )

tests/plugins/core/rules/test_lat_lon_coordinate.py

@@ -84,9 +84,9 @@
         RuleTest(dataset=invalid_lon_dataset_2),
     ],
     invalid=[
-        RuleTest(dataset=invalid_lat_dataset_0),
-        RuleTest(dataset=invalid_lat_dataset_1),
-        RuleTest(dataset=invalid_lat_dataset_2),
+        RuleTest(dataset=invalid_lat_dataset_0, expected=1),
+        RuleTest(dataset=invalid_lat_dataset_1, expected=1),
+        RuleTest(dataset=invalid_lat_dataset_2, expected=2),
     ],
 )
 
@@ -104,8 +104,8 @@
         RuleTest(dataset=invalid_lat_dataset_2),
     ],
     invalid=[
-        RuleTest(dataset=invalid_lon_dataset_0),
-        RuleTest(dataset=invalid_lon_dataset_1),
-        RuleTest(dataset=invalid_lon_dataset_2),
+        RuleTest(dataset=invalid_lon_dataset_0, expected=1),
+        RuleTest(dataset=invalid_lon_dataset_1, expected=1),
+        RuleTest(dataset=invalid_lon_dataset_2, expected=2),
     ],
 )

tests/plugins/core/rules/test_no_empty_attrs.py

@@ -25,8 +25,8 @@
         RuleTest(dataset=valid_dataset_2),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_1),
-        RuleTest(dataset=invalid_dataset_2),
-        RuleTest(dataset=invalid_dataset_3),
+        RuleTest(dataset=invalid_dataset_1, expected=1),
+        RuleTest(dataset=invalid_dataset_2, expected=1),
+        RuleTest(dataset=invalid_dataset_3, expected=1),
     ],
 )

tests/plugins/core/rules/test_no_empty_chunks.py

@@ -36,6 +36,6 @@
         RuleTest(dataset=valid_dataset_3),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_0),
+        RuleTest(dataset=invalid_dataset_0, expected=1),
     ],
 )

tests/plugins/core/rules/test_time_coordinate.py

@@ -103,17 +103,17 @@
         RuleTest(dataset=valid_dataset_5),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_0),
-        RuleTest(dataset=invalid_dataset_1),
-        RuleTest(dataset=invalid_dataset_2),
-        RuleTest(dataset=invalid_dataset_3),
-        RuleTest(dataset=invalid_dataset_4),
-        RuleTest(dataset=invalid_dataset_5),
-        RuleTest(dataset=invalid_dataset_6),
-        RuleTest(dataset=invalid_dataset_7),
-        RuleTest(dataset=invalid_dataset_8),
-        RuleTest(dataset=invalid_dataset_9),
-        RuleTest(dataset=invalid_dataset_10),
-        RuleTest(dataset=invalid_dataset_11),
+        RuleTest(dataset=invalid_dataset_0, expected=1),
+        RuleTest(dataset=invalid_dataset_1, expected=1),
+        RuleTest(dataset=invalid_dataset_2, expected=1),
+        RuleTest(dataset=invalid_dataset_3, expected=1),
+        RuleTest(dataset=invalid_dataset_4, expected=1),
+        RuleTest(dataset=invalid_dataset_5, expected=1),
+        RuleTest(dataset=invalid_dataset_6, expected=1),
+        RuleTest(dataset=invalid_dataset_7, expected=1),
+        RuleTest(dataset=invalid_dataset_8, expected=1),
+        RuleTest(dataset=invalid_dataset_9, expected=1),
+        RuleTest(dataset=invalid_dataset_10, expected=1),
+        RuleTest(dataset=invalid_dataset_11, expected=1),
     ],
 )

tests/plugins/core/rules/test_var_units_attr.py

@@ -27,8 +27,8 @@
         RuleTest(dataset=valid_dataset_2),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_1),
-        RuleTest(dataset=invalid_dataset_2),
-        RuleTest(dataset=invalid_dataset_3),
+        RuleTest(dataset=invalid_dataset_1, expected=1),
+        RuleTest(dataset=invalid_dataset_2, expected=1),
+        RuleTest(dataset=invalid_dataset_3, expected=1),
     ],
 )

tests/plugins/xcube/rules/test_any_spatial_data_var.py

@@ -14,6 +14,6 @@
         RuleTest(dataset=valid_dataset),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset),
+        RuleTest(dataset=invalid_dataset, expected=1),
     ],
 )

tests/plugins/xcube/rules/test_cube_dims_order.py

@@ -30,30 +30,30 @@ def make_dataset(dims: tuple[str, str, str]):
     )
 
 
-valid_dataset_1 = make_dataset(("time", "y", "x"))
-valid_dataset_2 = make_dataset(("time", "lat", "lon"))
-valid_dataset_3 = make_dataset(("level", "y", "x"))
+valid_dataset_0 = make_dataset(("time", "y", "x"))
+valid_dataset_1 = make_dataset(("time", "lat", "lon"))
+valid_dataset_2 = make_dataset(("level", "y", "x"))
 
-invalid_dataset_1 = make_dataset(("time", "x", "y"))
-invalid_dataset_2 = make_dataset(("x", "y", "time"))
-invalid_dataset_3 = make_dataset(("time", "lon", "lat"))
-invalid_dataset_4 = make_dataset(("lon", "lat", "level"))
-invalid_dataset_5 = make_dataset(("x", "y", "level"))
+invalid_dataset_0 = make_dataset(("time", "x", "y"))
+invalid_dataset_1 = make_dataset(("x", "y", "time"))
+invalid_dataset_2 = make_dataset(("time", "lon", "lat"))
+invalid_dataset_3 = make_dataset(("lon", "lat", "level"))
+invalid_dataset_4 = make_dataset(("x", "y", "level"))
 
 
 CubeDimsOrderTest = RuleTester.define_test(
     "cube-dims-order",
     CubeDimsOrder,
     valid=[
+        RuleTest(dataset=valid_dataset_0),
         RuleTest(dataset=valid_dataset_1),
         RuleTest(dataset=valid_dataset_2),
-        RuleTest(dataset=valid_dataset_3),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_1),
-        RuleTest(dataset=invalid_dataset_2),
-        RuleTest(dataset=invalid_dataset_3),
-        RuleTest(dataset=invalid_dataset_4),
-        RuleTest(dataset=invalid_dataset_5),
+        RuleTest(dataset=invalid_dataset_0, expected=2),
+        RuleTest(dataset=invalid_dataset_1, expected=2),
+        RuleTest(dataset=invalid_dataset_2, expected=2),
+        RuleTest(dataset=invalid_dataset_3, expected=2),
+        RuleTest(dataset=invalid_dataset_4, expected=2),
     ],
 )

tests/plugins/xcube/rules/test_data_var_colors.py

@@ -75,8 +75,8 @@ def make_dataset():
         RuleTest(dataset=valid_dataset_1),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_1),
-        RuleTest(dataset=invalid_dataset_2),
-        RuleTest(dataset=invalid_dataset_3),
+        RuleTest(dataset=invalid_dataset_1, expected=1),
+        RuleTest(dataset=invalid_dataset_2, expected=1),
+        RuleTest(dataset=invalid_dataset_3, expected=1),
     ],
 )

tests/plugins/xcube/rules/test_grid_mapping_naming.py

@@ -52,6 +52,6 @@ def make_dataset():
         RuleTest(dataset=valid_dataset_3),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_1),
+        RuleTest(dataset=invalid_dataset_1, expected=1),
     ],
 )

tests/plugins/xcube/rules/test_increasing_time.py

@@ -58,7 +58,7 @@ def make_dataset():
         RuleTest(dataset=valid_dataset_1),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_1),
-        RuleTest(dataset=invalid_dataset_2),
+        RuleTest(dataset=invalid_dataset_1, expected=1),
+        RuleTest(dataset=invalid_dataset_2, expected=1),
     ],
 )

tests/plugins/xcube/rules/test_lat_lon_naming.py

@@ -53,11 +53,11 @@ def make_dataset(lat_dim: str, lon_dim: str):
         RuleTest(dataset=valid_dataset_1),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_1),
-        RuleTest(dataset=invalid_dataset_2),
-        RuleTest(dataset=invalid_dataset_3),
-        RuleTest(dataset=invalid_dataset_4),
-        RuleTest(dataset=invalid_dataset_5),
-        RuleTest(dataset=invalid_dataset_6),
+        RuleTest(dataset=invalid_dataset_1, expected=1),
+        RuleTest(dataset=invalid_dataset_2, expected=1),
+        RuleTest(dataset=invalid_dataset_3, expected=1),
+        RuleTest(dataset=invalid_dataset_4, expected=1),
+        RuleTest(dataset=invalid_dataset_5, expected=1),
+        RuleTest(dataset=invalid_dataset_6, expected=1),
     ],
 )

tests/plugins/xcube/rules/test_ml_dataset_meta.py

@@ -79,9 +79,9 @@ def _replace_meta(dataset: xr.Dataset, meta: LevelsMeta) -> xr.Dataset:
         RuleTest(dataset=valid_dataset_3),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_0),
-        RuleTest(dataset=invalid_dataset_1),
-        RuleTest(dataset=invalid_dataset_2),
-        RuleTest(dataset=invalid_dataset_3),
+        RuleTest(dataset=invalid_dataset_0, expected=1),
+        RuleTest(dataset=invalid_dataset_1, expected=4),
+        RuleTest(dataset=invalid_dataset_2, expected=3),
+        RuleTest(dataset=invalid_dataset_3, expected=2),
     ],
 )

tests/plugins/xcube/rules/test_ml_dataset_time.py

@@ -34,6 +34,6 @@
         RuleTest(dataset=valid_dataset_3),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_0),
+        RuleTest(dataset=invalid_dataset_0, expected=1),
     ],
 )

tests/plugins/xcube/rules/test_ml_dataset_xy.py

@@ -52,6 +52,6 @@
         RuleTest(dataset=valid_dataset_5),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_0),
+        RuleTest(dataset=invalid_dataset_0, expected=2),
     ],
 )

tests/plugins/xcube/rules/test_no_chunked_coords.py

@@ -23,6 +23,6 @@
         RuleTest(dataset=valid_dataset_2),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_0),
+        RuleTest(dataset=invalid_dataset_0, expected=1),
     ],
 )

tests/plugins/xcube/rules/test_single_grid_mapping.py

@@ -66,7 +66,7 @@ def make_dataset():
         RuleTest(dataset=valid_dataset_5),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_1),
-        RuleTest(dataset=invalid_dataset_2),
+        RuleTest(dataset=invalid_dataset_1, expected=1),
+        RuleTest(dataset=invalid_dataset_2, expected=1),
     ],
 )

tests/plugins/xcube/rules/test_time_naming.py

@@ -65,11 +65,11 @@ def make_dataset(time_var: str, time_dim: str | None = None):
         RuleTest(dataset=valid_dataset_1),
     ],
     invalid=[
-        RuleTest(dataset=invalid_dataset_0),
-        RuleTest(dataset=invalid_dataset_1),
-        RuleTest(dataset=invalid_dataset_2),
-        RuleTest(dataset=invalid_dataset_3),
-        RuleTest(dataset=invalid_dataset_4),
-        RuleTest(dataset=invalid_dataset_5),
+        RuleTest(dataset=invalid_dataset_0, expected=1),
+        RuleTest(dataset=invalid_dataset_1, expected=1),
+        RuleTest(dataset=invalid_dataset_2, expected=1),
+        RuleTest(dataset=invalid_dataset_3, expected=1),
+        RuleTest(dataset=invalid_dataset_4, expected=1),
+        RuleTest(dataset=invalid_dataset_5, expected=2),
     ],
 )