diff --git a/CHANGES.md b/CHANGES.md index a56980b..ee54e97 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,9 +11,6 @@ - Now supporting xcube multi-level datasets `*.levels`: - Added xcube plugin processor `"xcube/multi-level-dataset"` that is used inside the predefined xcube configurations "all" and "recommended". -- Directories that are recognized by file patterns associated with a non-empty - configuration object are no longer recursively - traversed. - Introduced method `Plugin.define_config` which defines a named plugin configuration. It takes a name and a configuration object or list of configuration objects. @@ -24,11 +21,13 @@ - The returned value should be a list of values that can be converted into configuration objects: mixed `Config` instances, dictionary, or a name that refers to a named configuration of a plugin. -- Node path names now contain the dataset index if a file path - has been opened by a processor produced multiple - datasets to validate. - Other changes: + - Directories that are recognized by file patterns associated with a non-empty + configuration object are no longer recursively traversed. + - Node path names now contain the dataset index if a file path + has been opened by a processor produced multiple + datasets to validate. - Changed type of `Plugin.configs` from `dict[str, Config]` to `dict[str, list[Config]]`. - Inbuilt plugin rules now import their `plugin` instance from @@ -37,7 +36,8 @@ serializes property values that are also default values. - Pinned zarr dependency to be >=2.18, <3 until test `tests.plugins.xcube.processors.test_mldataset.MultiLevelDatasetProcessorTest` - is adjusted or fsspec's memory filesystem is updated. + is adjusted or `fsspec`'s memory filesystem is updated. + - Now making use of the `expected` property of `RuleTest`. ## Version 0.3.0 (from 2025-01-20) diff --git a/docs/todo.md b/docs/todo.md index a47ee63..1ddf56b 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -16,11 +16,10 @@ - project logo - add some more tests so we reach 99% coverage - apply rule op args/kwargs validation schema -- support `RuleTest.expected`, it is currently unused - provide core rule that checks for configurable list of std attributes - measure time it takes to open a dataset and pass time into rule context so we can write a configurable rule that checks the opening time -- allow outputting suggestions that me be included in rule results +- allow outputting suggestions, if any, that emitted by some rules ## Nice to have diff --git a/examples/rule_testing.py b/examples/rule_testing.py index 98373f1..4035737 100644 --- a/examples/rule_testing.py +++ b/examples/rule_testing.py @@ -36,7 +36,7 @@ def dataset(self, ctx: RuleContext, node: DatasetNode): "good-title", GoodTitle, valid=[RuleTest(dataset=valid_dataset)], - invalid=[RuleTest(dataset=invalid_dataset)], + invalid=[RuleTest(dataset=invalid_dataset, expected=1)], ) # or define a test class derived from unitest.TestCase diff --git a/tests/plugins/core/rules/test_coords_for_dims.py b/tests/plugins/core/rules/test_coords_for_dims.py index 4a9d26a..18292fa 100644 --- a/tests/plugins/core/rules/test_coords_for_dims.py +++ b/tests/plugins/core/rules/test_coords_for_dims.py @@ -20,6 +20,6 @@ RuleTest(dataset=valid_dataset_2), ], invalid=[ - RuleTest(dataset=invalid_dataset_2), + RuleTest(dataset=invalid_dataset_2, expected=1), ], ) diff --git a/tests/plugins/core/rules/test_dataset_title_attr.py b/tests/plugins/core/rules/test_dataset_title_attr.py index f128eda..8fb9c55 100644 --- a/tests/plugins/core/rules/test_dataset_title_attr.py +++ b/tests/plugins/core/rules/test_dataset_title_attr.py @@ -17,7 +17,7 @@ RuleTest(dataset=valid_dataset_2), ], invalid=[ - RuleTest(dataset=invalid_dataset_1), - RuleTest(dataset=invalid_dataset_2), + RuleTest(dataset=invalid_dataset_1, expected=1), + RuleTest(dataset=invalid_dataset_2, expected=1), ], ) diff --git a/tests/plugins/core/rules/test_flags.py b/tests/plugins/core/rules/test_flags.py index 521832d..bd216de 100644 --- a/tests/plugins/core/rules/test_flags.py +++ b/tests/plugins/core/rules/test_flags.py @@ -83,13 +83,13 @@ RuleTest(dataset=valid_dataset_3), ], invalid=[ - RuleTest(dataset=invalid_dataset_0), - RuleTest(dataset=invalid_dataset_1), - RuleTest(dataset=invalid_dataset_2), - RuleTest(dataset=invalid_dataset_3), - RuleTest(dataset=invalid_dataset_4), - RuleTest(dataset=invalid_dataset_5), - RuleTest(dataset=invalid_dataset_6), - RuleTest(dataset=invalid_dataset_7), + RuleTest(dataset=invalid_dataset_0, expected=2), + RuleTest(dataset=invalid_dataset_1, expected=1), + RuleTest(dataset=invalid_dataset_2, expected=1), + RuleTest(dataset=invalid_dataset_3, expected=1), + RuleTest(dataset=invalid_dataset_4, expected=1), + RuleTest(dataset=invalid_dataset_5, expected=1), + RuleTest(dataset=invalid_dataset_6, expected=1), + RuleTest(dataset=invalid_dataset_7, expected=1), ], ) diff --git a/tests/plugins/core/rules/test_grid_mappings.py b/tests/plugins/core/rules/test_grid_mappings.py index 70b5277..2ba9be6 100644 --- a/tests/plugins/core/rules/test_grid_mappings.py +++ b/tests/plugins/core/rules/test_grid_mappings.py @@ -60,9 +60,9 @@ def make_dataset(): RuleTest(dataset=valid_dataset_2), ], invalid=[ - RuleTest(dataset=invalid_dataset_1), - RuleTest(dataset=invalid_dataset_2), - RuleTest(dataset=invalid_dataset_3), - RuleTest(dataset=invalid_dataset_4), + RuleTest(dataset=invalid_dataset_1, expected=1), + RuleTest(dataset=invalid_dataset_2, expected=1), + RuleTest(dataset=invalid_dataset_3, expected=1), + RuleTest(dataset=invalid_dataset_4, expected=1), ], ) diff --git a/tests/plugins/core/rules/test_lat_lon_coordinate.py b/tests/plugins/core/rules/test_lat_lon_coordinate.py index 89bca33..a5a184e 100644 --- a/tests/plugins/core/rules/test_lat_lon_coordinate.py +++ b/tests/plugins/core/rules/test_lat_lon_coordinate.py @@ -84,9 +84,9 @@ RuleTest(dataset=invalid_lon_dataset_2), ], invalid=[ - RuleTest(dataset=invalid_lat_dataset_0), - RuleTest(dataset=invalid_lat_dataset_1), - RuleTest(dataset=invalid_lat_dataset_2), + RuleTest(dataset=invalid_lat_dataset_0, expected=1), + RuleTest(dataset=invalid_lat_dataset_1, expected=1), + RuleTest(dataset=invalid_lat_dataset_2, expected=2), ], ) @@ -104,8 +104,8 @@ RuleTest(dataset=invalid_lat_dataset_2), ], invalid=[ - RuleTest(dataset=invalid_lon_dataset_0), - RuleTest(dataset=invalid_lon_dataset_1), - RuleTest(dataset=invalid_lon_dataset_2), + RuleTest(dataset=invalid_lon_dataset_0, expected=1), + RuleTest(dataset=invalid_lon_dataset_1, expected=1), + RuleTest(dataset=invalid_lon_dataset_2, expected=2), ], ) diff --git a/tests/plugins/core/rules/test_no_empty_attrs.py b/tests/plugins/core/rules/test_no_empty_attrs.py index 26f98fa..62ecd7e 100644 --- a/tests/plugins/core/rules/test_no_empty_attrs.py +++ b/tests/plugins/core/rules/test_no_empty_attrs.py @@ -25,8 +25,8 @@ RuleTest(dataset=valid_dataset_2), ], invalid=[ - RuleTest(dataset=invalid_dataset_1), - RuleTest(dataset=invalid_dataset_2), - RuleTest(dataset=invalid_dataset_3), + RuleTest(dataset=invalid_dataset_1, expected=1), + RuleTest(dataset=invalid_dataset_2, expected=1), + RuleTest(dataset=invalid_dataset_3, expected=1), ], ) diff --git a/tests/plugins/core/rules/test_no_empty_chunks.py b/tests/plugins/core/rules/test_no_empty_chunks.py index 89b24d9..b54378e 100644 --- a/tests/plugins/core/rules/test_no_empty_chunks.py +++ b/tests/plugins/core/rules/test_no_empty_chunks.py @@ -36,6 +36,6 @@ RuleTest(dataset=valid_dataset_3), ], invalid=[ - RuleTest(dataset=invalid_dataset_0), + RuleTest(dataset=invalid_dataset_0, expected=1), ], ) diff --git a/tests/plugins/core/rules/test_time_coordinate.py b/tests/plugins/core/rules/test_time_coordinate.py index 3c56de4..44e278b 100644 --- a/tests/plugins/core/rules/test_time_coordinate.py +++ b/tests/plugins/core/rules/test_time_coordinate.py @@ -103,17 +103,17 @@ RuleTest(dataset=valid_dataset_5), ], invalid=[ - RuleTest(dataset=invalid_dataset_0), - RuleTest(dataset=invalid_dataset_1), - RuleTest(dataset=invalid_dataset_2), - RuleTest(dataset=invalid_dataset_3), - RuleTest(dataset=invalid_dataset_4), - RuleTest(dataset=invalid_dataset_5), - RuleTest(dataset=invalid_dataset_6), - RuleTest(dataset=invalid_dataset_7), - RuleTest(dataset=invalid_dataset_8), - RuleTest(dataset=invalid_dataset_9), - RuleTest(dataset=invalid_dataset_10), - RuleTest(dataset=invalid_dataset_11), + RuleTest(dataset=invalid_dataset_0, expected=1), + RuleTest(dataset=invalid_dataset_1, expected=1), + RuleTest(dataset=invalid_dataset_2, expected=1), + RuleTest(dataset=invalid_dataset_3, expected=1), + RuleTest(dataset=invalid_dataset_4, expected=1), + RuleTest(dataset=invalid_dataset_5, expected=1), + RuleTest(dataset=invalid_dataset_6, expected=1), + RuleTest(dataset=invalid_dataset_7, expected=1), + RuleTest(dataset=invalid_dataset_8, expected=1), + RuleTest(dataset=invalid_dataset_9, expected=1), + RuleTest(dataset=invalid_dataset_10, expected=1), + RuleTest(dataset=invalid_dataset_11, expected=1), ], ) diff --git a/tests/plugins/core/rules/test_var_units_attr.py b/tests/plugins/core/rules/test_var_units_attr.py index 99bd715..a33620a 100644 --- a/tests/plugins/core/rules/test_var_units_attr.py +++ b/tests/plugins/core/rules/test_var_units_attr.py @@ -27,8 +27,8 @@ RuleTest(dataset=valid_dataset_2), ], invalid=[ - RuleTest(dataset=invalid_dataset_1), - RuleTest(dataset=invalid_dataset_2), - RuleTest(dataset=invalid_dataset_3), + RuleTest(dataset=invalid_dataset_1, expected=1), + RuleTest(dataset=invalid_dataset_2, expected=1), + RuleTest(dataset=invalid_dataset_3, expected=1), ], ) diff --git a/tests/plugins/xcube/rules/test_any_spatial_data_var.py b/tests/plugins/xcube/rules/test_any_spatial_data_var.py index a9f27e6..f5e847e 100644 --- a/tests/plugins/xcube/rules/test_any_spatial_data_var.py +++ b/tests/plugins/xcube/rules/test_any_spatial_data_var.py @@ -14,6 +14,6 @@ RuleTest(dataset=valid_dataset), ], invalid=[ - RuleTest(dataset=invalid_dataset), + RuleTest(dataset=invalid_dataset, expected=1), ], ) diff --git a/tests/plugins/xcube/rules/test_cube_dims_order.py b/tests/plugins/xcube/rules/test_cube_dims_order.py index aacbcb4..b0d1da8 100644 --- a/tests/plugins/xcube/rules/test_cube_dims_order.py +++ b/tests/plugins/xcube/rules/test_cube_dims_order.py @@ -30,30 +30,30 @@ def make_dataset(dims: tuple[str, str, str]): ) -valid_dataset_1 = make_dataset(("time", "y", "x")) -valid_dataset_2 = make_dataset(("time", "lat", "lon")) -valid_dataset_3 = make_dataset(("level", "y", "x")) +valid_dataset_0 = make_dataset(("time", "y", "x")) +valid_dataset_1 = make_dataset(("time", "lat", "lon")) +valid_dataset_2 = make_dataset(("level", "y", "x")) -invalid_dataset_1 = make_dataset(("time", "x", "y")) -invalid_dataset_2 = make_dataset(("x", "y", "time")) -invalid_dataset_3 = make_dataset(("time", "lon", "lat")) -invalid_dataset_4 = make_dataset(("lon", "lat", "level")) -invalid_dataset_5 = make_dataset(("x", "y", "level")) +invalid_dataset_0 = make_dataset(("time", "x", "y")) +invalid_dataset_1 = make_dataset(("x", "y", "time")) +invalid_dataset_2 = make_dataset(("time", "lon", "lat")) +invalid_dataset_3 = make_dataset(("lon", "lat", "level")) +invalid_dataset_4 = make_dataset(("x", "y", "level")) CubeDimsOrderTest = RuleTester.define_test( "cube-dims-order", CubeDimsOrder, valid=[ + RuleTest(dataset=valid_dataset_0), RuleTest(dataset=valid_dataset_1), RuleTest(dataset=valid_dataset_2), - RuleTest(dataset=valid_dataset_3), ], invalid=[ - RuleTest(dataset=invalid_dataset_1), - RuleTest(dataset=invalid_dataset_2), - RuleTest(dataset=invalid_dataset_3), - RuleTest(dataset=invalid_dataset_4), - RuleTest(dataset=invalid_dataset_5), + RuleTest(dataset=invalid_dataset_0, expected=2), + RuleTest(dataset=invalid_dataset_1, expected=2), + RuleTest(dataset=invalid_dataset_2, expected=2), + RuleTest(dataset=invalid_dataset_3, expected=2), + RuleTest(dataset=invalid_dataset_4, expected=2), ], ) diff --git a/tests/plugins/xcube/rules/test_data_var_colors.py b/tests/plugins/xcube/rules/test_data_var_colors.py index 051fbfb..4c86cf9 100644 --- a/tests/plugins/xcube/rules/test_data_var_colors.py +++ b/tests/plugins/xcube/rules/test_data_var_colors.py @@ -75,8 +75,8 @@ def make_dataset(): RuleTest(dataset=valid_dataset_1), ], invalid=[ - RuleTest(dataset=invalid_dataset_1), - RuleTest(dataset=invalid_dataset_2), - RuleTest(dataset=invalid_dataset_3), + RuleTest(dataset=invalid_dataset_1, expected=1), + RuleTest(dataset=invalid_dataset_2, expected=1), + RuleTest(dataset=invalid_dataset_3, expected=1), ], ) diff --git a/tests/plugins/xcube/rules/test_grid_mapping_naming.py b/tests/plugins/xcube/rules/test_grid_mapping_naming.py index f4b0800..c6ca4bd 100644 --- a/tests/plugins/xcube/rules/test_grid_mapping_naming.py +++ b/tests/plugins/xcube/rules/test_grid_mapping_naming.py @@ -52,6 +52,6 @@ def make_dataset(): RuleTest(dataset=valid_dataset_3), ], invalid=[ - RuleTest(dataset=invalid_dataset_1), + RuleTest(dataset=invalid_dataset_1, expected=1), ], ) diff --git a/tests/plugins/xcube/rules/test_increasing_time.py b/tests/plugins/xcube/rules/test_increasing_time.py index 6444007..820f23b 100644 --- a/tests/plugins/xcube/rules/test_increasing_time.py +++ b/tests/plugins/xcube/rules/test_increasing_time.py @@ -58,7 +58,7 @@ def make_dataset(): RuleTest(dataset=valid_dataset_1), ], invalid=[ - RuleTest(dataset=invalid_dataset_1), - RuleTest(dataset=invalid_dataset_2), + RuleTest(dataset=invalid_dataset_1, expected=1), + RuleTest(dataset=invalid_dataset_2, expected=1), ], ) diff --git a/tests/plugins/xcube/rules/test_lat_lon_naming.py b/tests/plugins/xcube/rules/test_lat_lon_naming.py index 2a986cd..73f958c 100644 --- a/tests/plugins/xcube/rules/test_lat_lon_naming.py +++ b/tests/plugins/xcube/rules/test_lat_lon_naming.py @@ -53,11 +53,11 @@ def make_dataset(lat_dim: str, lon_dim: str): RuleTest(dataset=valid_dataset_1), ], invalid=[ - RuleTest(dataset=invalid_dataset_1), - RuleTest(dataset=invalid_dataset_2), - RuleTest(dataset=invalid_dataset_3), - RuleTest(dataset=invalid_dataset_4), - RuleTest(dataset=invalid_dataset_5), - RuleTest(dataset=invalid_dataset_6), + RuleTest(dataset=invalid_dataset_1, expected=1), + RuleTest(dataset=invalid_dataset_2, expected=1), + RuleTest(dataset=invalid_dataset_3, expected=1), + RuleTest(dataset=invalid_dataset_4, expected=1), + RuleTest(dataset=invalid_dataset_5, expected=1), + RuleTest(dataset=invalid_dataset_6, expected=1), ], ) diff --git a/tests/plugins/xcube/rules/test_ml_dataset_meta.py b/tests/plugins/xcube/rules/test_ml_dataset_meta.py index 5ad27a0..f26d962 100644 --- a/tests/plugins/xcube/rules/test_ml_dataset_meta.py +++ b/tests/plugins/xcube/rules/test_ml_dataset_meta.py @@ -79,9 +79,9 @@ def _replace_meta(dataset: xr.Dataset, meta: LevelsMeta) -> xr.Dataset: RuleTest(dataset=valid_dataset_3), ], invalid=[ - RuleTest(dataset=invalid_dataset_0), - RuleTest(dataset=invalid_dataset_1), - RuleTest(dataset=invalid_dataset_2), - RuleTest(dataset=invalid_dataset_3), + RuleTest(dataset=invalid_dataset_0, expected=1), + RuleTest(dataset=invalid_dataset_1, expected=4), + RuleTest(dataset=invalid_dataset_2, expected=3), + RuleTest(dataset=invalid_dataset_3, expected=2), ], ) diff --git a/tests/plugins/xcube/rules/test_ml_dataset_time.py b/tests/plugins/xcube/rules/test_ml_dataset_time.py index 8d3d6f4..66d6e6d 100644 --- a/tests/plugins/xcube/rules/test_ml_dataset_time.py +++ b/tests/plugins/xcube/rules/test_ml_dataset_time.py @@ -34,6 +34,6 @@ RuleTest(dataset=valid_dataset_3), ], invalid=[ - RuleTest(dataset=invalid_dataset_0), + RuleTest(dataset=invalid_dataset_0, expected=1), ], ) diff --git a/tests/plugins/xcube/rules/test_ml_dataset_xy.py b/tests/plugins/xcube/rules/test_ml_dataset_xy.py index 6faa58a..c56ea72 100644 --- a/tests/plugins/xcube/rules/test_ml_dataset_xy.py +++ b/tests/plugins/xcube/rules/test_ml_dataset_xy.py @@ -52,6 +52,6 @@ RuleTest(dataset=valid_dataset_5), ], invalid=[ - RuleTest(dataset=invalid_dataset_0), + RuleTest(dataset=invalid_dataset_0, expected=2), ], ) diff --git a/tests/plugins/xcube/rules/test_no_chunked_coords.py b/tests/plugins/xcube/rules/test_no_chunked_coords.py index 51bd7ba..052fcbc 100644 --- a/tests/plugins/xcube/rules/test_no_chunked_coords.py +++ b/tests/plugins/xcube/rules/test_no_chunked_coords.py @@ -23,6 +23,6 @@ RuleTest(dataset=valid_dataset_2), ], invalid=[ - RuleTest(dataset=invalid_dataset_0), + RuleTest(dataset=invalid_dataset_0, expected=1), ], ) diff --git a/tests/plugins/xcube/rules/test_single_grid_mapping.py b/tests/plugins/xcube/rules/test_single_grid_mapping.py index 551776f..a77984e 100644 --- a/tests/plugins/xcube/rules/test_single_grid_mapping.py +++ b/tests/plugins/xcube/rules/test_single_grid_mapping.py @@ -66,7 +66,7 @@ def make_dataset(): RuleTest(dataset=valid_dataset_5), ], invalid=[ - RuleTest(dataset=invalid_dataset_1), - RuleTest(dataset=invalid_dataset_2), + RuleTest(dataset=invalid_dataset_1, expected=1), + RuleTest(dataset=invalid_dataset_2, expected=1), ], ) diff --git a/tests/plugins/xcube/rules/test_time_naming.py b/tests/plugins/xcube/rules/test_time_naming.py index a8ee91b..bea89ea 100644 --- a/tests/plugins/xcube/rules/test_time_naming.py +++ b/tests/plugins/xcube/rules/test_time_naming.py @@ -65,11 +65,11 @@ def make_dataset(time_var: str, time_dim: str | None = None): RuleTest(dataset=valid_dataset_1), ], invalid=[ - RuleTest(dataset=invalid_dataset_0), - RuleTest(dataset=invalid_dataset_1), - RuleTest(dataset=invalid_dataset_2), - RuleTest(dataset=invalid_dataset_3), - RuleTest(dataset=invalid_dataset_4), - RuleTest(dataset=invalid_dataset_5), + RuleTest(dataset=invalid_dataset_0, expected=1), + RuleTest(dataset=invalid_dataset_1, expected=1), + RuleTest(dataset=invalid_dataset_2, expected=1), + RuleTest(dataset=invalid_dataset_3, expected=1), + RuleTest(dataset=invalid_dataset_4, expected=1), + RuleTest(dataset=invalid_dataset_5, expected=2), ], ) diff --git a/tests/test_testing.py b/tests/test_testing.py index 6d1d7ab..fc1ac7f 100644 --- a/tests/test_testing.py +++ b/tests/test_testing.py @@ -33,8 +33,10 @@ def test_ok(self): RuleTest(dataset=VALID_DATASET_2), ], invalid=[ - RuleTest(dataset=INVALID_DATASET_1), - RuleTest(dataset=INVALID_DATASET_2), + RuleTest(dataset=INVALID_DATASET_1, expected=1), + RuleTest( + dataset=INVALID_DATASET_2, expected=["Datasets must have a title"] + ), ], ) @@ -44,7 +46,8 @@ def test_raises_valid(self): AssertionError, match=( "Rule 'force-title': test_valid_2:" - " expected no problem, but got one error" + " expected no problems, but got one error:\nActual message:\n" + " 0: Datasets must have a title" ), ): tester.run( @@ -57,20 +60,90 @@ def test_raises_valid(self): ], ) - def test_raises_invalid(self): + def test_raises_invalid_with_count(self): tester = RuleTester(rules={"testing/force-title": "error"}) with pytest.raises( AssertionError, match=( - "Rule 'force-title': test_invalid_1:" - " expected one or more problems, but got no problems" + "Rule 'force-title': test_invalid_0:" + " expected one problem, but got no problems." ), ): tester.run( "force-title", ForceTitle, invalid=[ - RuleTest(dataset=INVALID_DATASET_1), - RuleTest(dataset=VALID_DATASET_1), + RuleTest(dataset=VALID_DATASET_1, expected=1), + ], + ) + + def test_raises_valid_with_count(self): + tester = RuleTester(rules={"testing/force-title": "error"}) + with pytest.raises( + AssertionError, + match=( + "Rule 'force-title':" + " test_invalid_raises_valid_with_count:" + " expected one problem, but got no problems." + ), + ): + tester.run( + "force-title", + ForceTitle, + invalid=[ + RuleTest( + dataset=VALID_DATASET_1, + expected=1, + name="raises_valid_with_count", + ), + ], + ) + + def test_raises_invalid_with_matching_message(self): + tester = RuleTester(rules={"testing/force-title": "error"}) + with pytest.raises( + AssertionError, + match=( + "Rule 'force-title':" + " test_invalid_raises_invalid_with_matching_message:" + " expected one problem, but got no problems:\n" + "Expected message:\n" + " 0: Datasets must have a title" + ), + ): + tester.run( + "force-title", + ForceTitle, + invalid=[ + RuleTest( + dataset=VALID_DATASET_1, + expected=["Datasets must have a title"], + name="raises_invalid_with_matching_message", + ), + ], + ) + + def test_raises_invalid_with_mismatching_message(self): + tester = RuleTester(rules={"testing/force-title": "error"}) + with pytest.raises( + AssertionError, + match=( + "Rule 'force-title':" + " test_invalid_raises_invalid_with_mismatching_message:" + " got one error as expected, but encountered message mismatch:\n" + "Message 0:\n" + " Expected: Batasets bust bave a bitle\n" + " Actual: Datasets must have a title" + ), + ): + tester.run( + "force-title", + ForceTitle, + invalid=[ + RuleTest( + dataset=INVALID_DATASET_1, + expected=["Batasets bust bave a bitle"], + name="raises_invalid_with_mismatching_message", + ), ], ) diff --git a/xrlint/testing.py b/xrlint/testing.py index 3a75bf6..85e3079 100644 --- a/xrlint/testing.py +++ b/xrlint/testing.py @@ -7,9 +7,9 @@ from xrlint.constants import SEVERITY_ERROR from xrlint.linter import Linter from xrlint.plugin import new_plugin -from xrlint.result import Message, Result +from xrlint.result import Message from xrlint.rule import Rule, RuleMeta, RuleOp -from xrlint.util.formatting import format_problems +from xrlint.util.formatting import format_problems, format_item, format_count from xrlint.util.naming import to_snake_case _PLUGIN_NAME: Final = "testing" @@ -31,10 +31,10 @@ class RuleTest: kwargs: dict[str, Any] | None = None """Optional keyword arguments passed to the rule operation's constructor.""" - expected: list[Message] | int | None = 0 + expected: list[Message | str] | int | None = None """Expected messages. Either a list of expected message objects or - the number of expected message. + the number of expected messages. Must not be provided for valid checks and must be provided for invalid checks. """ @@ -182,35 +182,71 @@ def _test_rule( }, ) - assert_ok = _assert_valid if test_mode == "valid" else _assert_invalid - if assert_ok(result): - return None - else: - return _format_error_message(rule_name, test_id, test_mode, result) - + result_message_count = len(result.messages) -def _assert_valid(r: Result): - return r.error_count == 0 and r.warning_count == 0 + expected = test.expected + expected_message_count = 0 + expected_messages = None + if test_mode == "valid": + assert expected is None, ( + f"{test_id}: you cannot provide the keyword argument" + f" `expected` for a RuleTest in 'valid' mode." + ) + else: + if isinstance(expected, int): + expected_message_count = max(1, expected) + expected_messages = None + elif isinstance(expected, list): + expected_message_count = len(expected) + expected_messages = expected + assert expected_message_count > 0, ( + f"{test_id}: you must provide a valid keyword argument" + f" `expected` for a RuleTest in 'invalid' mode. Pass a list" + f" of expected message or str objects or an int specifying" + f" the expected number of messages." + ) -def _assert_invalid(r: Result): - return r.error_count != 0 or r.warning_count != 0 + lines: list[str] = [] + if result_message_count == expected_message_count: + if expected_messages is None: + return None + all_ok = True + texts = map(_get_message_text, expected_messages) + result_texts = map(_get_message_text, result.messages) + for i, (expected_text, result_text) in enumerate(zip(texts, result_texts)): + if expected_text != result_text: + all_ok = False + lines.append(f"Message {i}:") + lines.append(f" Expected: {expected_text}") + lines.append(f" Actual: {result_text}") + if all_ok: + return None + else: + if expected_messages: + lines.append( + f"Expected {format_item(expected_message_count, 'message')}:" + ) + for i, text in enumerate(map(_get_message_text, expected_messages)): + lines.append(f" {i}: {text}") + if result.messages: + lines.append(f"Actual {format_item(result_message_count, 'message')}:") + for i, text in enumerate(map(_get_message_text, result.messages)): + lines.append(f" {i}: {text}") + + result_text = format_problems(result.error_count, result.warning_count) + if expected_message_count == result_message_count: + problem_text = ( + f"got {result_text} as expected, but encountered message mismatch" + ) + else: + expected_text = format_count(expected_message_count, "problem") + problem_text = f"expected {expected_text}, but got {result_text}" -def _format_error_message( - rule_name: str, - test_id: str, - test_mode: Literal["valid", "invalid"], - result: Result, -) -> str: - actual = format_problems(result.error_count, result.warning_count) - expected = f"{'no problem' if test_mode == 'valid' else 'one or more problems'}" - messages = "\n".join(f"- {m.message}" for m in result.messages) - messages = (":\n" + messages) if messages else "." - return ( - f"Rule {rule_name!r}: {test_id}:" - f" expected {expected}, but got {actual}{messages}" - ) + messages_text = "\n".join(lines) + messages_text = (":\n" + messages_text) if messages_text else "." + return f"Rule {rule_name!r}: {test_id}: {problem_text}{messages_text}" def _format_test_id( @@ -220,3 +256,7 @@ def _format_test_id( return f"test_{test_mode}_{to_snake_case(test.name)}" else: return f"test_{test_mode}_{test_index}" + + +def _get_message_text(m: Message | str) -> str: + return m if isinstance(m, str) else m.message