Skip to content

Commit 0c83f69

Browse files
sol1105sol1105
committed
Adjusting test suite to allow multiple results per check.
1 parent b08c617 commit 0c83f69

File tree

1 file changed

+104
-68
lines changed

1 file changed

+104
-68
lines changed

‎tests/test_cc6.py‎

Lines changed: 104 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -49,45 +49,51 @@
4949
# Expected check failures
5050
expected_failures = dict()
5151
expected_failures["TAS_REMO"] = {
52-
# "check_version_realization": [
53-
# "DRS filename building block 'version_realization' does not comply",
54-
# "DRS path building block 'version_realization' does not comply",
55-
# "Global attribute 'version_realization' does not comply",
56-
# ],
57-
"check_compression": [
58-
"It is recommended that data should be compressed with a 'deflate level' of '1' and enabled 'shuffle' option."
59-
" The 'shuffle' option is disabled.",
60-
],
61-
# "check_version_realization_info": [
62-
# "The global attribute 'version_realization_info' is missing. It is however recommended"
63-
# ],
64-
"check_horizontal_axes_bounds": [
65-
"It is recommended for the variables 'rlat' and 'rlon' or 'x' and 'y' to have bounds defined."
66-
],
67-
"check_grid_mapping": [
68-
"The grid_mapping variable 'rotated_latitude_longitude' needs to include information regarding the shape and size of the Earth"
69-
],
52+
"check_compression": {
53+
2: [
54+
"It is recommended that data should be compressed with a 'deflate level' of '1' and enabled 'shuffle' option."
55+
" The 'shuffle' option is disabled.",
56+
],
57+
},
58+
"check_horizontal_axes_bounds": {
59+
2: [
60+
"It is recommended for the variables 'rlat' and 'rlon' or 'x' and 'y' to have bounds defined."
61+
],
62+
},
63+
"check_grid_mapping": {
64+
3: [
65+
"The grid_mapping variable 'rotated_latitude_longitude' needs to include information regarding the shape and size of the Earth"
66+
],
67+
},
68+
"check_variable_definition": {
69+
1: ["'tas:comment' needs to include the specified comment from the CMOR table"],
70+
},
7071
}
7172
expected_failures["FXOROG_REMO"] = {
72-
# "check_version_realization": [
73-
# "DRS filename building block 'version_realization' does not comply",
74-
# "DRS path building block 'version_realization' does not comply",
75-
# "Global attribute 'version_realization' does not comply",
76-
# ],
77-
"check_compression": [
78-
"It is recommended that data should be compressed with a 'deflate level' of '1' and enabled 'shuffle' option."
79-
" The 'shuffle' option is disabled.",
80-
],
81-
# "check_version_realization_info": [
82-
# "The global attribute 'version_realization_info' is missing. It is however recommended"
83-
# ],
84-
"check_horizontal_axes_bounds": [
85-
"It is recommended for the variables 'rlat' and 'rlon' or 'x' and 'y' to have bounds defined."
86-
],
87-
"check_grid_mapping": [
88-
"The grid_mapping variable 'rotated_latitude_longitude' needs to include information regarding the shape and size of the Earth"
89-
],
73+
"check_compression": {
74+
2: [
75+
"It is recommended that data should be compressed with a 'deflate level' of '1' and enabled 'shuffle' option."
76+
" The 'shuffle' option is disabled.",
77+
],
78+
},
79+
"check_horizontal_axes_bounds": {
80+
2: [
81+
"It is recommended for the variables 'rlat' and 'rlon' or 'x' and 'y' to have bounds defined."
82+
],
83+
},
84+
"check_grid_mapping": {
85+
3: [
86+
"The grid_mapping variable 'rotated_latitude_longitude' needs to include information regarding the shape and size of the Earth"
87+
],
88+
},
89+
"check_variable_definition": {
90+
1: [
91+
"'orog:comment' needs to include the specified comment from the CMOR table"
92+
],
93+
},
9094
}
95+
# Checks that report multiple results (at various severity levels)
96+
multi_result_checks = ["check_variable_definition"]
9197

9298

9399
def test_cc6_basic(load_test_data):
@@ -163,42 +169,72 @@ def _run_check(self, cs, dataset, checks):
163169
assert (
164170
len(checks) == 1
165171
), f"Expected 1 check, found {len(checks)}: {', '.join(checks)}"
166-
assert len(res["cc6"][0]) == 1, f"Expected 1 check, found {len(res['cc6'][0])}"
167-
168-
# Check result object
169-
check_result = res["cc6"][0][0]
170-
171-
# Check the score results
172-
if dataset in expected_failures and checks[0] in expected_failures[dataset]:
172+
if checks[0] in multi_result_checks:
173173
assert (
174-
len(set(check_result.value)) == 2
175-
), f"Inconsistent check values: {check_result.value}"
174+
len(res["cc6"][0]) == 2
175+
), f"Expected 2 checks, found {len(res['cc6'][0])}"
176176
else:
177177
assert (
178-
len(set(check_result.value)) == 1
179-
), f"Inconsistent check values: {check_result.value}"
180-
181-
# Are there any expected check failures?
182-
if dataset in expected_failures and len(expected_failures[dataset].keys()) > 0:
183-
# If this specific check failed, was that expected?
184-
if checks[0] in expected_failures[dataset]:
185-
substrs = expected_failures[dataset][checks[0]]
186-
if substrs:
187-
for substr in substrs:
188-
assert any(substr in msg for msg in check_result.msgs), (
189-
"Expected check "
190-
f"'{checks[0]}' to fail with all of \"{', '.join(substrs)}\" but got \"{', '.join(check_result.msgs)}\"."
191-
)
192-
else:
193-
assert (
194-
len(check_result.msgs) > 0
195-
), f"Expected check '{checks[0]}' to fail, but it passed."
196-
# Else, assert that no messages were returned (== the check did not fail)
178+
len(res["cc6"][0]) == 1
179+
), f"Expected 1 check, found {len(res['cc6'][0])}"
180+
181+
# Check result objects
182+
severity_encountered = []
183+
for check_result in res["cc6"][0]:
184+
185+
# Get check severity
186+
severity = check_result.weight
187+
severity_encountered.append(severity)
188+
189+
# Check the score results
190+
if (
191+
dataset in expected_failures
192+
and checks[0] in expected_failures[dataset]
193+
and severity in expected_failures[dataset][checks[0]]
194+
):
195+
assert (
196+
len(set(check_result.value)) == 2
197+
), f"Inconsistent check values: {check_result.value}"
197198
else:
198-
assert check_result.msgs == [], (
199-
"Expected no messages for check "
200-
f"'{checks[0]}' but got: {', '.join(check_result.msgs)}"
201-
)
199+
assert (
200+
len(set(check_result.value)) == 1
201+
), f"Inconsistent check values: {check_result.value}"
202+
203+
# Are there any expected check failures?
204+
if (
205+
dataset in expected_failures
206+
and len(expected_failures[dataset].keys()) > 0
207+
):
208+
# If this specific check failed, was that expected?
209+
if (
210+
checks[0] in expected_failures[dataset]
211+
and severity in expected_failures[dataset][checks[0]]
212+
):
213+
substrs = expected_failures[dataset][checks[0]][severity]
214+
if substrs:
215+
for substr in substrs:
216+
assert any(substr in msg for msg in check_result.msgs), (
217+
"Expected check "
218+
f"'{checks[0]}' to fail with all of \"{', '.join(substrs)}\" but got \"{', '.join(check_result.msgs)}\"."
219+
)
220+
else:
221+
assert (
222+
len(check_result.msgs) > 0
223+
), f"Expected check '{checks[0]}' to fail, but it passed."
224+
# Else, assert that no messages were returned (== the check did not fail)
225+
else:
226+
assert check_result.msgs == [], (
227+
"Expected no messages for check "
228+
f"'{checks[0]}' but got: {', '.join(check_result.msgs)}"
229+
)
230+
# Ensure all expected check severities were encountered
231+
if dataset in expected_failures and checks[0] in expected_failures[dataset]:
232+
assert all(
233+
[
234+
si in severity_encountered
235+
for si in expected_failures[dataset][checks[0]].keys()
236+
]
237+
)
202238

203239
@pytest.mark.parametrize(
204240
"dataset",

0 commit comments

Comments
 (0)