|
9 | 9 | from collections import Counter |
10 | 10 | from pathlib import Path |
11 | 11 |
|
| 12 | +# Total number of environments expected |
| 13 | +TOTAL_ENVS = 10 |
12 | 14 |
|
13 | 15 | # \u200c is zero-width space. It is added so that len of the string corresponds to real width. |
14 | 16 | # ❌, ✅, 🔄 each take space of 2 characters. |
|
17 | 19 | PASS = "✅\u200cpass" |
18 | 20 | SKIP = "🙈\u200cskip" |
19 | 21 |
|
| 22 | +# FAIL is replaced with BUG when test fails in all environments (and when we have >=TOTAL_ENVS-1 environments) |
| 23 | +# This indicate that it's very likely that PR did broke this test rather than environment being flaky. |
| 24 | +BUG = "🪲\u200cBUG" |
| 25 | + |
20 | 26 | # This happens when Eventually is used - there is output for the test but no result. |
21 | 27 | MISSING = "🤯\u200cMISS" |
22 | 28 | PANIC = "💥\u200cPANIC" |
23 | 29 |
|
24 | | -INTERESTING_ACTIONS = (FAIL, FLAKY, PANIC, MISSING) |
| 30 | +INTERESTING_ACTIONS = (FAIL, BUG, FLAKY, PANIC, MISSING) |
25 | 31 | ACTIONS_WITH_ICON = INTERESTING_ACTIONS + (PASS, SKIP) |
26 | 32 |
|
27 | 33 | ACTION_MESSAGES = { |
@@ -151,6 +157,34 @@ def print_report(filenames, filter, filter_env, show_output, markdown=False): |
151 | 157 | if e not in test_results: |
152 | 158 | test_results.setdefault(e, Counter())[MISSING] += 1 |
153 | 159 |
|
| 160 | + # Check if we can convert FAIL to BUG |
| 161 | + def is_bug(test_results): |
| 162 | + if len(test_results) < TOTAL_ENVS - 1: |
| 163 | + # incomplete results |
| 164 | + return False |
| 165 | + count = 0 |
| 166 | + for e, env_results in test_results.items(): |
| 167 | + if PASS in env_results: |
| 168 | + return False |
| 169 | + if FLAKY in env_results: |
| 170 | + return False |
| 171 | + if SKIP in env_results: |
| 172 | + count -= 1 |
| 173 | + else: |
| 174 | + count += 1 |
| 175 | + return count >= 0 |
| 176 | + |
| 177 | + for testname in all_testnames: |
| 178 | + test_results = per_test_per_env_stats.get(testname, {}) |
| 179 | + if not is_bug(test_results): |
| 180 | + continue |
| 181 | + for e, env_results in sorted(test_results.items()): |
| 182 | + if env_results[FAIL] > 0: |
| 183 | + env_results[FAIL] -= 1 |
| 184 | + if not env_results[FAIL]: |
| 185 | + env_results.pop(FAIL) |
| 186 | + env_results[BUG] += 1 |
| 187 | + |
154 | 188 | per_env_stats = {} # env -> action -> count |
155 | 189 | for testname, items in per_test_per_env_stats.items(): |
156 | 190 | for env, stats in items.items(): |
|
0 commit comments