Skip to content

Commit 5f364d0

Browse files
authored
Add bug detection to gh_parse.py (#3167)
## Changes Convert FAIL to BUG if test failed in all environments and there is enough evidence (more failures than skips). ## Why When environments have issues, it's easy to miss real bugs. ## Tests This commit is where it would be useful: `./tools/gh_report.py --commit 6d67410 --markdown` #3167 (comment)
1 parent a0580e4 commit 5f364d0

File tree

1 file changed

+35
-1
lines changed

1 file changed

+35
-1
lines changed

tools/gh_parse.py

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
from collections import Counter
1010
from pathlib import Path
1111

12+
# Total number of environments expected
13+
TOTAL_ENVS = 10
1214

1315
# \u200c is zero-width space. It is added so that len of the string corresponds to real width.
1416
# ❌, ✅, 🔄 each take space of 2 characters.
@@ -17,11 +19,15 @@
1719
PASS = "✅\u200cpass"
1820
SKIP = "🙈\u200cskip"
1921

22+
# FAIL is replaced with BUG when test fails in all environments (and when we have >=TOTAL_ENVS-1 environments)
23+
# This indicate that it's very likely that PR did broke this test rather than environment being flaky.
24+
BUG = "🪲\u200cBUG"
25+
2026
# This happens when Eventually is used - there is output for the test but no result.
2127
MISSING = "🤯\u200cMISS"
2228
PANIC = "💥\u200cPANIC"
2329

24-
INTERESTING_ACTIONS = (FAIL, FLAKY, PANIC, MISSING)
30+
INTERESTING_ACTIONS = (FAIL, BUG, FLAKY, PANIC, MISSING)
2531
ACTIONS_WITH_ICON = INTERESTING_ACTIONS + (PASS, SKIP)
2632

2733
ACTION_MESSAGES = {
@@ -151,6 +157,34 @@ def print_report(filenames, filter, filter_env, show_output, markdown=False):
151157
if e not in test_results:
152158
test_results.setdefault(e, Counter())[MISSING] += 1
153159

160+
# Check if we can convert FAIL to BUG
161+
def is_bug(test_results):
162+
if len(test_results) < TOTAL_ENVS - 1:
163+
# incomplete results
164+
return False
165+
count = 0
166+
for e, env_results in test_results.items():
167+
if PASS in env_results:
168+
return False
169+
if FLAKY in env_results:
170+
return False
171+
if SKIP in env_results:
172+
count -= 1
173+
else:
174+
count += 1
175+
return count >= 0
176+
177+
for testname in all_testnames:
178+
test_results = per_test_per_env_stats.get(testname, {})
179+
if not is_bug(test_results):
180+
continue
181+
for e, env_results in sorted(test_results.items()):
182+
if env_results[FAIL] > 0:
183+
env_results[FAIL] -= 1
184+
if not env_results[FAIL]:
185+
env_results.pop(FAIL)
186+
env_results[BUG] += 1
187+
154188
per_env_stats = {} # env -> action -> count
155189
for testname, items in per_test_per_env_stats.items():
156190
for env, stats in items.items():

0 commit comments

Comments
 (0)