Skip to content

Commit ed10409

Browse files
authored
Add reporting of flaky tests in browser suite (#25250)
Add reporting of flaky tests in browser suite to get orange colored swimlane visualizations of flake, and unify reporting of flaky tests in browser and core suites. EMTEST_VISUALIZE=1 then shows <img width="1540" height="874" alt="image" src="https://github.com/user-attachments/assets/232dc021-6e84-46a0-9763-3f9e3294d28b" /> (in above, `flakyX` tests had 50% chance of flaking, or succeeding) and I pattern match the "Retrying flaky test" reports in stdout log to visualize flaked runs as orange: <img width="1244" height="541" alt="image" src="https://github.com/user-attachments/assets/53fe18b1-b6af-44c8-b094-8014897c030b" />
1 parent 8f34f2e commit ed10409

File tree

2 files changed

+9
-5
lines changed

2 files changed

+9
-5
lines changed

test/common.py

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,11 @@ def decorated(self, *args, **kwargs):
231231
return decorated
232232

233233

234+
def record_flaky_test(test_name, attempt_count, exception_msg):
235+
logging.info(f'Retrying flaky test "{test_name}" (attempt {attempt_count}/{EMTEST_RETRY_FLAKY} failed):\n{exception_msg}')
236+
open(flaky_tests_log_filename, 'a').write(f'{test_name}\n')
237+
238+
234239
def flaky(note=''):
235240
assert not callable(note)
236241

@@ -253,9 +258,7 @@ def modified(self, *args, **kwargs):
253258
return f(self, *args, **kwargs)
254259
except (AssertionError, subprocess.TimeoutExpired) as exc:
255260
preserved_exc = exc
256-
logging.info(f'Retrying flaky test "{f.id()}" (attempt {i}/{EMTEST_RETRY_FLAKY} failed): {exc}')
257-
# Mark down that this was a flaky test.
258-
open(flaky_tests_log_filename, 'a').write(f'{f.__name__}\n')
261+
record_flaky_test(self.id(), i, exc)
259262

260263
raise AssertionError('Flaky test has failed too many times') from preserved_exc
261264

@@ -2640,8 +2643,7 @@ def run_browser(self, html_file, expected=None, message=None, timeout=None, extr
26402643
self.assertContained(expected, output)
26412644
except self.failureException as e:
26422645
if extra_tries > 0:
2643-
print(f'[test error in: {self.id()} (see below), automatically retrying]')
2644-
print(e)
2646+
record_flaky_test(self.id(), EMTEST_RETRY_FLAKY - extra_tries, e)
26452647
if not self.capture_stdio:
26462648
print('[enabling stdio/stderr reporting]')
26472649
self.capture_stdio = True

test/parallel_testsuite.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,8 @@ def combine_results(self, result, buffered_results):
203203
# shared data structures are hard in the python multi-processing world, so
204204
# use a file to share the flaky test information across test processes.
205205
flaky_tests = open(common.flaky_tests_log_filename).read().split() if os.path.isfile(common.flaky_tests_log_filename) else []
206+
# Extract only the test short names
207+
flaky_tests = [x.split('.')[-1] for x in flaky_tests]
206208

207209
# The next updateResult loop will print a *lot* of lines really fast. This
208210
# will cause a Python exception being thrown when attempting to print to

0 commit comments

Comments
 (0)