From eafdeaf7985d2f8aeb1f42c49a8580c1a8099ea2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 9 Oct 2025 00:46:05 +0300 Subject: [PATCH 1/6] Add new env. var. EMTEST_RETRY_COUNT to enable forcing retrying of any and all failing tests as flaky. --- test/common.py | 11 ++++++----- test/parallel_testsuite.py | 4 ++++ test/retryable_unit_test.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 test/retryable_unit_test.py diff --git a/test/common.py b/test/common.py index fff3673d6a58b..a7306ade73c52 100644 --- a/test/common.py +++ b/test/common.py @@ -10,6 +10,7 @@ from typing import Dict, Tuple from urllib.parse import unquote, unquote_plus, urlparse, parse_qs from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler +from retryable_unit_test import RetryableTestCase import contextlib import difflib import hashlib @@ -286,8 +287,8 @@ def is_slow_test(func): return decorated -def record_flaky_test(test_name, attempt_count, exception_msg): - logging.info(f'Retrying flaky test "{test_name}" (attempt {attempt_count}/{EMTEST_RETRY_FLAKY} failed):\n{exception_msg}') +def record_flaky_test(test_name, attempt_count, max_attempts, exception_msg): + logging.info(f'Retrying flaky test "{test_name}" (attempt {attempt_count}/{max_attempts} failed):\n{exception_msg}') open(flaky_tests_log_filename, 'a').write(f'{test_name}\n') @@ -313,7 +314,7 @@ def modified(self, *args, **kwargs): return func(self, *args, **kwargs) except (AssertionError, subprocess.TimeoutExpired) as exc: preserved_exc = exc - record_flaky_test(self.id(), i, exc) + record_flaky_test(self.id(), i, EMTEST_RETRY_FLAKY, exc) raise AssertionError('Flaky test has failed too many times') from preserved_exc @@ -1032,7 +1033,7 @@ def __new__(mcs, name, bases, attrs): return type.__new__(mcs, name, bases, new_attrs) -class RunnerCore(unittest.TestCase, metaclass=RunnerMeta): +class RunnerCore(RetryableTestCase, metaclass=RunnerMeta): # default temporary directory settings. set_temp_dir may be called later to # override these temp_dir = shared.TEMP_DIR @@ -2774,7 +2775,7 @@ def run_browser(self, html_file, expected=None, message=None, timeout=None, extr self.assertContained(expected, output) except self.failureException as e: if extra_tries > 0: - record_flaky_test(self.id(), EMTEST_RETRY_FLAKY - extra_tries, e) + record_flaky_test(self.id(), EMTEST_RETRY_FLAKY - extra_tries, EMTEST_RETRY_FLAKY, e) if not self.capture_stdio: print('[enabling stdio/stderr reporting]') self.capture_stdio = True diff --git a/test/parallel_testsuite.py b/test/parallel_testsuite.py index 9969cbace9ff7..20ee23da62a97 100644 --- a/test/parallel_testsuite.py +++ b/test/parallel_testsuite.py @@ -244,6 +244,8 @@ def __init__(self, lock, progress_counter, num_tests): self.lock = lock self.progress_counter = progress_counter self.num_tests = num_tests + self.failures = [] + self.errors = [] @property def test(self): @@ -336,12 +338,14 @@ def addFailure(self, test, err): errlog(f'{self.compute_progress()}{with_color(RED, msg)}') self.buffered_result = BufferedTestFailure(test, err) self.test_result = 'failed' + self.failures += [test] def addError(self, test, err): msg = f'{test} ... ERROR' errlog(f'{self.compute_progress()}{with_color(RED, msg)}') self.buffered_result = BufferedTestError(test, err) self.test_result = 'errored' + self.errors += [test] class BufferedTestBase: diff --git a/test/retryable_unit_test.py b/test/retryable_unit_test.py new file mode 100644 index 0000000000000..d641c5d2f4cd3 --- /dev/null +++ b/test/retryable_unit_test.py @@ -0,0 +1,35 @@ +import common +import logging +import os +import unittest + +# This class patches in to the Python unittest TestCase object to incorporate +# support for an environment variable EMTEST_RETRY_COUNT=x, which enables a +# failed test to be automatically re-run to test if the failure might have been +# due to an instability. +class RetryableTestCase(unittest.TestCase): + def run(self, result=None): + self.origTestMethodName = self._testMethodName + test_retry_count = int(os.getenv('EMTEST_RETRY_COUNT', '0')) + retries_left = test_retry_count + + num_fails = len(result.failures) + num_errors = len(result.errors) + + while retries_left >= 0: + super(RetryableTestCase, self).run(result) + + # The test passed if it didn't accumulate an error. + if len(result.failures) == num_fails and len(result.errors) == num_errors: + return + + retries_left -= 1 + if retries_left >= 0: + if len(result.failures) != num_fails: + err = result.failures.pop(-1) + elif len(result.errors) != num_errors: + err = result.errors.pop(-1) + else: + raise Exception('Internal error in RetryableTestCase: did not detect an error') + + common.record_flaky_test(self.id(), test_retry_count - retries_left, test_retry_count, str(err)) From d77d4cfb056c6fab840a6e22fd64759ef832dee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 16 Oct 2025 04:17:05 +0300 Subject: [PATCH 2/6] ruff --- test/retryable_unit_test.py | 3 +-- test/test_core.py | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test/retryable_unit_test.py b/test/retryable_unit_test.py index d641c5d2f4cd3..2ea8dc7f61deb 100644 --- a/test/retryable_unit_test.py +++ b/test/retryable_unit_test.py @@ -1,5 +1,4 @@ import common -import logging import os import unittest @@ -17,7 +16,7 @@ def run(self, result=None): num_errors = len(result.errors) while retries_left >= 0: - super(RetryableTestCase, self).run(result) + super().run(result) # The test passed if it didn't accumulate an error. if len(result.failures) == num_fails and len(result.errors) == num_errors: diff --git a/test/test_core.py b/test/test_core.py index 7fd563448bb99..957f9788f6049 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -617,6 +617,10 @@ def test_double_i64_conversion(self): def test_float32_precise(self): self.do_core_test('test_float32_precise.c') + def test_flaky(self): + import random + self.assertTrue(random.randint(0,2) == 0) + def test_negative_zero(self): self.do_core_test('test_negative_zero.c') From 0f2a93082c32080ce4b73c2f8c5e7a4d894157b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 16 Oct 2025 04:17:24 +0300 Subject: [PATCH 3/6] Remove test --- test/test_core.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/test_core.py b/test/test_core.py index 957f9788f6049..7fd563448bb99 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -617,10 +617,6 @@ def test_double_i64_conversion(self): def test_float32_precise(self): self.do_core_test('test_float32_precise.c') - def test_flaky(self): - import random - self.assertTrue(random.randint(0,2) == 0) - def test_negative_zero(self): self.do_core_test('test_negative_zero.c') From 47f85872eede2a94f2c58743ef38da3e418eb4b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 16 Oct 2025 14:03:07 +0300 Subject: [PATCH 4/6] Address review --- test/common.py | 2 +- ...ryable_unit_test.py => retryable_unittest.py} | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) rename test/{retryable_unit_test.py => retryable_unittest.py} (69%) diff --git a/test/common.py b/test/common.py index a7306ade73c52..3cd0a8fae6772 100644 --- a/test/common.py +++ b/test/common.py @@ -10,7 +10,7 @@ from typing import Dict, Tuple from urllib.parse import unquote, unquote_plus, urlparse, parse_qs from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler -from retryable_unit_test import RetryableTestCase +from retryable_unittest import RetryableTestCase import contextlib import difflib import hashlib diff --git a/test/retryable_unit_test.py b/test/retryable_unittest.py similarity index 69% rename from test/retryable_unit_test.py rename to test/retryable_unittest.py index 2ea8dc7f61deb..86b34194e2ec3 100644 --- a/test/retryable_unit_test.py +++ b/test/retryable_unittest.py @@ -2,15 +2,17 @@ import os import unittest -# This class patches in to the Python unittest TestCase object to incorporate -# support for an environment variable EMTEST_RETRY_COUNT=x, which enables a -# failed test to be automatically re-run to test if the failure might have been -# due to an instability. +EMTEST_RETRY_COUNT = int(os.getenv('EMTEST_RETRY_COUNT', '0')) + + class RetryableTestCase(unittest.TestCase): + ''' This class patches in to the Python unittest TestCase object to incorporate + support for an environment variable EMTEST_RETRY_COUNT=x, which enables a + failed test to be automatically re-run to test if the failure might have been + due to an instability. ''' + def run(self, result=None): - self.origTestMethodName = self._testMethodName - test_retry_count = int(os.getenv('EMTEST_RETRY_COUNT', '0')) - retries_left = test_retry_count + retries_left = EMTEST_RETRY_COUNT num_fails = len(result.failures) num_errors = len(result.errors) From 6dec5b83aa6ec842d79aaee8527190754af21438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 16 Oct 2025 14:30:47 +0300 Subject: [PATCH 5/6] Fix --- test/retryable_unittest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/retryable_unittest.py b/test/retryable_unittest.py index 86b34194e2ec3..cd37818d3a1be 100644 --- a/test/retryable_unittest.py +++ b/test/retryable_unittest.py @@ -33,4 +33,4 @@ def run(self, result=None): else: raise Exception('Internal error in RetryableTestCase: did not detect an error') - common.record_flaky_test(self.id(), test_retry_count - retries_left, test_retry_count, str(err)) + common.record_flaky_test(self.id(), EMTEST_RETRY_COUNT - retries_left, EMTEST_RETRY_COUNT, str(err)) From a32323078a34cf063026ff530a76f1e0c27410ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 16 Oct 2025 17:45:06 +0300 Subject: [PATCH 6/6] Whitespace --- test/retryable_unittest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/retryable_unittest.py b/test/retryable_unittest.py index cd37818d3a1be..116d4f953afb3 100644 --- a/test/retryable_unittest.py +++ b/test/retryable_unittest.py @@ -6,10 +6,10 @@ class RetryableTestCase(unittest.TestCase): - ''' This class patches in to the Python unittest TestCase object to incorporate + '''This class patches in to the Python unittest TestCase object to incorporate support for an environment variable EMTEST_RETRY_COUNT=x, which enables a failed test to be automatically re-run to test if the failure might have been - due to an instability. ''' + due to an instability.''' def run(self, result=None): retries_left = EMTEST_RETRY_COUNT