Skip to content

Commit 2f4c518

Browse files
committed
ci: run only 10% (5 times) of test runs for long tests
If it is a really long test (clickhouse-test define it as 180 seconds for execution), the with default flaky check of --test-runs 50, it will take 2.5 hours to excute, which is way too much, let's run them only 10% of retries. Example: https://github.com/ClickHouse/ClickHouse/actions/runs/14397097786/job/40375284762?pr=79013
1 parent aa02020 commit 2f4c518

File tree

1 file changed

+26
-14
lines changed

1 file changed

+26
-14
lines changed

tests/clickhouse-test

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2329,13 +2329,11 @@ class TestSuite:
23292329
_ = x
23302330
return True
23312331

2332-
self.all_tests: List[str] = self.get_tests_list(
2333-
self.tests_in_suite_key_func, filter_func
2334-
)
2332+
all_tests = list(self.get_selected_tests(filter_func))
23352333

23362334
all_tags_and_random_settings_limits = (
23372335
self.read_test_tags_and_random_settings_limits(
2338-
self.suite_path, self.all_tests
2336+
self.suite_path, all_tests
23392337
)
23402338
)
23412339
self.all_tags: Dict[str, Set[str]] = all_tags_and_random_settings_limits[0]
@@ -2344,6 +2342,10 @@ class TestSuite:
23442342
)
23452343
self.sequential_tests = []
23462344
self.parallel_tests = []
2345+
2346+
self.all_tests = self.apply_test_runs(all_tests)
2347+
self.all_tests.sort(key=self.tests_in_suite_key_func)
2348+
23472349
for test_name in self.all_tests:
23482350
if self.is_sequential_test(test_name):
23492351
if not args.no_sequential:
@@ -2352,6 +2354,19 @@ class TestSuite:
23522354
if not args.no_parallel:
23532355
self.parallel_tests.append(test_name)
23542356

2357+
def apply_test_runs(self, all_tests):
2358+
test_runs = self.args.test_runs
2359+
long_test_runs = max(int(self.args.test_runs * self.args.long_test_runs_ratio), 1)
2360+
2361+
all_tests = map(lambda test: [test] * (long_test_runs if self.is_long_test(test) else test_runs), all_tests)
2362+
all_tests = [item for sublist in all_tests for item in sublist]
2363+
return all_tests
2364+
2365+
def is_long_test(self, test_name):
2366+
if test_name not in self.all_tags:
2367+
return False
2368+
return "long" in self.all_tags[test_name]
2369+
23552370
def is_sequential_test(self, test_name):
23562371
if args.sequential:
23572372
if any(s in test_name for s in args.sequential):
@@ -2366,16 +2381,6 @@ class TestSuite:
23662381
or ("stateful" in self.all_tags[test_name])
23672382
)
23682383

2369-
def get_tests_list(self, sort_key, filter_func):
2370-
"""
2371-
Return list of tests file names to run
2372-
"""
2373-
2374-
all_tests = list(self.get_selected_tests(filter_func))
2375-
all_tests = all_tests * self.args.test_runs
2376-
all_tests.sort(key=sort_key)
2377-
return all_tests
2378-
23792384
def get_selected_tests(self, filter_func):
23802385
"""
23812386
Find all files with tests, filter, render templates
@@ -3531,6 +3536,13 @@ def parse_args():
35313536
type=int,
35323537
help="Run each test many times (useful for e.g. flaky check)",
35333538
)
3539+
parser.add_argument(
3540+
"--long-test-runs-ratio",
3541+
default=0.1,
3542+
nargs="?",
3543+
type=float,
3544+
help="Ratio from --test-runs for long tests",
3545+
)
35343546
parser.add_argument(
35353547
"-U",
35363548
"--unified",

0 commit comments

Comments
 (0)