Skip to content

Commit 730766e

Browse files
aemersonaemerson
authored
Add --drop-exec option to filter warmup samples (#120)
This commit introduces a new --drop-exec option that allows users to drop the first N execution samples when running with --exec-multisample. This is useful for mitigating warmup effects that can skew performance measurements. The option accepts an integer N specifying how many initial samples to drop, and works with all execution modes (normal, --exec, and --exec-interleaved-builds).
1 parent bb76956 commit 730766e

File tree

2 files changed

+187
-3
lines changed

2 files changed

+187
-3
lines changed

‎lnt/tests/test_suite.py‎

Lines changed: 59 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,23 @@ def run_test(self, opts):
308308
self._fatal("--single-result must be given a single test name, "
309309
"not a directory name")
310310

311+
# Parse and validate --drop-exec option
312+
if opts.drop_exec is not None:
313+
# Check for incompatible options
314+
if opts.only_compile:
315+
self._fatal("--drop-exec cannot be used with --only-compile")
316+
if opts.build:
317+
self._fatal("--drop-exec cannot be used with --build")
318+
if opts.exec_multisample <= 1:
319+
self._fatal("--drop-exec requires --exec-multisample > 1")
320+
321+
# opts.drop_exec is already an integer from click
322+
if opts.drop_exec < 1:
323+
self._fatal("--drop-exec must be at least 1")
324+
325+
if opts.drop_exec >= opts.exec_multisample:
326+
self._fatal("--drop-exec would drop all %d samples" % opts.exec_multisample)
327+
311328
opts.cppflags = ' '.join(opts.cppflags)
312329
opts.cflags = ' '.join(opts.cflags)
313330
opts.cxxflags = ' '.join(opts.cxxflags)
@@ -428,6 +445,9 @@ def run_test(self, opts):
428445
reports.append(run_report)
429446
json_reports.append(json_data)
430447

448+
# Filter execution samples if --drop-exec was specified
449+
reports, json_reports = self._filter_exec_samples(reports, json_reports)
450+
431451
report = self._create_merged_report(reports)
432452

433453
# Write the report out so it can be read by the submission tool.
@@ -537,8 +557,14 @@ def _run_interleaved_builds(self, opts):
537557
build_dir = build_info['build_dir']
538558
logger.info("Writing report for build: %s" % build_dir)
539559

560+
# Filter execution samples if --drop-exec was specified
561+
build_reports, build_json_reports = self._filter_exec_samples(
562+
reports_by_build[build_idx],
563+
json_by_build[build_idx]
564+
)
565+
540566
# Merge reports for this build
541-
merged_report = self._create_merged_report(reports_by_build[build_idx])
567+
merged_report = self._create_merged_report(build_reports)
542568

543569
# Write JSON report to build directory
544570
report_path = os.path.join(build_dir, 'report.json')
@@ -548,13 +574,13 @@ def _run_interleaved_builds(self, opts):
548574

549575
# Write xUnit XML to build directory
550576
xml_path = os.path.join(build_dir, 'test-results.xunit.xml')
551-
str_template = _lit_json_to_xunit_xml(json_by_build[build_idx])
577+
str_template = _lit_json_to_xunit_xml(build_json_reports)
552578
with open(xml_path, 'w') as fd:
553579
fd.write(str_template)
554580

555581
# Write CSV to build directory
556582
csv_path = os.path.join(build_dir, 'test-results.csv')
557-
str_template = _lit_json_to_csv(json_by_build[build_idx])
583+
str_template = _lit_json_to_csv(build_json_reports)
558584
with open(csv_path, 'w') as fd:
559585
fd.write(str_template)
560586

@@ -607,6 +633,32 @@ def _create_merged_report(self, reports):
607633
test_samples = sum([r.tests for r in reports], [])
608634
return lnt.testing.Report(machine, run, test_samples)
609635

636+
def _filter_exec_samples(self, reports, json_reports):
637+
"""Filter out execution samples based on --drop-exec option.
638+
639+
Returns filtered (reports, json_reports) tuples.
640+
"""
641+
drop_exec_count = self.opts.drop_exec
642+
643+
if drop_exec_count is None or drop_exec_count == 0 or len(reports) == 0:
644+
return reports, json_reports
645+
646+
# Drop the first N samples
647+
filtered_reports = reports[drop_exec_count:]
648+
filtered_json_reports = json_reports[drop_exec_count:]
649+
650+
# Log what we're dropping
651+
if drop_exec_count == 1:
652+
logger.info("Dropping first execution sample (iteration 0)")
653+
else:
654+
logger.info("Dropping first %d execution samples (iterations 0-%d)" %
655+
(drop_exec_count, drop_exec_count - 1))
656+
657+
logger.info("Kept %d of %d execution samples after --drop-exec filtering" %
658+
(len(filtered_reports), len(reports)))
659+
660+
return filtered_reports, filtered_json_reports
661+
610662
def _test_suite_dir(self):
611663
return self.opts.test_suite_root
612664

@@ -1341,6 +1393,10 @@ def diagnose(self):
13411393
@click.option("--compile-multisample", "compile_multisample",
13421394
help="Accumulate compile test data from multiple runs",
13431395
type=int, default=1, metavar="N")
1396+
@click.option("--drop-exec", "drop_exec",
1397+
help="Drop the first N execution samples to mitigate warmup effects. "
1398+
"Used with --exec-multisample.",
1399+
type=int, default=None, metavar="N")
13441400
@click.option("-d", "--diagnose", "diagnose",
13451401
help="Produce a diagnostic report for a particular "
13461402
"test, this will not run all the tests. Must be"

‎tests/runtest/test_suite-drop-exec.shtest‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
# Check --drop-exec feature for filtering execution samples
2+
# This test verifies that --drop-exec correctly drops first N execution samples
3+
4+
# Test 1: --drop-exec without --exec-multisample should fail
5+
# RUN: not lnt runtest test-suite \
6+
# RUN: --sandbox %t.SANDBOX-ERR1 \
7+
# RUN: --no-timestamp \
8+
# RUN: --test-suite %S/Inputs/test-suite-cmake \
9+
# RUN: --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
10+
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
11+
# RUN: --use-make %S/Inputs/test-suite-cmake/fake-make \
12+
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
13+
# RUN: --drop-exec 1 \
14+
# RUN: > %t.err1.log 2> %t.err1.err
15+
# RUN: filecheck --check-prefix CHECK-ERR1 < %t.err1.err %s
16+
# CHECK-ERR1: --drop-exec requires --exec-multisample > 1
17+
18+
# Test 2: --drop-exec with --only-compile should fail
19+
# RUN: not lnt runtest test-suite \
20+
# RUN: --sandbox %t.SANDBOX-ERR2 \
21+
# RUN: --no-timestamp \
22+
# RUN: --test-suite %S/Inputs/test-suite-cmake \
23+
# RUN: --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
24+
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
25+
# RUN: --use-make %S/Inputs/test-suite-cmake/fake-make \
26+
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
27+
# RUN: --only-compile \
28+
# RUN: --drop-exec 1 \
29+
# RUN: > %t.err2.log 2> %t.err2.err
30+
# RUN: filecheck --check-prefix CHECK-ERR2 < %t.err2.err %s
31+
# CHECK-ERR2: --drop-exec cannot be used with --only-compile
32+
33+
# Test 3: --drop-exec with --build should fail
34+
# RUN: not lnt runtest test-suite \
35+
# RUN: --sandbox %t.SANDBOX-ERR3 \
36+
# RUN: --no-timestamp \
37+
# RUN: --test-suite %S/Inputs/test-suite-cmake \
38+
# RUN: --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
39+
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
40+
# RUN: --use-make %S/Inputs/test-suite-cmake/fake-make \
41+
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
42+
# RUN: --build \
43+
# RUN: --drop-exec 1 \
44+
# RUN: > %t.err3.log 2> %t.err3.err
45+
# RUN: filecheck --check-prefix CHECK-ERR3 < %t.err3.err %s
46+
# CHECK-ERR3: --drop-exec cannot be used with --build
47+
48+
# Test 4: --drop-exec dropping all samples should fail
49+
# RUN: not lnt runtest test-suite \
50+
# RUN: --sandbox %t.SANDBOX-ERR4 \
51+
# RUN: --no-timestamp \
52+
# RUN: --test-suite %S/Inputs/test-suite-cmake \
53+
# RUN: --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
54+
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
55+
# RUN: --use-make %S/Inputs/test-suite-cmake/fake-make \
56+
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
57+
# RUN: --exec-multisample 2 \
58+
# RUN: --drop-exec 2 \
59+
# RUN: > %t.err4.log 2> %t.err4.err
60+
# RUN: filecheck --check-prefix CHECK-ERR4 < %t.err4.err %s
61+
# CHECK-ERR4: --drop-exec would drop all 2 samples
62+
63+
# Test 5: --drop-exec 1 with 3 samples should work
64+
# RUN: rm -rf %t.SANDBOX-DROP1
65+
# RUN: lnt runtest test-suite \
66+
# RUN: --sandbox %t.SANDBOX-DROP1 \
67+
# RUN: --no-timestamp \
68+
# RUN: --test-suite %S/Inputs/test-suite-cmake \
69+
# RUN: --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
70+
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
71+
# RUN: --use-make %S/Inputs/test-suite-cmake/fake-make \
72+
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
73+
# RUN: --exec-multisample 3 \
74+
# RUN: --drop-exec 1 \
75+
# RUN: --output %t.drop1.json \
76+
# RUN: > %t.drop1.log 2> %t.drop1.err
77+
# RUN: filecheck --check-prefix CHECK-DROP1 < %t.drop1.err %s
78+
# CHECK-DROP1: Dropping first execution sample (iteration 0)
79+
# CHECK-DROP1: Kept 2 of 3 execution samples after --drop-exec filtering
80+
81+
# Test 6: --drop-exec 2 with 5 samples should work
82+
# RUN: rm -rf %t.SANDBOX-DROP2
83+
# RUN: lnt runtest test-suite \
84+
# RUN: --sandbox %t.SANDBOX-DROP2 \
85+
# RUN: --no-timestamp \
86+
# RUN: --test-suite %S/Inputs/test-suite-cmake \
87+
# RUN: --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
88+
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
89+
# RUN: --use-make %S/Inputs/test-suite-cmake/fake-make \
90+
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
91+
# RUN: --exec-multisample 5 \
92+
# RUN: --drop-exec 2 \
93+
# RUN: --output %t.drop2.json \
94+
# RUN: > %t.drop2.log 2> %t.drop2.err
95+
# RUN: filecheck --check-prefix CHECK-DROP2 < %t.drop2.err %s
96+
# CHECK-DROP2: Dropping first 2 execution samples (iterations 0-1)
97+
# CHECK-DROP2: Kept 3 of 5 execution samples after --drop-exec filtering
98+
99+
# Test 7: --drop-exec with test-prebuilt mode
100+
# First build
101+
# RUN: rm -rf %t.SANDBOX-BUILD-PREBUILT
102+
# RUN: lnt runtest test-suite \
103+
# RUN: --sandbox %t.SANDBOX-BUILD-PREBUILT \
104+
# RUN: --no-timestamp \
105+
# RUN: --test-suite %S/Inputs/test-suite-cmake \
106+
# RUN: --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
107+
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
108+
# RUN: --use-make %S/Inputs/test-suite-cmake/fake-make \
109+
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
110+
# RUN: --build \
111+
# RUN: > %t.build-prebuilt.log 2> %t.build-prebuilt.err
112+
113+
# Now test prebuilt with --drop-exec
114+
# RUN: rm -rf %t.SANDBOX-PREBUILT-TEST
115+
# RUN: lnt runtest test-suite \
116+
# RUN: --sandbox %t.SANDBOX-PREBUILT-TEST \
117+
# RUN: --no-timestamp \
118+
# RUN: --exec \
119+
# RUN: --build-dir %t.SANDBOX-BUILD-PREBUILT/build \
120+
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
121+
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
122+
# RUN: --exec-multisample 3 \
123+
# RUN: --drop-exec 1 \
124+
# RUN: --output %t.prebuilt-drop.json \
125+
# RUN: > %t.prebuilt-drop.log 2> %t.prebuilt-drop.err
126+
# RUN: filecheck --check-prefix CHECK-PREBUILT-DROP < %t.prebuilt-drop.err %s
127+
# CHECK-PREBUILT-DROP: Dropping first execution sample (iteration 0)
128+
# CHECK-PREBUILT-DROP: Kept 2 of 3 execution samples after --drop-exec filtering

0 commit comments

Comments
 (0)