Skip to content

Commit c9a984d

Browse files
DSouzaMDSouzaM
committed
Remove polybench list/run subcommands; use --list to list benchmarks
1 parent 1f00cc8 commit c9a984d

File tree

3 files changed

+60
-63
lines changed

3 files changed

+60
-63
lines changed

‎truffle/mx.truffle/mx_polybench/README.md‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,44 +18,46 @@ details.
1818

1919

2020
```commandline
21-
# Use 'list' to list available benchmarks.
21+
# Use '--list' to list available benchmarks.
2222
$ cd graal/truffle
23-
$ mx polybench list
24-
Benchmark files (run using "mx polybench run <glob_pattern>"):
23+
$ mx polybench --list
24+
Listing all available polybench benchmarks.
25+
Benchmark files (run using "mx polybench <glob_pattern>"):
2526
interpreter/fibonacci.sl
26-
Suites (run using "mx polybench run --suite <suite_name>" or "mx polybench run --suite <suite_name>:<tag1>,<tag2>,..."):
27+
Suites (run using "mx polybench --suite <suite_name>" or "mx polybench --suite <suite_name>:<tag1>,<tag2>,..."):
2728
sl: {'benchmark', 'gate'}
2829
2930
# The list is populated based on the registrations of the loaded suites. When different suites are loaded, different benchmarks may become available.
3031
$ cd graal/sulong
31-
$ mx polybench list
32-
Benchmark files (run using "mx polybench run <glob_pattern>"):
32+
$ mx polybench --list
33+
Listing all available polybench benchmarks.
34+
Benchmark files (run using "mx polybench <glob_pattern>"):
3335
interpreter/deltablue.c.native.bc
3436
interpreter/sieve.c.native.bc
3537
interpreter/richards.c.native.bc
3638
interpreter/fibonacci.c.native.bc
3739
interpreter/fibonacci.sl
38-
Suites (run using "mx polybench run --suite <suite_name>" or "mx polybench run --suite <suite_name>:<tag1>,<tag2>,..."):
40+
Suites (run using "mx polybench --suite <suite_name>" or "mx polybench --suite <suite_name>:<tag1>,<tag2>,..."):
3941
sulong: {'gate', 'benchmark'}
4042
sl: {'gate', 'benchmark'}
4143
4244
# Use 'run' to run benchmarks. Below are some examples.
4345
4446
# Run fibonacci.sl on the JVM (default, but you can specify --jvm to be explicit)
45-
$ mx polybench run interpreter/fibonacci.sl
47+
$ mx polybench interpreter/fibonacci.sl
4648
4749
# Run all interpreter benchmarks in native mode.
48-
$ mx polybench run --native 'interpreter/*'
50+
$ mx polybench --native 'interpreter/*'
4951
5052
# Polybench always uses the mx Java home as its host VM. It is recommended to use '--java-home' to change the host VM.
51-
$ mx --java-home $CUSTOM_GRAALVM_HOME polybench run interpreter/fibonacci.sl
53+
$ mx --java-home $CUSTOM_GRAALVM_HOME polybench interpreter/fibonacci.sl
5254
5355
# Run the sl 'gate' suite (suite definitions are described below, in the "Registering benchmarks" section).
54-
$ mx polybench run --suite sl:gate
56+
$ mx polybench --suite sl:gate
5557
5658
# 'run' accepts any number of arguments after the benchmark/suite name.
5759
# These arguments are passed to the polybench launcher (default), mx benchmark, or the VM:
58-
$ mx polybench run interpreter/fibonacci.sl polybenchArg1 --mx-benchmark-args mxBenchmarkArg --vm-args vmArg --polybench-args polybenchArg2
60+
$ mx polybench interpreter/fibonacci.sl polybenchArg1 --mx-benchmark-args mxBenchmarkArg --vm-args vmArg --polybench-args polybenchArg2
5961
```
6062

6163
## Registering benchmarks

‎truffle/mx.truffle/mx_polybench/command.py‎

Lines changed: 32 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@
4040
#
4141
import argparse
4242
import contextlib
43-
import fnmatch
4443
import shlex
4544
from argparse import ArgumentParser
4645
from enum import Enum
@@ -125,31 +124,27 @@ def append(self, other: "PolybenchArgumentsSpecification") -> "PolybenchArgument
125124
)
126125

127126

128-
def polybench_list(args, _raw_args):
127+
def polybench_list():
129128
benchmarks = _resolve_all_benchmarks()
130-
mx.log('Benchmark files (run using "mx polybench run <glob_pattern>"):')
129+
mx.log("Listing all available polybench benchmarks.")
130+
mx.log('Benchmark files (run using "mx polybench <glob_pattern>"):')
131131
file_found = False
132132
for benchmark_name, resolved_benchmark in benchmarks.items():
133-
if args.glob and not fnmatch.fnmatchcase(benchmark_name, f"*{args.glob}*"):
134-
continue
135133
file_found = True
136134
mx.log(f"\t{benchmark_name}")
137-
if args.verbose:
138-
mx.log(f"\t\tabsolute path: {resolved_benchmark.absolute_path}")
139-
mx.log(f"\t\tdistribution: {resolved_benchmark.suite.benchmark_distribution}")
140-
mx.log(f"\t\tdeclaring suite: {resolved_benchmark.suite.mx_suite.name}")
141-
mx.log(f"\t\trequired languages: {resolved_benchmark.suite.languages}")
135+
mx.logv(f"\t\tabsolute path: {resolved_benchmark.absolute_path}")
136+
mx.logv(f"\t\tdistribution: {resolved_benchmark.suite.benchmark_distribution}")
137+
mx.logv(f"\t\tdeclaring suite: {resolved_benchmark.suite.mx_suite.name}")
138+
mx.logv(f"\t\trequired languages: {resolved_benchmark.suite.languages}")
142139
if not file_found:
143140
mx.log("\tno benchmark files found")
144141

145142
suites = _get_all_suites()
146143
mx.log(
147-
'Suites (run using "mx polybench run --suite <suite_name>" or "mx polybench run --suite <suite_name>:<tag1>,<tag2>,..."):'
144+
'Suites (run using "mx polybench --suite <suite_name>" or "mx polybench --suite <suite_name>:<tag1>,<tag2>,..."):'
148145
)
149146
suite_found = False
150147
for suite_name, suite in suites.items():
151-
if args.glob and not fnmatch.fnmatchcase(suite_name, f"*{args.glob}*"):
152-
continue
153148
suite_found = True
154149
mx.log(f"\t{suite_name}: {suite.tags if suite.tags else '{}'}")
155150
if not suite_found:
@@ -200,7 +195,7 @@ def _run_suite(args):
200195
.to_normalized_command_line()
201196
)
202197

203-
base_args = ["run"]
198+
base_args = []
204199
if args.dry_run:
205200
base_args.append("--dry-run")
206201
elif args.dry_run_polybench:
@@ -293,7 +288,7 @@ def _parse_mx_benchmark_pattern(pattern: str, pattern_is_glob: bool) -> str:
293288
if pattern.count(":") == 1:
294289
message += ' This pattern looks like a suite. Did you forget "--suite"?'
295290
else:
296-
message += ' Use "mx polybench run -h" to view the expected format for benchmark patterns.'
291+
message += ' Use "mx polybench -h" to view the expected format for benchmark patterns.'
297292
mx.abort(message)
298293

299294
if pattern_is_glob:
@@ -412,24 +407,20 @@ def _create_parser() -> ArgumentParser:
412407
prog="mx polybench",
413408
description=(
414409
"mx polybench is a simple command line interface for Polybench, the system used to benchmark Truffle languages. "
415-
"It is a thin wrapper around Polybench's mx benchmark integration that makes it easy to discover benchmarks "
416-
'(using "mx polybench list") and to run them (using "mx polybench run"). '
410+
"It is a thin wrapper around Polybench's mx benchmark integration that makes it easy to discover and run benchmarks. "
417411
"It also supports batch execution of the benchmarks in a suite, which is convenient for defining CI jobs."
418412
),
419413
)
420-
subparsers = parser.add_subparsers(dest="command", required=True, help="the polybench command to run")
421414

422-
list_parser = subparsers.add_parser("list", help="list the available benchmarks")
423-
list_parser.add_argument("glob", nargs="?", type=str, help="a glob pattern to filter benchmark output")
424-
list_parser.add_argument("--verbose", action="store_true", help="print a detailed view of the benchmarks")
425-
list_parser.set_defaults(func=polybench_list)
426-
427-
run_parser = subparsers.add_parser("run", help="run one or more benchmarks")
428-
run_parser.add_argument(
415+
parser.add_argument(
429416
"benchmarks",
417+
nargs="?",
430418
help='a glob pattern representing benchmarks to run (e.g., "interpreter/*"), '
431419
'or a suite specification if "--suite" is provided. '
432-
'Use "mx polybench list" to see a list of available benchmarks.',
420+
'Use "--list" to see a list of available benchmarks.',
421+
)
422+
parser.add_argument(
423+
"--list", action="store_true", help="list all available benchmarks (without actually executing them)."
433424
)
434425

435426
run_flags = set()
@@ -438,7 +429,7 @@ def run_flag(run_arg):
438429
run_flags.add(run_arg)
439430
return run_arg
440431

441-
dry_run_group = run_parser.add_mutually_exclusive_group()
432+
dry_run_group = parser.add_mutually_exclusive_group()
442433
dry_run_group.add_argument(
443434
run_flag("--dry-run"),
444435
action="store_true",
@@ -450,7 +441,7 @@ def run_flag(run_arg):
450441
help="log the mx polybench commands that would be executed by this command (without actually executing them)",
451442
)
452443

453-
mode_group = run_parser.add_mutually_exclusive_group()
444+
mode_group = parser.add_mutually_exclusive_group()
454445
mode_group.add_argument(
455446
run_flag("--jvm"),
456447
action="store_false",
@@ -464,13 +455,13 @@ def run_flag(run_arg):
464455
dest="is_native",
465456
help="run the benchmark with a native image of the Truffle interpreter",
466457
)
467-
run_parser.add_argument(
458+
parser.add_argument(
468459
run_flag("--pgo"), action="store_true", default=False, help="use PGO (only valid for native runs)"
469460
)
470-
run_parser.add_argument(
461+
parser.add_argument(
471462
run_flag("--g1gc"), action="store_true", default=False, help="use G1GC (only valid for native runs)"
472463
)
473-
benchmark_pattern_group = run_parser.add_mutually_exclusive_group()
464+
benchmark_pattern_group = parser.add_mutually_exclusive_group()
474465
benchmark_pattern_group.add_argument(
475466
run_flag("--suite"),
476467
action="store_true",
@@ -491,19 +482,18 @@ def benchmark_argument(arg):
491482
"""Prevents users from accidentally passing a run flag in the benchmark arguments section."""
492483
if arg in run_flags:
493484
mx.abort(
494-
f'Flag "{arg}" is an "mx polybench run" flag, but it was specified in the extra benchmark arguments section.\n'
495-
f'Move it before the benchmark pattern/suite specification to fix this error (e.g., "mx polybench run {arg} ... benchmark_or_suite ...").'
485+
f'Flag "{arg}" is an "mx polybench" flag, but it was specified in the extra benchmark arguments section.\n'
486+
f'Move it before the benchmark pattern/suite specification to fix this error (e.g., "mx polybench {arg} ... benchmark_or_suite ...").'
496487
)
497488
return arg
498489

499-
run_parser.add_argument(
490+
parser.add_argument(
500491
"benchmark_arguments",
501492
nargs=argparse.REMAINDER,
502493
type=benchmark_argument,
503494
help=PolybenchArgumentsSpecification.parser_help_message(),
504495
)
505496

506-
run_parser.set_defaults(func=polybench_run)
507497
return parser
508498

509499

@@ -514,5 +504,10 @@ def benchmark_argument(arg):
514504
def polybench_command(args):
515505
"""Run one or more benchmarks using polybench."""
516506
parsed_args = parser.parse_args(args)
517-
mx.logv(f"Running polybench command {parsed_args.command} with arguments {parsed_args}")
518-
parsed_args.func(parsed_args, args)
507+
mx.logv(f"Running polybench with arguments {parsed_args}")
508+
if parsed_args.list:
509+
polybench_list()
510+
elif parsed_args.benchmarks:
511+
polybench_run(parsed_args, args)
512+
else:
513+
mx.abort('A benchmark pattern or --list must be specified. Use "mx polybench -h" for more info.')

‎vm/ci/ci_common/common-bench.jsonnet‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
171171
['mx', '--dy', '/sulong', 'build', '--dependencies', 'SULONG_POLYBENCH_BENCHMARKS'],
172172
],
173173
run+: [
174-
self.polybench_wrap(['mx', '--dy', '/sulong', '--java-home', '${POLYBENCH_JVM}', 'polybench', 'run', '--suite', 'sulong:gate']),
174+
self.polybench_wrap(['mx', '--dy', '/sulong', '--java-home', '${POLYBENCH_JVM}', 'polybench', '--suite', 'sulong:gate']),
175175
],
176176
notify_groups +: ['sulong'],
177177
},
@@ -181,7 +181,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
181181
['mx', '--dy', '/sulong', 'build', '--dependencies', 'SULONG_POLYBENCH_BENCHMARKS'],
182182
],
183183
run+: [
184-
self.polybench_wrap(['mx', '--dy', '/sulong', '--java-home', '${POLYBENCH_JVM}', 'polybench', 'run', '--suite', 'sulong:benchmark']),
184+
self.polybench_wrap(['mx', '--dy', '/sulong', '--java-home', '${POLYBENCH_JVM}', 'polybench', '--suite', 'sulong:benchmark']),
185185
],
186186
notify_groups +: ['sulong'],
187187
}
@@ -193,7 +193,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
193193
['mx', '--dy', '/wasm', 'build', '--dependencies', 'WASM_POLYBENCH_BENCHMARKS']
194194
],
195195
run+: [
196-
self.polybench_wrap(['mx', '--dy', '/wasm', '--java-home', '${POLYBENCH_JVM}', 'polybench', 'run', '--suite', 'wasm:gate']),
196+
self.polybench_wrap(['mx', '--dy', '/wasm', '--java-home', '${POLYBENCH_JVM}', 'polybench', '--suite', 'wasm:gate']),
197197
],
198198
notify_groups +: ['wasm'],
199199
},
@@ -203,7 +203,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
203203
['mx', '--dy', '/wasm', 'build', '--dependencies', 'WASM_POLYBENCH_BENCHMARKS']
204204
],
205205
run+: [
206-
self.polybench_wrap(['mx', '--dy', '/wasm', '--java-home', '${POLYBENCH_JVM}', 'polybench', 'run', '--suite', 'wasm:benchmark']),
206+
self.polybench_wrap(['mx', '--dy', '/wasm', '--java-home', '${POLYBENCH_JVM}', 'polybench', '--suite', 'wasm:benchmark']),
207207
],
208208
notify_groups +: ['wasm'],
209209
}
@@ -214,7 +214,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
214214
['mx', '--dy', '/espresso', 'build']
215215
],
216216
run+: [
217-
self.polybench_wrap(['mx', '--dy', '/espresso', '--java-home', '${POLYBENCH_JVM}', 'polybench', 'run', '--suite', 'espresso:gate']),
217+
self.polybench_wrap(['mx', '--dy', '/espresso', '--java-home', '${POLYBENCH_JVM}', 'polybench', '--suite', 'espresso:gate']),
218218
],
219219
notify_groups +: ['espresso'],
220220
},
@@ -223,7 +223,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
223223
['mx', '--dy', '/espresso', 'build']
224224
],
225225
run+: [
226-
self.polybench_wrap(['mx', '--dy', '/espresso', '--java-home', '${POLYBENCH_JVM}', 'polybench', 'run', '--suite', 'espresso:benchmark']),
226+
self.polybench_wrap(['mx', '--dy', '/espresso', '--java-home', '${POLYBENCH_JVM}', 'polybench', '--suite', 'espresso:benchmark']),
227227
],
228228
notify_groups +: ['espresso'],
229229
}
@@ -237,7 +237,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
237237
['mx', '--dy', 'truffleruby', 'build']
238238
],
239239
run+: [
240-
self.polybench_wrap(['mx', '--dy', 'truffleruby', '--java-home', '${POLYBENCH_JVM}', 'polybench', 'run', '--suite', 'ruby:gate']),
240+
self.polybench_wrap(['mx', '--dy', 'truffleruby', '--java-home', '${POLYBENCH_JVM}', 'polybench', '--suite', 'ruby:gate']),
241241
],
242242
notify_groups +: ['ruby'],
243243
},
@@ -249,7 +249,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
249249
['mx', '--dy', 'truffleruby', 'build']
250250
],
251251
run+: [
252-
self.polybench_wrap(['mx', '--dy', 'truffleruby', '--java-home', '${POLYBENCH_JVM}', 'polybench', 'run', '--suite', 'ruby:benchmark']),
252+
self.polybench_wrap(['mx', '--dy', 'truffleruby', '--java-home', '${POLYBENCH_JVM}', 'polybench', '--suite', 'ruby:benchmark']),
253253
],
254254
notify_groups +: ['ruby'],
255255
}
@@ -260,7 +260,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
260260
['mx', '--dy', 'graalpython', 'build']
261261
],
262262
run+: [
263-
self.polybench_wrap(['mx', '--dy', 'graalpython', '--java-home', '${POLYBENCH_JVM}', 'polybench', 'run', '--suite', 'python:gate']),
263+
self.polybench_wrap(['mx', '--dy', 'graalpython', '--java-home', '${POLYBENCH_JVM}', 'polybench', '--suite', 'python:gate']),
264264
],
265265
notify_groups +: ['python'],
266266
},
@@ -269,7 +269,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
269269
['mx', '--dy', 'graalpython', 'build']
270270
],
271271
run+: [
272-
self.polybench_wrap(['mx', '--dy', 'graalpython', '--java-home', '${POLYBENCH_JVM}', 'polybench', 'run', '--suite', 'python:benchmark']),
272+
self.polybench_wrap(['mx', '--dy', 'graalpython', '--java-home', '${POLYBENCH_JVM}', 'polybench', '--suite', 'python:benchmark']),
273273
],
274274
notify_groups +: ['python'],
275275
}
@@ -280,15 +280,15 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
280280
['mx', '--dy', '/graal-js', 'build']
281281
],
282282
run+: [
283-
self.polybench_wrap(['mx', '--dy', '/graal-js', '--java-home', '${POLYBENCH_JVM}', 'polybench', 'run', '--suite', 'nfi:gate']),
283+
self.polybench_wrap(['mx', '--dy', '/graal-js', '--java-home', '${POLYBENCH_JVM}', 'polybench', '--suite', 'nfi:gate']),
284284
],
285285
},
286286
self.polybench_vm_daily('linux', 'amd64', 'nfi') + {
287287
setup+: [
288288
['mx', '--dy', '/graal-js', 'build']
289289
],
290290
run+: [
291-
self.polybench_wrap(['mx', '--dy', '/graal-js', '--java-home', '${POLYBENCH_JVM}', 'polybench', 'run', '--suite', 'nfi:benchmark']),
291+
self.polybench_wrap(['mx', '--dy', '/graal-js', '--java-home', '${POLYBENCH_JVM}', 'polybench', '--suite', 'nfi:benchmark']),
292292
],
293293
}
294294
] + [
@@ -298,7 +298,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
298298
['mx', '--dy', '/graal-js', 'build']
299299
],
300300
run+: [
301-
self.polybench_wrap(['mx', '--dy', '/graal-js', '--java-home', '${POLYBENCH_JVM}', 'polybench', 'run', '--suite', 'js:gate']),
301+
self.polybench_wrap(['mx', '--dy', '/graal-js', '--java-home', '${POLYBENCH_JVM}', 'polybench', '--suite', 'js:gate']),
302302
],
303303
notify_groups +: ['javascript'],
304304
},
@@ -307,7 +307,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
307307
['mx', '--dy', '/graal-js', 'build']
308308
],
309309
run+: [
310-
self.polybench_wrap(['mx', '--dy', '/graal-js', '--java-home', '${POLYBENCH_JVM}', 'polybench', 'run', '--suite', 'js:benchmark']),
310+
self.polybench_wrap(['mx', '--dy', '/graal-js', '--java-home', '${POLYBENCH_JVM}', 'polybench', '--suite', 'js:benchmark']),
311311
],
312312
notify_groups +: ['javascript'],
313313
}

0 commit comments

Comments
 (0)