Skip to content

Commit 79f80a4

Browse files
authored
Add a feature for using the same number of loops as a previous run (#327)
Motivation: On the Faster CPython team, we often collect pystats (counters of various interpreter events) by running the benchmark suite. It is very useful to compare the stats between two commits to see how a pull request affects the interpreter. Unfortunately, with pyperformance's default behavior where the number of loops is automatically calibrated, each benchmark may not be run the same number of times from run-to-run, making the data hard to compare. This change adds a new argument to the "run" command which will use the same number of loops as a previous run. The loops for each benchmark is looked up from the metadata in the .json output of that previous run, and passed to the underlying call to pyperf using the --loops argument. Additionally, this modifies one of the benchmarks (sqlglot) to be compatible with that scheme. sqlglot is the only run_benchmark.py script that runs multiple benchmarks within it in a single call to the script. This makes it impossible to set the number of loops independently for each of these benchmarks. It's been updated to use the pattern from other "suites" of benchmarks (e.g. async_tree) where each benchmark has its own .toml file and is run independently. This should still be backward compatible with older data collected from this benchmark, but doing "pyperformance run -b sqlglot" will now only run a single benchmark.
1 parent dcf71dc commit 79f80a4

File tree

11 files changed

+78
-5
lines changed

11 files changed

+78
-5
lines changed

doc/changelog.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
Changelog
22
=========
33

4+
* Add a --same-loops option to the run command to use the exact same number of
5+
loops as a previous run (without recalibrating).
6+
47
Version 1.10.0 (2023-10-22)
58
--------------
69
* Add benchmark for asyncio_webockets

doc/usage.rst

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,10 @@ options::
140140
-p PYTHON, --python PYTHON
141141
Python executable (default: use running
142142
Python)
143+
--same-loops SAME_LOOPS
144+
Use the same number of loops as a previous run
145+
(i.e., don't recalibrate). Should be a path to a
146+
.json file from a previous run.
143147

144148
show
145149
----

pyperformance/cli.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,10 @@ def parse_args():
7575
cmd.add_argument("--min-time", metavar="MIN_TIME",
7676
help="Minimum duration in seconds of a single "
7777
"value, used to calibrate the number of loops")
78+
cmd.add_argument("--same-loops",
79+
help="Use the same number of loops as a previous run "
80+
"(i.e., don't recalibrate). Should be a path to a "
81+
".json file from a previous run.")
7882
filter_opts(cmd)
7983

8084
# show

pyperformance/compile.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -543,6 +543,8 @@ def run_benchmark(self, python=None):
543543
cmd.extend(('--affinity', self.conf.affinity))
544544
if self.conf.debug:
545545
cmd.append('--debug-single-value')
546+
if self.conf.same_loops:
547+
cmd.append('--same_loops=%s' % self.conf.same_loops)
546548
exitcode = self.run_nocheck(*cmd)
547549

548550
if os.path.exists(self.filename):
@@ -812,6 +814,7 @@ def getint(section, key, default=None):
812814
conf.benchmarks = getstr('run_benchmark', 'benchmarks', default='')
813815
conf.affinity = getstr('run_benchmark', 'affinity', default='')
814816
conf.upload = getboolean('run_benchmark', 'upload', False)
817+
conf.same_loops = getfile('run_benchmark', 'same_loops', default='')
815818

816819
# paths
817820
conf.build_dir = os.path.join(conf.directory, 'build')

pyperformance/data-files/benchmarks/MANIFEST

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,9 @@ spectral_norm <local>
7777
sqlalchemy_declarative <local>
7878
sqlalchemy_imperative <local>
7979
sqlglot <local>
80+
sqlglot_parse <local:sqlglot>
81+
sqlglot_transpile <local:sqlglot>
82+
sqlglot_optimize <local:sqlglot>
8083
sqlite_synth <local>
8184
sympy <local>
8285
telco <local>
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[tool.pyperformance]
2+
name = "sqlglot_optimize"
3+
extra_opts = ["optimize"]
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[tool.pyperformance]
2+
name = "sqlglot_parse"
3+
extra_opts = ["parse"]
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[tool.pyperformance]
2+
name = "sqlglot_transpile"
3+
extra_opts = ["transpile"]

pyperformance/data-files/benchmarks/bm_sqlglot/pyproject.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,4 @@ dynamic = ["version"]
1010

1111
[tool.pyperformance]
1212
name = "sqlglot"
13+
extra_opts = ["normalize"]

pyperformance/data-files/benchmarks/bm_sqlglot/run_benchmark.py

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -164,10 +164,31 @@ def bench_normalize(loops):
164164
return elapsed
165165

166166

167+
BENCHMARKS = {
168+
"parse": bench_parse,
169+
"transpile": bench_transpile,
170+
"optimize": bench_optimize,
171+
"normalize": bench_normalize
172+
}
173+
174+
175+
def add_cmdline_args(cmd, args):
176+
cmd.append(args.benchmark)
177+
178+
179+
def add_parser_args(parser):
180+
parser.add_argument(
181+
"benchmark",
182+
choices=BENCHMARKS,
183+
help="Which benchmark to run."
184+
)
185+
186+
167187
if __name__ == "__main__":
168-
runner = pyperf.Runner()
188+
runner = pyperf.Runner(add_cmdline_args=add_cmdline_args)
169189
runner.metadata['description'] = "SQLGlot benchmark"
170-
runner.bench_time_func("sqlglot_parse", bench_parse)
171-
runner.bench_time_func("sqlglot_transpile", bench_transpile)
172-
runner.bench_time_func("sqlglot_optimize", bench_optimize)
173-
runner.bench_time_func("sqlglot_normalize", bench_normalize)
190+
add_parser_args(runner.argparser)
191+
args = runner.parse_args()
192+
benchmark = args.benchmark
193+
194+
runner.bench_time_func(f"sqlglot_{benchmark}", BENCHMARKS[benchmark])

0 commit comments

Comments
 (0)