Skip to content

Commit 3e3437c

Browse files
steve-ssteve-s
committed
Add interpreter benchmarks
1 parent d7da3e1 commit 3e3437c

File tree

9 files changed

+103
-27
lines changed

9 files changed

+103
-27
lines changed

‎graalpython/com.oracle.graal.python.benchmarks/python/meso/raytrace-simple.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -365,5 +365,5 @@ def _main(width=400, height=400):
365365
s.addObject(Halfspace(Point(0,0,0), Vector.UP), CheckerboardSurface())
366366
s.render(c)
367367

368-
def __benchmark__():
369-
_main()
368+
def __benchmark__(*args):
369+
_main(*args)

‎graalpython/com.oracle.graal.python.benchmarks/python/micro/builtin-len.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,13 @@ def call_len(num, ll):
3333
return length
3434

3535

36-
def measure():
36+
def measure(num=500000000):
3737
ll = [x * 2 for x in range(1000)]
38-
length = call_len(500000000, ll) # 1000000000
38+
length = call_len(num, ll) # 1000000000
3939

4040
print("Final length ", length)
4141

4242

4343

4444
def __benchmark__(*args):
45-
measure()
45+
measure(*args)

‎graalpython/com.oracle.graal.python.benchmarks/python/micro/generator-expression-sized.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,11 @@ def call_generator_localvar(num, iteration):
4545
return item
4646

4747

48-
def measure(num):
49-
last_item = call_generator(num, 10000)
48+
def measure(num, iteration):
49+
last_item = call_generator(num, iteration)
5050

5151
print("Last item ", last_item)
5252

5353

54-
def __benchmark__(num):
55-
measure(num)
54+
def __benchmark__(num, it=10000):
55+
measure(num, it)

‎graalpython/com.oracle.graal.python.benchmarks/python/micro/generator-notaligned-sized.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,11 @@ def call_generator(num, iteration):
4040
return item
4141

4242

43-
def measure(num):
44-
last_item = call_generator(num, 10000)
43+
def measure(num, iteration):
44+
last_item = call_generator(num, iteration)
4545

4646
print("Last item ", last_item)
4747

4848

49-
def __benchmark__(num):
50-
measure(num)
49+
def __benchmark__(num, it=10000):
50+
measure(num, it)

‎graalpython/com.oracle.graal.python.benchmarks/python/micro/generator-sized.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,11 +39,11 @@ def call_generator(num, iteration):
3939
return item
4040

4141

42-
def measure(num):
43-
last_item = call_generator(num, 10000)
42+
def measure(num, iteration):
43+
last_item = call_generator(num, iteration)
4444

4545
print("Last item ", last_item)
4646

4747

48-
def __benchmark__(num):
49-
measure(num)
48+
def __benchmark__(num, it=10000):
49+
measure(num, it)

‎graalpython/com.oracle.graal.python.benchmarks/python/micro/genexp-builtin-call-sized.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,10 @@ def call_generator_localvar(num, iteration):
5555
return item
5656

5757

58-
def measure(num):
59-
last_item = call_generator(num, 10000)
58+
def measure(num, iteration):
59+
last_item = call_generator(num, iteration)
6060
print("Last item ", last_item)
6161

6262

63-
def __benchmark__(num=1000):
64-
measure(num)
63+
def __benchmark__(num=1000, it=10000):
64+
measure(num, it)

‎mx.graalpython/mx_graalpython.py

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,15 @@
2727
import contextlib
2828
import datetime
2929
import glob
30-
import itertools
3130
import json
3231
import os
3332
import pathlib
3433
import platform
3534
import re
3635
import shlex
3736
import shutil
37+
38+
import itertools
3839
import sys
3940

4041
HPY_IMPORT_ORPHAN_BRANCH_NAME = "hpy-import"
@@ -57,12 +58,14 @@
5758
import mx_graalpython_bisect
5859
from mx_gate import Task
5960
from mx_graalpython_bench_param import PATH_MESO, BENCHMARKS, WARMUP_BENCHMARKS, JBENCHMARKS, PARSER_BENCHMARKS
60-
from mx_graalpython_benchmark import PythonBenchmarkSuite, python_vm_registry, CPythonVm, PyPyVm, JythonVm, GraalPythonVm, \
61+
from mx_graalpython_benchmark import PythonBenchmarkSuite, python_vm_registry, CPythonVm, PyPyVm, JythonVm, \
62+
GraalPythonVm, \
6163
CONFIGURATION_DEFAULT, CONFIGURATION_SANDBOXED, CONFIGURATION_NATIVE, \
6264
CONFIGURATION_DEFAULT_MULTI, CONFIGURATION_SANDBOXED_MULTI, CONFIGURATION_NATIVE_MULTI, \
6365
CONFIGURATION_DEFAULT_MULTI_TIER, CONFIGURATION_NATIVE_MULTI_TIER, \
64-
PythonInteropBenchmarkSuite, PythonVmWarmupBenchmarkSuite, PythonParserBenchmarkSuite
65-
66+
PythonInteropBenchmarkSuite, PythonVmWarmupBenchmarkSuite, PythonParserBenchmarkSuite, \
67+
CONFIGURATION_INTERPRETER, CONFIGURATION_INTERPRETER_MULTI, CONFIGURATION_NATIVE_INTERPRETER, \
68+
CONFIGURATION_NATIVE_INTERPRETER_MULTI
6669

6770
if not sys.modules.get("__main__"):
6871
# workaround for pdb++
@@ -1490,9 +1493,15 @@ def _register_vms(namespace):
14901493

14911494
# graalpython
14921495
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_DEFAULT), SUITE, 10)
1496+
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_INTERPRETER, extra_polyglot_args=[
1497+
'--experimental-options', '--engine.Compilation=false'
1498+
]), SUITE, 10)
14931499
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_DEFAULT_MULTI, extra_polyglot_args=[
14941500
'--experimental-options', '-multi-context',
14951501
]), SUITE, 10)
1502+
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_INTERPRETER_MULTI, extra_polyglot_args=[
1503+
'--experimental-options', '-multi-context', '--engine.Compilation=false'
1504+
]), SUITE, 10)
14961505
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_DEFAULT_MULTI_TIER, extra_polyglot_args=[
14971506
'--experimental-options', '--engine.MultiTier=true',
14981507
]), SUITE, 10)
@@ -1501,12 +1510,17 @@ def _register_vms(namespace):
15011510
]), SUITE, 10)
15021511
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_NATIVE, extra_polyglot_args=[
15031512
]), SUITE, 10)
1513+
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_NATIVE_INTERPRETER, extra_polyglot_args=[
1514+
'--experimental-options', '--engine.Compilation=false']), SUITE, 10)
15041515
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_SANDBOXED_MULTI, extra_polyglot_args=[
15051516
'--experimental-options', '-multi-context', '--llvm.managed',
15061517
]), SUITE, 10)
15071518
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_NATIVE_MULTI, extra_polyglot_args=[
15081519
'--experimental-options', '-multi-context',
15091520
]), SUITE, 10)
1521+
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_NATIVE_INTERPRETER_MULTI, extra_polyglot_args=[
1522+
'--experimental-options', '-multi-context', '--engine.Compilation=false',
1523+
]), SUITE, 10)
15101524
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_NATIVE_MULTI_TIER, extra_polyglot_args=[
15111525
'--experimental-options', '--engine.MultiTier=true',
15121526
]), SUITE, 10)

‎mx.graalpython/mx_graalpython_bench_param.py

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
ITER_20 = ['-i', '20']
5353
ITER_15 = ['-i', '15']
5454
ITER_10 = ['-i', '10']
55+
WARMUP_2 = ['-w', '2']
5556

5657
MICRO_BENCHMARKS = {
5758
'arith-binop': ITER_10 + ['5'],
@@ -101,6 +102,52 @@
101102
'tuple-indexing-from-literal': ITER_10 + ['10000000'],
102103
}
103104

105+
MICRO_BENCHMARKS_SMALL = {
106+
'arith-modulo-sized': ITER_10 + WARMUP_2 + ['1'],
107+
'attribute-access-polymorphic': ITER_10 + WARMUP_2 + ['20'],
108+
'attribute-access': ITER_10 + WARMUP_2 + ['100'],
109+
'attribute-access-super': ITER_10 + WARMUP_2 + ['100'],
110+
'attribute-bool': ITER_10 + WARMUP_2 + ['10'],
111+
'boolean-logic-sized': ITER_10 + WARMUP_2 + ['10'],
112+
'builtin-len-tuple-sized': ITER_10 + WARMUP_2 + ['10_000_000'],
113+
'builtin-len': ITER_10 + WARMUP_2 + ['1_000_000'],
114+
'class-access': ITER_15 + WARMUP_2 + ['100'],
115+
'call-method-polymorphic': ITER_10 + WARMUP_2 + ['10'],
116+
'for-range': ITER_10 + WARMUP_2 + WARMUP_2 + ['100'],
117+
'function-call-sized': ITER_10 + WARMUP_2 + ['10_000_000'],
118+
'generator-expression-sized': ITER_10 + WARMUP_2 + ['3000', '500'],
119+
'generator-notaligned-sized': ITER_10 + WARMUP_2 + ['3000', '500'],
120+
'generator-sized': ITER_10 + WARMUP_2 + ['3000', '500'],
121+
'genexp-builtin-call-sized': ITER_10 + WARMUP_2 + ['3000', '500'],
122+
'list-comp': ITER_10 + WARMUP_2 + ['100'],
123+
'list-indexing': ITER_10 + WARMUP_2 + ['10_000'],
124+
'list-indexing-from-constructor': ITER_10 + WARMUP_2 + ['1_000_000'],
125+
'list-indexing-from-literal': ITER_10 + WARMUP_2 + ['1_000_000'],
126+
'list-iterating-explicit': ITER_10 + WARMUP_2 + ['100_000'],
127+
'list-iterating': ITER_10 + WARMUP_2 + ['100_000'],
128+
'list-iterating-obj-sized': ITER_10 + WARMUP_2 + ['1_000_000'],
129+
'list-constructions-sized': ITER_10 + WARMUP_2 + ['1_000'],
130+
'dict-getitem-sized': ITER_10 + WARMUP_2 + ['1_000_000'],
131+
'math-sqrt': ITER_10 + WARMUP_2 + ['100_000_000'],
132+
'object-allocate': ITER_10 + WARMUP_2 + ['100'],
133+
'object-layout-change': ITER_10 + WARMUP_2 + ['10_000'],
134+
'special-add-int-sized': ITER_10 + WARMUP_2 + ['1_000'],
135+
'special-add-sized': ITER_10 + WARMUP_2 + ['1_000'],
136+
'special-len': ITER_10 + WARMUP_2 + ['5'],
137+
'member-access': ITER_10 + WARMUP_2 + ['10'],
138+
'magic-bool-sized': ITER_10 + WARMUP_2 + ['1_000_000'],
139+
'magic-iter': ITER_10 + WARMUP_2 + ['250000'],
140+
'call-classmethod-sized': ITER_10 + WARMUP_2 + ['1_000_000'],
141+
'mmap-anonymous-sized': ITER_10 + WARMUP_2 + ['1_000'],
142+
'mmap-file': ITER_10 + WARMUP_2 + ['100'],
143+
'generate-functions-sized': ITER_10 + WARMUP_2 + ['1_000_000'],
144+
'try-except-simple': ITER_10 + WARMUP_2 + ['10_000_000'],
145+
'try-except-store-simple': ITER_10 + WARMUP_2 + ['1_000_000'],
146+
'try-except-store-two-types': ITER_10 + WARMUP_2 + ['1_000_000'],
147+
'try-except-two-types': ITER_10 + WARMUP_2 + ['1_000_000'],
148+
'tuple-indexing-from-constructor': ITER_10 + WARMUP_2 + ['100_000'],
149+
'tuple-indexing-from-literal': ITER_10 + WARMUP_2 + ['1_000_000'],
150+
}
104151

105152
def _pickling_benchmarks(module='pickle'):
106153
return {
@@ -179,6 +226,15 @@ def _pickling_benchmarks(module='pickle'):
179226
}
180227

181228

229+
MESO_BENCHMARKS_SMALL = {
230+
'pads-eratosthenes': ITER_10 + WARMUP_2 + ['2000'],
231+
'richards3': ITER_10 + WARMUP_2 + ['2'],
232+
'chaos': ITER_10 + WARMUP_2 + ['2'],
233+
'image-magix': ITER_10 + WARMUP_2 + ['100'],
234+
'raytrace-simple': ITER_10 + WARMUP_2 + ['110', '110'],
235+
}
236+
237+
182238
MACRO_BENCHMARKS = {
183239
'gcbench': ITER_10 + ['10'],
184240
}
@@ -242,6 +298,8 @@ def _pickling_benchmarks(module='pickle'):
242298
"meso": [PATH_MESO, MESO_BENCHMARKS],
243299
"macro": [PATH_MACRO, MACRO_BENCHMARKS],
244300
"interop": [PATH_INTEROP, INTEROP_BENCHMARKS],
301+
"micro-small": [PATH_MICRO, MICRO_BENCHMARKS_SMALL],
302+
"meso-small": [PATH_MESO, MESO_BENCHMARKS_SMALL],
245303
}
246304

247305
WARMUP_BENCHMARKS = {

‎mx.graalpython/mx_graalpython_benchmark.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,11 @@
5858
SUBGROUP_GRAAL_PYTHON = "graalpython"
5959
PYTHON_VM_REGISTRY_NAME = "Python"
6060
CONFIGURATION_DEFAULT = "default"
61+
CONFIGURATION_INTERPRETER = "interpreter"
62+
CONFIGURATION_NATIVE_INTERPRETER = "native-interpreter"
6163
CONFIGURATION_DEFAULT_MULTI = "default-multi"
64+
CONFIGURATION_INTERPRETER_MULTI = "interpreter-multi"
65+
CONFIGURATION_NATIVE_INTERPRETER_MULTI = "native-interpreter-multi"
6266
CONFIGURATION_DEFAULT_MULTI_TIER = "default-multi-tier"
6367
CONFIGURATION_NATIVE = "native"
6468
CONFIGURATION_NATIVE_MULTI = "native-multi"
@@ -334,7 +338,7 @@ def run(self, cwd, args):
334338
if len(cp) > 0:
335339
extra_polyglot_args.append("--vm.classpath="+":".join(cp))
336340

337-
return host_vm.run_lang('graalpython', extra_polyglot_args + args, cwd)
341+
return host_vm.run_launcher('graalpython', extra_polyglot_args + args, cwd)
338342

339343
# Otherwise, we're running from the source tree
340344
truffle_options = [
@@ -790,7 +794,7 @@ def createCommandLineArgs(self, benchmarks, bmSuiteArgs):
790794
if mx.suite("tools", fatalIfMissing=False):
791795
dists.extend(('CHROMEINSPECTOR', 'TRUFFLE_PROFILER'))
792796
if mx.suite("sulong", fatalIfMissing=False):
793-
dists.append('SULONG')
797+
dists.append('SULONG_NATIVE')
794798
if mx.suite("sulong-managed", fatalIfMissing=False):
795799
dists.append('SULONG_MANAGED')
796800

0 commit comments

Comments
 (0)