Skip to content

Commit 2ffb430

Browse files
DSouzaMDSouzaM
committed
Add CalculatorBenchmark and PEBenchmark; parametrize Bytecode DSL benchmarks over a "benchmark spec"
1 parent 3fab0c0 commit 2ffb430

21 files changed

+2180
-363
lines changed

‎truffle/mx.truffle/mx_truffle.py‎

Lines changed: 82 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,10 +90,91 @@ def subgroup(self):
9090

9191
def extraVmArgs(self):
9292
extraVmArgs = super(JMHRunnerTruffleBenchmarkSuite, self).extraVmArgs()
93-
# com.oracle.truffle.api.benchmark.InterpreterCallBenchmark$BenchmarkState needs DefaultTruffleRuntime
93+
# org.graalvm.truffle.benchmark.InterpreterCallBenchmark$BenchmarkState needs DefaultTruffleRuntime
9494
extraVmArgs.append('--add-exports=org.graalvm.truffle/com.oracle.truffle.api.impl=ALL-UNNAMED')
95+
# org.graalvm.truffle.compiler.benchmark.* needs OptimizedTruffleRuntime
96+
extraVmArgs.append('--add-exports=org.graalvm.truffle.runtime/com.oracle.truffle.runtime=ALL-UNNAMED')
9597
return extraVmArgs
9698

99+
def rules(self, out, benchmarks, bmSuiteArgs):
100+
result = super().rules(out, benchmarks, bmSuiteArgs)
101+
result_file = self.get_jmh_result_file(bmSuiteArgs)
102+
suite_name = self.benchSuiteName(bmSuiteArgs)
103+
result.extend([
104+
JMHJsonCompilationTimingRule(result_file, suite_name, "pe-time"),
105+
JMHJsonCompilationTimingRule(result_file, suite_name, "compile-time"),
106+
JMHJsonCompilationTimingRule(result_file, suite_name, "code-install-time"),
107+
])
108+
return result
109+
110+
class JMHJsonCompilationTimingRule(mx_benchmark.JMHJsonRule):
111+
def __init__(self, filename, suite_name, metric_name):
112+
super().__init__(filename, suite_name)
113+
self.metric_name = metric_name
114+
115+
def parse(self, text):
116+
r = []
117+
with open(self._prepend_working_dir(self.filename)) as fp:
118+
for result in json.load(fp):
119+
benchmark = self.getBenchmarkNameFromResult(result)
120+
metric = result.get("secondaryMetrics", {}).get(self.metric_name)
121+
if metric is None:
122+
return []
123+
124+
unit = JMHJsonCompilationTimingRule.standardize_unit(metric["scoreUnit"])
125+
126+
template = {
127+
"bench-suite" : self.suiteName,
128+
"benchmark" : self.shortenPackageName(benchmark),
129+
"metric.unit": unit,
130+
"metric.score-function": "id",
131+
"metric.better": "lower",
132+
"metric.type": "numeric",
133+
# full name
134+
"extra.jmh.benchmark" : benchmark,
135+
}
136+
137+
if "params" in result:
138+
# add all parameter as a single string
139+
template["extra.jmh.params"] = ", ".join(["=".join(kv) for kv in result["params"].items()])
140+
# and also the individual values
141+
for k, v in result["params"].items():
142+
template["extra.jmh.param." + k] = str(v)
143+
144+
for k in self.getExtraJmhKeys():
145+
extra_value = None
146+
if k in result:
147+
extra_value = result[k]
148+
template["extra.jmh." + k] = str(extra_value)
149+
150+
151+
summary_datapoint = template.copy()
152+
summary_datapoint.update({
153+
"metric.name": self.metric_name,
154+
"metric.value": float(metric["score"])
155+
})
156+
r.append(summary_datapoint)
157+
158+
score_percentiles = metric.get("scorePercentiles")
159+
if score_percentiles:
160+
distribution_metric_name = f"{self.metric_name}-distribution"
161+
for percentile, score in score_percentiles.items():
162+
percentile_datapoint = template.copy()
163+
percentile_datapoint.update({
164+
"metric.name": distribution_metric_name,
165+
"metric.value": float(score),
166+
"metric.percentile": float(percentile)
167+
})
168+
r.append(percentile_datapoint)
169+
return r
170+
171+
@staticmethod
172+
def standardize_unit(unit: str) -> str:
173+
if unit.endswith("/op"):
174+
# JMH emits timings "per operation", e.g., "ms/op". Drop the suffix.
175+
return unit[:-len("/op")]
176+
return unit
177+
97178
mx_benchmark.add_bm_suite(JMHRunnerTruffleBenchmarkSuite())
98179
#mx_benchmark.add_java_vm(mx_benchmark.DefaultJavaVm("server", "default"), priority=3)
99180

‎truffle/src/org.graalvm.truffle.benchmark/src/org/graalvm/truffle/benchmark/bytecode_dsl/AbstractBytecodeBenchmark.java‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*
2+
* Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* The Universal Permissive License (UPL), Version 1.0
6+
*
7+
* Subject to the condition set forth below, permission is hereby granted to any
8+
* person obtaining a copy of this software, associated documentation and/or
9+
* data (collectively the "Software"), free of charge and under any and all
10+
* copyright rights in the Software, and any and all patent rights owned or
11+
* freely licensable by each licensor hereunder covering either (i) the
12+
* unmodified Software as contributed to or provided by such licensor, or (ii)
13+
* the Larger Works (as defined below), to deal in both
14+
*
15+
* (a) the Software, and
16+
*
17+
* (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
18+
* one is included with the Software each a "Larger Work" to which the Software
19+
* is contributed by such licensors),
20+
*
21+
* without restriction, including without limitation the rights to copy, create
22+
* derivative works of, display, perform, and distribute the Software and make,
23+
* use, sell, offer for sale, import, export, have made, and have sold the
24+
* Software and the Larger Work(s), and to sublicense the foregoing rights on
25+
* either these or other terms.
26+
*
27+
* This license is subject to the following condition:
28+
*
29+
* The above copyright notice and either this complete permission notice or at a
30+
* minimum a reference to the UPL must be included in all copies or substantial
31+
* portions of the Software.
32+
*
33+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
38+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39+
* SOFTWARE.
40+
*/
41+
package org.graalvm.truffle.benchmark.bytecode_dsl;
42+
43+
import org.graalvm.truffle.benchmark.TruffleBenchmark;
44+
import org.graalvm.truffle.benchmark.bytecode_dsl.specs.BenchmarkSpec;
45+
import org.openjdk.jmh.annotations.Level;
46+
import org.openjdk.jmh.annotations.Setup;
47+
import org.openjdk.jmh.infra.BenchmarkParams;
48+
49+
public abstract class AbstractBytecodeBenchmark extends TruffleBenchmark {
50+
51+
private static final boolean CHECK_RESULTS = System.getProperty("CheckResults") != null;
52+
53+
@Setup(Level.Trial)
54+
public void abstractSetup() {
55+
if (CHECK_RESULTS) {
56+
// Checkstyle: stop benchmark debug logging
57+
System.err.println(
58+
"-DCheckResults was set. Output will be validated after each iteration if the benchmark invokes checkExpectedResult. This property should be unset when measuring actual benchmark numbers.");
59+
// Checkstyle: resume
60+
}
61+
}
62+
63+
/**
64+
* Returns the name of the current benchmark spec class.
65+
*/
66+
protected abstract String getBenchmarkSpecClassName();
67+
68+
/**
69+
* Helper for checking the expected result, when result checking is enabled.
70+
*/
71+
protected final void checkExpectedResult(Object result, String benchMethod) {
72+
if (CHECK_RESULTS) {
73+
BenchmarkSpec spec = getBenchmarkSpec();
74+
if (!spec.expectedResult().equals(result)) {
75+
throw new AssertionError(getBenchmarkSpecClassName() + ":" + benchMethod + " produced the wrong result. Received " + result + " but expected " + spec.expectedResult());
76+
}
77+
}
78+
}
79+
80+
/**
81+
* Returns the current benchmark spec.
82+
*/
83+
protected BenchmarkSpec getBenchmarkSpec() {
84+
String benchmarkSpecClassName = getBenchmarkSpecClassName();
85+
String fullClassName = String.join(".", AbstractBytecodeBenchmark.class.getPackageName(), "specs", benchmarkSpecClassName);
86+
try {
87+
return (BenchmarkSpec) Class.forName(fullClassName).getDeclaredConstructor().newInstance();
88+
} catch (ReflectiveOperationException ex) {
89+
throw new IllegalArgumentException(String.format("Bad benchmark spec %s: class %s could not be reflectively instantiated.", benchmarkSpecClassName, fullClassName), ex);
90+
}
91+
}
92+
93+
/**
94+
* Returns the name of the method currently being benchmarked by JMH.
95+
*/
96+
protected String getBenchmarkMethod(BenchmarkParams params) {
97+
String[] parts = params.getBenchmark().split("\\.");
98+
return parts[parts.length - 1];
99+
}
100+
101+
}

‎truffle/src/org.graalvm.truffle.benchmark/src/org/graalvm/truffle/benchmark/bytecode_dsl/BenchmarkLanguage.java‎

Lines changed: 11 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -40,14 +40,11 @@
4040
*/
4141
package org.graalvm.truffle.benchmark.bytecode_dsl;
4242

43-
import java.util.HashMap;
44-
import java.util.Map;
45-
import java.util.function.Function;
43+
import java.lang.reflect.Method;
4644

47-
import org.graalvm.polyglot.Context;
48-
import org.graalvm.polyglot.Source;
45+
import org.graalvm.truffle.benchmark.bytecode_dsl.manual.BaseBytecodeRootNode;
46+
import org.graalvm.truffle.benchmark.bytecode_dsl.manual.Builder;
4947

50-
import com.oracle.truffle.api.CallTarget;
5148
import com.oracle.truffle.api.TruffleLanguage;
5249
import com.oracle.truffle.api.TruffleLanguage.Registration;
5350
import com.oracle.truffle.api.bytecode.BytecodeConfig;
@@ -59,45 +56,28 @@
5956

6057
/**
6158
* Basic language used for benchmarks.
62-
* <p>
63-
* Before execution, languages should use {@link #registerName} to map a benchmark name to a parsing
64-
* function. Then, {@link Context#eval eval} a {@link Source} with the same benchmark name to parse
65-
* a given benchmark.
6659
*/
6760
@Registration(id = "bm", name = "bm")
6861
@ProvidedTags({RootTag.class, RootBodyTag.class})
6962
public class BenchmarkLanguage extends TruffleLanguage<Object> {
7063

71-
private static final Map<String, Function<BenchmarkLanguage, CallTarget>> NAMES = new HashMap<>();
72-
7364
@Override
7465
protected Object createContext(Env env) {
7566
return new Object();
7667
}
7768

78-
public static void registerName(String name, Class<? extends BytecodeDSLBenchmarkRootNode> cls, BytecodeParser<BytecodeDSLBenchmarkRootNodeBuilder> parser) {
79-
registerName(name, l -> {
80-
BytecodeRootNodes<BytecodeDSLBenchmarkRootNode> nodes = createNodes(cls, l, parser);
81-
return nodes.getNode(nodes.count() - 1).getCallTarget();
82-
});
83-
}
84-
85-
private static BytecodeRootNodes<BytecodeDSLBenchmarkRootNode> createNodes(Class<? extends BytecodeDSLBenchmarkRootNode> interpreterClass,
69+
public static BytecodeRootNodes<BytecodeDSLBenchmarkRootNode> createBytecodeDSLNodes(Class<? extends BytecodeDSLBenchmarkRootNode> interpreterClass,
8670
BenchmarkLanguage language, BytecodeParser<BytecodeDSLBenchmarkRootNodeBuilder> builder) {
8771
return BytecodeDSLBenchmarkRootNodeBuilder.invokeCreate(interpreterClass, language, BytecodeConfig.DEFAULT, builder);
8872
}
8973

90-
public static void registerName(String name, Function<BenchmarkLanguage, CallTarget> parser) {
91-
NAMES.put(name, parser);
92-
}
93-
94-
@Override
95-
protected CallTarget parse(ParsingRequest request) throws Exception {
96-
String name = request.getSource().getCharacters().toString();
97-
if (!NAMES.containsKey(name)) {
98-
throw new AssertionError("source not registered: " + name);
74+
public static BaseBytecodeRootNode createBytecodeNodes(Class<? extends BaseBytecodeRootNode> interpreterClass,
75+
BenchmarkLanguage language, Builder builder) {
76+
try {
77+
Method method = interpreterClass.getMethod("create", BenchmarkLanguage.class, Builder.class);
78+
return (BaseBytecodeRootNode) method.invoke(null, language, builder);
79+
} catch (ReflectiveOperationException ex) {
80+
throw new IllegalArgumentException(String.format("Bad interpreter class %s: could not reflectively invoke create method.", interpreterClass.getName()), ex);
9981
}
100-
101-
return NAMES.get(name).apply(this);
10282
}
10383
}

‎truffle/src/org.graalvm.truffle.benchmark/src/org/graalvm/truffle/benchmark/bytecode_dsl/BytecodeDSLBenchmarkRootNode.java‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,9 @@
4040
*/
4141
package org.graalvm.truffle.benchmark.bytecode_dsl;
4242

43+
import com.oracle.truffle.api.CompilerDirectives;
4344
import com.oracle.truffle.api.bytecode.BytecodeRootNode;
45+
import com.oracle.truffle.api.bytecode.ConstantOperand;
4446
import com.oracle.truffle.api.bytecode.GenerateBytecode;
4547
import com.oracle.truffle.api.bytecode.GenerateBytecodeTestVariants;
4648
import com.oracle.truffle.api.bytecode.GenerateBytecodeTestVariants.Variant;
@@ -69,6 +71,50 @@ static int doInts(int left, int right) {
6971
}
7072
}
7173

74+
@Operation
75+
@ConstantOperand(type = int.class)
76+
static final class AddConst {
77+
@Specialization
78+
static int doInt(int c, int operand) {
79+
return c + operand;
80+
}
81+
}
82+
83+
@Operation
84+
static final class Mult {
85+
@Specialization
86+
static int doInt(int left, int right) {
87+
return left * right;
88+
}
89+
}
90+
91+
@Operation
92+
@ConstantOperand(type = int.class)
93+
static final class MultConst {
94+
@Specialization
95+
static int doInt(int c, int operand) {
96+
return c * operand;
97+
}
98+
}
99+
100+
@Operation
101+
@ConstantOperand(type = int.class, specifyAtEnd = true)
102+
static final class DivConst {
103+
@Specialization
104+
static int doInt(int operand, int c) {
105+
return operand / c;
106+
}
107+
}
108+
109+
@Operation
110+
@ConstantOperand(type = int.class)
111+
static final class EqConst {
112+
@Specialization
113+
static boolean doInt(int c, int operand) {
114+
return c == operand;
115+
}
116+
}
117+
72118
@Operation
73119
static final class Mod {
74120
@Specialization
@@ -85,4 +131,28 @@ static boolean doInts(int left, int right) {
85131
}
86132
}
87133

134+
@Operation
135+
static final class ArrayLength {
136+
@Specialization
137+
static int doArray(int[] array) {
138+
return array.length;
139+
}
140+
}
141+
142+
@Operation
143+
static final class ArrayIndex {
144+
@Specialization
145+
static int doArray(int[] array, int i) {
146+
return array[i];
147+
}
148+
}
149+
150+
@Operation
151+
static final class Unreachable {
152+
@Specialization
153+
static void assertUnreachable() {
154+
CompilerDirectives.shouldNotReachHere();
155+
}
156+
}
157+
88158
}

0 commit comments

Comments
 (0)