Skip to content

Commit dbb9ace

Browse files
qunaibitqunaibit
committed
[GR-20838] Add Java Python Interop benchmarks
1 parent 83bd222 commit dbb9ace

File tree

10 files changed

+1261
-50
lines changed

10 files changed

+1261
-50
lines changed

‎graalpython/com.oracle.graal.python.benchmarks/java/com/oracle/graal/python/benchmarks/interop/BenchOutputFormat.java

Lines changed: 348 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,348 @@
1+
/*
2+
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* The Universal Permissive License (UPL), Version 1.0
6+
*
7+
* Subject to the condition set forth below, permission is hereby granted to any
8+
* person obtaining a copy of this software, associated documentation and/or
9+
* data (collectively the "Software"), free of charge and under any and all
10+
* copyright rights in the Software, and any and all patent rights owned or
11+
* freely licensable by each licensor hereunder covering either (i) the
12+
* unmodified Software as contributed to or provided by such licensor, or (ii)
13+
* the Larger Works (as defined below), to deal in both
14+
*
15+
* (a) the Software, and
16+
*
17+
* (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
18+
* one is included with the Software each a "Larger Work" to which the Software
19+
* is contributed by such licensors),
20+
*
21+
* without restriction, including without limitation the rights to copy, create
22+
* derivative works of, display, perform, and distribute the Software and make,
23+
* use, sell, offer for sale, import, export, have made, and have sold the
24+
* Software and the Larger Work(s), and to sublicense the foregoing rights on
25+
* either these or other terms.
26+
*
27+
* This license is subject to the following condition:
28+
*
29+
* The above copyright notice and either this complete permission notice or at a
30+
* minimum a reference to the UPL must be included in all copies or substantial
31+
* portions of the Software.
32+
*
33+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
38+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39+
* SOFTWARE.
40+
*/
41+
package com.oracle.graal.python.benchmarks.interop;
42+
43+
import java.io.IOException;
44+
import java.io.PrintStream;
45+
import java.util.ArrayList;
46+
import java.util.Arrays;
47+
import java.util.Collection;
48+
import java.util.HashMap;
49+
50+
import org.openjdk.jmh.infra.BenchmarkParams;
51+
import org.openjdk.jmh.infra.IterationParams;
52+
import org.openjdk.jmh.results.BenchmarkResult;
53+
import org.openjdk.jmh.results.IterationResult;
54+
import org.openjdk.jmh.results.RunResult;
55+
import org.openjdk.jmh.runner.format.OutputFormat;
56+
import org.openjdk.jmh.runner.options.VerboseMode;
57+
import org.openjdk.jmh.util.Utils;
58+
59+
public class BenchOutputFormat implements OutputFormat {
60+
61+
final VerboseMode verbose;
62+
final PrintStream out;
63+
64+
final String name;
65+
66+
final HashMap<String, ArrayList<Double>> raws;
67+
68+
public BenchOutputFormat(PrintStream out, VerboseMode verbose, String name) {
69+
this.out = out;
70+
this.verbose = verbose;
71+
this.raws = new HashMap<>();
72+
this.name = name;
73+
}
74+
75+
public BenchOutputFormat(PrintStream out, VerboseMode verbose) {
76+
this(out, verbose, null);
77+
}
78+
79+
private void add(BenchmarkParams params, double value) {
80+
if (!raws.containsKey(params.getBenchmark())) {
81+
raws.put(params.getBenchmark(), new ArrayList<>());
82+
}
83+
raws.get(params.getBenchmark()).add(value);
84+
}
85+
86+
protected String benchName(BenchmarkParams params) {
87+
if (name != null) {
88+
return name;
89+
}
90+
return params.getBenchmark();
91+
}
92+
93+
@Override
94+
public void startBenchmark(BenchmarkParams params) {
95+
String opts = Utils.join(params.getJvmArgs(), " ");
96+
if (opts.trim().isEmpty()) {
97+
opts = "<none>";
98+
}
99+
100+
println("# JMH version: " + params.getJmhVersion());
101+
println("# VM version: JDK " + params.getJdkVersion() + ", " + params.getVmName() + ", " + params.getVmVersion());
102+
103+
println("# VM invoker: " + params.getJvm());
104+
println("# VM options: " + opts);
105+
println("# Benchmark mode: " + params.getMode().longLabel());
106+
107+
hline();
108+
String benchName = benchName(params);
109+
IterationParams warmup = params.getWarmup();
110+
IterationParams measurement = params.getMeasurement();
111+
String info = "### %s, %d warmup iterations, %d bench iterations";
112+
println(String.format(info, benchName, warmup.getCount(), measurement.getCount()));
113+
String s = "";
114+
boolean isFirst = true;
115+
for (String k : params.getParamsKeys()) {
116+
if (isFirst) {
117+
isFirst = false;
118+
} else {
119+
s += ", ";
120+
}
121+
s += k + " = " + params.getParam(k);
122+
}
123+
124+
println(String.format("### args = [%s]", s));
125+
hline();
126+
println("### start benchmark ...");
127+
}
128+
129+
@Override
130+
public void iteration(BenchmarkParams benchmarkParams, IterationParams params, int iteration) {
131+
}
132+
133+
@Override
134+
public void iterationResult(BenchmarkParams benchmParams, IterationParams params, int iteration, IterationResult data) {
135+
double value = data.getPrimaryResult().getScore();
136+
add(benchmParams, value);
137+
String benchName = benchName(benchmParams);
138+
switch (params.getType()) {
139+
case WARMUP:
140+
out.println(String.format("### (pre)warming up for %s iteration=%d, duration=%.3f", benchName, iteration, value));
141+
break;
142+
case MEASUREMENT:
143+
out.println(String.format("### iteration=%d, name=%s, duration=%.3f", iteration, benchName, value));
144+
break;
145+
default:
146+
throw new IllegalStateException("Unknown iteration type: " + params.getType());
147+
}
148+
}
149+
150+
@Override
151+
public void endBenchmark(BenchmarkResult result) {
152+
final ArrayList<Double> raw = raws.get(result.getParams().getBenchmark());
153+
final double[] durations = new double[raw.size()];
154+
for (int i = 0; i < raw.size(); i++) {
155+
durations[i] = raw.get(i);
156+
}
157+
hline();
158+
println("### teardown ...");
159+
println("### benchmark complete");
160+
hline();
161+
double best = min(durations);
162+
println(String.format("### BEST duration: %.3f s", best));
163+
double worst = max(durations);
164+
println(String.format("### WORST duration: %.3f s", worst));
165+
double avg = avg(durations);
166+
println(String.format("### AVG (all runs) duration: %.3f s", avg));
167+
int warmupIndex = detect_warmup(durations);
168+
println(String.format("### WARMUP detected at iteration: %d", warmupIndex));
169+
double avg2 = avg(Arrays.copyOfRange(durations, Math.min(durations.length - 1, warmupIndex + 1), durations.length));
170+
println(String.format("### AVG (no warmup) duration: %.3f s", avg2));
171+
hline();
172+
String s = "";
173+
for (double d : durations) {
174+
s += d + ", ";
175+
}
176+
println(String.format("### RAW DURATIONS: [%s]", s));
177+
hline();
178+
}
179+
180+
private static int[] cusum(double[] values, double threshold) {
181+
// double threshold=1.0;
182+
double drift = 0.0;
183+
int size = values.length;
184+
double[] csum_pos = new double[size];
185+
double[] csum_neg = new double[size];
186+
int[] change_points = new int[size];
187+
int cp_idx = -1;
188+
for (int i = 1; i < size; i++) {
189+
double diff = values[i] - values[i - 1];
190+
csum_pos[i] = csum_pos[i - 1] + diff - drift;
191+
csum_neg[i] = csum_neg[i - 1] - diff - drift;
192+
193+
if (csum_pos[i] < 0) {
194+
csum_pos[i] = 0;
195+
}
196+
if (csum_neg[i] < 0) {
197+
csum_neg[i] = 0;
198+
}
199+
200+
if (csum_pos[i] > threshold || csum_neg[i] > threshold) {
201+
cp_idx++;
202+
change_points[cp_idx] = i;
203+
csum_pos[i] = 0.;
204+
csum_neg[i] = 0.;
205+
}
206+
}
207+
208+
return Arrays.copyOf(change_points, cp_idx + 1);
209+
210+
}
211+
212+
private static double avg(double[] values) {
213+
return Arrays.stream(values).average().getAsDouble();
214+
}
215+
216+
private static double min(double[] values) {
217+
return Arrays.stream(values).min().getAsDouble();
218+
}
219+
220+
private static double max(double[] values) {
221+
return Arrays.stream(values).max().getAsDouble();
222+
}
223+
224+
private static double[] norm(double[] values) {
225+
double min = min(values);
226+
double max = max(values);
227+
return Arrays.stream(values).map(v -> (v - min) / (max - min) * 100.0).toArray();
228+
}
229+
230+
private static double[] pairwise_slopes(double[] values, int[] cp) {
231+
double[] copy = Arrays.copyOf(values, values.length - 1);
232+
for (int i = 0; i < copy.length; i++) {
233+
copy[i] = Math.abs((values[i + 1] - values[i]) / (cp[i + 1] - cp[i]));
234+
}
235+
return copy;
236+
}
237+
238+
private static double[] last_n_percent_runs(double[] values, double n) {
239+
int size = values.length;
240+
int newSize = size - (int) (size * n);
241+
if (newSize >= size) {
242+
newSize = size - 1;
243+
}
244+
return Arrays.copyOfRange(values, newSize, size);
245+
}
246+
247+
private static int warmup(int idx, int[] cp, int max) {
248+
return cp[idx] < max ? cp[idx] : -1;
249+
}
250+
251+
private static int detect_warmup(double[] durations) {
252+
double cp_threshold = 0.03, stability_slope_grade = 0.01;
253+
stability_slope_grade *= 100.0;
254+
cp_threshold *= 100;
255+
double[] values = norm(durations);
256+
int size = values.length;
257+
int[] cp = cusum(values, cp_threshold);
258+
double[] rolling_avg = new double[cp.length];
259+
for (int i = 0; i < cp.length; i++) {
260+
// [avg(values[i:]) for i in cp]
261+
rolling_avg[i] = avg(Arrays.copyOfRange(values, cp[i], values.length));
262+
}
263+
264+
// find the point where the duration avg is below the cp threshold
265+
for (int i = 0; i < rolling_avg.length; i++) {
266+
if (rolling_avg[i] <= cp_threshold) {
267+
return warmup(i, cp, size);
268+
}
269+
}
270+
271+
// could not find something below the CP threshold (noise in the data), use the
272+
// stabilisation of slopes
273+
double n = 0.1;
274+
double[] last_n_vals = last_n_percent_runs(values, n);
275+
int last_n_idx = size - (int) (size * n);
276+
int totalSize = cp.length + last_n_vals.length;
277+
double[] rolling_avg2 = new double[totalSize];
278+
int[] cp2 = new int[totalSize];
279+
for (int i = 0; i < totalSize; i++) {
280+
if (i < cp.length) {
281+
rolling_avg2[i] = rolling_avg[i];
282+
cp2[i] = cp[i];
283+
} else {
284+
int j = i - cp.length;
285+
rolling_avg2[i] = last_n_vals[j];
286+
cp2[i] = last_n_idx++;
287+
}
288+
}
289+
double[] slopes = pairwise_slopes(rolling_avg2, cp2);
290+
291+
for (int i = 0; i < slopes.length; i++) {
292+
if (slopes[i] <= stability_slope_grade) {
293+
return warmup(i, cp, size);
294+
}
295+
}
296+
297+
return -1;
298+
}
299+
300+
private void hline() {
301+
println("-------------------------------------------------------------------------------");
302+
}
303+
304+
@Override
305+
public void print(String s) {
306+
out.print(s);
307+
}
308+
309+
@Override
310+
public void println(String s) {
311+
out.println(s);
312+
}
313+
314+
@Override
315+
public void flush() {
316+
out.flush();
317+
}
318+
319+
@Override
320+
public void verbosePrintln(String s) {
321+
if (verbose == VerboseMode.EXTRA) {
322+
out.println(s);
323+
}
324+
}
325+
326+
@Override
327+
public void write(int b) {
328+
out.write(b);
329+
}
330+
331+
@Override
332+
public void write(byte[] b) throws IOException {
333+
out.write(b);
334+
}
335+
336+
@Override
337+
public void close() {
338+
}
339+
340+
@Override
341+
public void startRun() {
342+
}
343+
344+
@Override
345+
public void endRun(Collection<RunResult> runResults) {
346+
}
347+
348+
}

0 commit comments

Comments
 (0)