Introduce Summary interface and implementations to Polybench harness.

Author: Andrija Kolic

truffle/mx.truffle/mx_polybench/model.py

@@ -520,7 +520,8 @@ def rules(self, output, benchmarks, bmSuiteArgs):
         if metric_name == "time":
             # For metric "time", two metrics are reported:
             # - "warmup" (per-iteration data for "warmup" and "run" iterations)
-            # - "time" (per-iteration data for only the "run" iterations)
+            # - "time-sample" (per-iteration data for only the "run" iterations)
+            # - "time" (aggregation of per-iteration data for the "run" iterations after outlier removal)
             rules += [
                 mx_benchmark.StdOutRule(
                     r"\[.*\] iteration ([0-9]*): (?P<value>.*) (?P<unit>.*)",
@@ -540,7 +541,7 @@ def rules(self, output, benchmarks, bmSuiteArgs):
                     {
                         "benchmark": benchmark_name,
                         "metric.better": "lower",
-                        "metric.name": "time",
+                        "metric.name": "time-sample",
                         "metric.unit": ("<unit>", str),
                         "metric.value": ("<value>", float),
                         "metric.type": "numeric",
@@ -549,6 +550,19 @@ def rules(self, output, benchmarks, bmSuiteArgs):
                     },
                     startPattern=r"::: Running :::",
                 ),
+                ExcludeWarmupRule(
+                    r"\[.*\] run aggregate summary: (?P<value>.*) (?P<unit>.*)",
+                    {
+                        "benchmark": benchmark_name,
+                        "metric.better": "lower",
+                        "metric.name": "time",
+                        "metric.unit": ("<unit>", str),
+                        "metric.value": ("<value>", float),
+                        "metric.type": "numeric",
+                        "metric.score-function": "id",
+                    },
+                    startPattern=r"::: Running :::",
+                ),
             ]
         elif metric_name in ("allocated-memory", "metaspace-memory", "application-memory", "instructions"):
             rules += [

truffle/src/org.graalvm.polybench/src/org/graalvm/polybench/AverageSummary.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.polybench;
+
+import java.util.Optional;
+
+/**
+ * Summarizes the results of a polybench benchmark with an average value computed from iteration
+ * datapoints.
+ */
+class AverageSummary implements Summary {
+    @Override
+    public Optional<Double> postprocess(double[] results) {
+        double sum = 0;
+        for (Double result : results) {
+            sum += result;
+        }
+        return results.length > 0 ? Optional.of(sum / results.length) : Optional.empty();
+    }
+
+    @Override
+    public String toString() {
+        return "AverageSummary{}";
+    }
+}

truffle/src/org.graalvm.polybench/src/org/graalvm/polybench/Config.java

@@ -26,6 +26,7 @@
 
 import org.graalvm.polyglot.Value;
 
+import java.io.InvalidObjectException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -44,6 +45,7 @@ public class Config {
     int iterations;
     Metric metric;
     boolean evalSourceOnlyDefault;
+    Summary summary;
 
     final List<String> unrecognizedArguments = new ArrayList<>();
 
@@ -79,7 +81,7 @@ public MultiEngineConfig initMultiEngine() {
         return multiEngine;
     }
 
-    public void parseBenchSpecificDefaults(Value benchmark) {
+    public void parseBenchSpecificDefaults(Value benchmark) throws InvalidObjectException {
         if (warmupIterations == UNINITIALIZED_ITERATIONS) {
             if (benchmark.hasMember("warmupIterations")) {
                 Value warmupIterationsMember = benchmark.getMember("warmupIterations");
@@ -96,6 +98,36 @@ public void parseBenchSpecificDefaults(Value benchmark) {
                 iterations = DEFAULT_ITERATIONS;
             }
         }
+        parseBenchSpecificSummary(benchmark);
+    }
+
+    private void parseBenchSpecificSummary(Value benchmark) throws InvalidObjectException {
+        if (!benchmark.hasMember("summary")) {
+            // No 'summary' member provided in the benchmark
+            return;
+        }
+        Value summaryMember = benchmark.getMember("summary");
+        if (summaryMember.canExecute()) {
+            summaryMember = summaryMember.execute();
+        }
+        if (!summaryMember.hasMember("get")) {
+            throw new InvalidObjectException("Failed at parsing the 'summary' benchmark member due to it missing a 'get' member!");
+        }
+        Value summaryGetMethod = summaryMember.getMember("get");
+        String summaryClassName = summaryGetMethod.execute("name").asString();
+        if (OutlierRemovalAverageSummary.class.getSimpleName().equals(summaryClassName)) {
+            Value lowerThresholdValue = summaryGetMethod.execute("lower-threshold");
+            Value upperThresholdValue = summaryGetMethod.execute("upper-threshold");
+            if (!lowerThresholdValue.fitsInDouble() || !upperThresholdValue.fitsInDouble()) {
+                String msg = "Failed at parsing 'summary' benchmark member with name of '" + summaryClassName + "' due to it missing either a 'lower-threshold' or an 'upper-threshold' member!";
+                throw new InvalidObjectException(msg);
+            }
+            summary = new OutlierRemovalAverageSummary(lowerThresholdValue.asDouble(), upperThresholdValue.asDouble());
+        } else if (AverageSummary.class.getSimpleName().equals(summaryClassName)) {
+            summary = new AverageSummary();
+        } else {
+            throw new InvalidObjectException("Failed at parsing the 'summary' benchmark member due to unrecognized name of '" + summaryClassName + "'!");
+        }
     }
 
     @Override
@@ -107,7 +139,10 @@ public String toString() {
                         "iterations:        " + (iterations == UNINITIALIZED_ITERATIONS ? "default" : iterations + "\n");
         if (multiEngine != null) {
             config += "runs:              " + multiEngine.numberOfRuns + "\n" +
-                            "shared engine:     " + multiEngine.sharedEngine;
+                            "shared engine:     " + multiEngine.sharedEngine + "\n";
+        }
+        if (summary != null) {
+            config += "summary:           " + summary + "\n";
         }
         return config;
     }

truffle/src/org.graalvm.polybench/src/org/graalvm/polybench/OutlierRemovalAverageSummary.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.polybench;
+
+import java.util.Arrays;
+import java.util.Optional;
+
+/**
+ * Summarizes the results of a polybench benchmark with an average of iteration datapoints that fall
+ * between `lowerThreshold` and `upperThreshold` percentiles.
+ */
+class OutlierRemovalAverageSummary extends AverageSummary {
+    private final double lowerThreshold;
+    private final double upperThreshold;
+
+    OutlierRemovalAverageSummary(double lowerThreshold, double upperThreshold) {
+        this.lowerThreshold = lowerThreshold;
+        this.upperThreshold = upperThreshold;
+    }
+
+    @Override
+    public Optional<Double> postprocess(double[] results) {
+        if (results.length == 0) {
+            return Optional.empty();
+        }
+
+        int n = results.length;
+        Arrays.sort(results);
+        int fromIndex = (int) Math.ceil(lowerThreshold * n);
+        int toIndex = (int) Math.floor(upperThreshold * n);
+        if (fromIndex >= toIndex) {
+            return Optional.empty();
+        }
+
+        return super.postprocess(Arrays.copyOfRange(results, fromIndex, toIndex));
+    }
+
+    @Override
+    public String toString() {
+        return "OutlierRemovalAverageSummary{lowerThreshold=" + lowerThreshold + ", upperThreshold=" + upperThreshold + "}";
+    }
+}

truffle/src/org.graalvm.polybench/src/org/graalvm/polybench/PolyBenchLauncher.java

@@ -489,12 +489,12 @@ private void runHarness(Context.Builder contextBuilder, boolean evalSourceOnly,
                 Workload workload = lookup(context, evalResult.languageId, evalResult.value, "run");
 
                 log("::: Running warmup :::");
-                repeatIterations(context, workload, evalResult.sourceName, true, config.warmupIterations);
+                repeatIterations(context, workload, evalResult.sourceName, true, config.warmupIterations, config.summary);
                 log("");
 
                 log("::: Running :::");
                 config.metric.reset();
-                repeatIterations(context, workload, evalResult.sourceName, false, config.iterations);
+                repeatIterations(context, workload, evalResult.sourceName, false, config.iterations, config.summary);
                 log("");
             }
 
@@ -515,9 +515,10 @@ private static String round(double v) {
         return String.format("%.2f", v);
     }
 
-    private void repeatIterations(Context context, Workload workload, String name, boolean warmup, int iterations) {
+    private void repeatIterations(Context context, Workload workload, String name, boolean warmup, int iterations, Summary summary) {
         // Enter explicitly to avoid context switches for each iteration.
         context.enter();
+        double[] iterationResults = new double[iterations];
         try {
             for (int i = 0; i < iterations; i++) {
                 config.metric.beforeIteration(warmup, i, config);
@@ -531,14 +532,19 @@ private void repeatIterations(Context context, Workload workload, String name, b
                 final Optional<Double> value = config.metric.reportAfterIteration(config);
                 if (value.isPresent()) {
                     log("[" + name + "] iteration " + i + ": " + round(value.get()) + " " + config.metric.unit());
+                    iterationResults[i] = value.get();
                 }
             }
 
+            log("------");
             final Optional<Double> value = config.metric.reportAfterAll();
             if (value.isPresent()) {
-                log("------");
                 log("[" + name + "] " + (warmup ? "after warmup: " : "after run: ") + round(value.get()) + " " + config.metric.unit());
             }
+            Optional<Double> summaryAggregate = summary != null ? summary.postprocess(iterationResults) : Optional.empty();
+            if (summaryAggregate.isPresent()) {
+                log("[" + name + "] " + (warmup ? "warmup" : "run") + " aggregate summary: " + round(summaryAggregate.get()) + " " + config.metric.unit());
+            }
         } finally {
             context.leave();
         }

truffle/src/org.graalvm.polybench/src/org/graalvm/polybench/Summary.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.polybench;
+
+import java.util.Optional;
+
+/**
+ * Summarizes the results of a polybench benchmark with an aggregate metric value computed from
+ * iteration datapoints.
+ */
+interface Summary {
+    Optional<Double> postprocess(double[] results);
+}

vm/ci/ci_common/common-bench.jsonnet

@@ -99,11 +99,45 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
   },
 
   build_polybenchmarks: [
-      ['mx', '--env', '${VM_ENV}', '--dy', 'polybenchmarks', 'sforceimports'],
-      ['mx', '-p', '../../polybenchmarks', 'build_benchmarks'],
-      ['mx', '--dy', 'polybenchmarks', 'build', '--dependencies', 'POLYBENCHMARKS_BENCHMARKS']
+    ['mx', '--env', '${VM_ENV}', '--dy', 'polybenchmarks', 'sforceimports'],
+    ['mx', '-p', '../../polybenchmarks', 'build_benchmarks'],
+    ['mx', '--dy', 'polybenchmarks', 'build', '--dependencies', 'POLYBENCHMARKS_BENCHMARKS']
   ],
 
+  polybench_vm_post_merge(os, arch, language, vm_config, name = null): vm_common.vm_base(os, arch, 'post-merge', bench=true) + self.polybench_vm_common(os, arch) + vm.vm_java_Latest + {
+    name: utils.hyphenize(['post-merge-bench-vm', vm.vm_setup.short_name, 'polybench', language, name, vm_config, utils.jdk_and_hardware(self)]),
+    setup+: [
+      ['mx', '--dy', 'graalpython', 'build']
+    ],
+    local benchmarks = ['interpreter/sieve.py', 'interpreter/fibonacci.py', 'interpreter/deltablue.py', 'interpreter/richards.py'],
+    run+: [
+      ['grep', '-q', '-E', 'polybench:' + benchmark + ' ([a-z0-9]|-)+: FAILURE', 'split-run.txt'] +
+      ['||'] +
+      ['hwloc-bind', '--cpubind', 'node:0.core:0-3.pu:0', '--membind', 'node:0', '--'] +
+      ['mx', '--env', '${VM_ENV}', '--dy', '/truffle-enterprise', '--dy', 'graalpython', '--kill-with-sigquit'] +
+      ['--java-home', '${POLYBENCH_JVM}'] +
+      ['benchmark', 'polybench:' + benchmark, '--results-file=' + self.result_file, '--append-results'] +
+      ['--', '--jvm=native-image-java-home', '--jvm-config=' + vm_config] +
+      ['-Dnative-image.benchmark.extra-image-build-argument=-R:MaxHeapSize=500m'] +
+      ['-Dnative-image.benchmark.extra-image-build-argument=-J-Xmx24g'] +
+      ['-Dnative-image.benchmark.benchmark-output-dir=mxbuild'] +
+      ['--split-run', 'split-run.txt'] +
+      ['--', '--experimental-options', '--engine.Compilation=false'] +
+      ['||'] +
+      ['echo', 'polybench:' + benchmark + ' general: FAILURE', '>>', 'split-run.txt']
+      for benchmark in benchmarks
+    ] + [
+      ['set-export', 'BENCH_EXIT_STATUS', '1'],
+      ['cat', 'split-run.txt'],
+      ['grep', '-q', 'FAILURE', 'split-run.txt', '||', 'set-export', 'BENCH_EXIT_STATUS', '0'],
+      ['exit', '$BENCH_EXIT_STATUS']
+    ],
+    teardown: [self.upload],
+    notify_groups +: ['python'],
+    notify_emails+: ['[email protected]'],
+    timelimit: '4:00:00',
+  },
+
   js_bench_compilation_throughput(pgo): self.vm_bench_common + common.heap.default + {
     local mx_libgraal = ["mx", "--env", repo_config.vm.mx_env.libgraal],
 
@@ -275,6 +309,10 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
       ],
       notify_groups +: ['python'],
     }
+  ] + [
+    # PR-bench Python Interpreter mx benchmark polybench jobs
+    self.polybench_vm_post_merge('linux', 'amd64', 'python', vm_config, 'interpreter')
+    for vm_config in ['default', 'g1gc']
   ] + [
     # NFI polybench jobs
     self.polybench_vm_gate('linux', 'amd64', 'nfi') + {

vm/mx.vm/suite.py

@@ -65,7 +65,11 @@
             },
             {
                 "name": "graalpython",
+<<<<<<< HEAD
                 "version": "b283db0979c0d24da6f2ddd55879ab6f93c0fcbd",
+=======
+                "version": "2315bc2d2e070546bf3a356c2ca6e4ba2455c7c8",
+>>>>>>> 3aa5450d718 (Introduce Summary interface and implementations to Polybench harness.)
                 "dynamic": True,
                 "urls": [
                     {"url": "https://github.com/graalvm/graalpython.git", "kind": "git"},