Migrate sulong and wasm to unchained polybench

Author: DSouzaM

sulong/benchmarks/interpreter/Makefile

@@ -0,0 +1,47 @@
+#
+# Copyright (c) 2025, 2025, Oracle and/or its affiliates.
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+# conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list of
+# conditions and the following disclaimer in the documentation and/or other materials provided
+# with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be used to
+# endorse or promote products derived from this software without specific prior written
+# permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+# OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+QUIETLY$(MX_VERBOSE) = @
+
+.PHONY: default
+
+SOURCE_FILES=$(wildcard ${VPATH}/*.c)
+SOURCES=${SOURCE_FILES:${VPATH}/%=%}
+BC_NATIVE_FILES=${SOURCES:%.c=interpreter/%.c.native.bc}
+
+default: ${BC_NATIVE_FILES}
+
+ifndef NATIVE_LLVM_CC
+$(error NATIVE_LLVM_CC not set)
+endif
+
+interpreter/%.c.native.bc: %.c Makefile
+	@mkdir -p $(shell dirname $@)
+	$(QUIETLY) $(NATIVE_LLVM_CC) -o $@ -c $<

vm/benchmarks/interpreter/deltablue.c renamed to sulong/benchmarks/interpreter/deltablue.c

@@ -0,0 +1 @@
+.*

vm/benchmarks/interpreter/fibonacci.c renamed to sulong/benchmarks/interpreter/fibonacci.c

@@ -1,2 +1,6 @@
 mx.sulong/suite.py,no.copyright
 projects/com.oracle.truffle.llvm.libraries.bitcode/src/qsort.c,qsort
+benchmarks/interpreter/fibonacci.c,no.copyright
+benchmarks/interpreter/deltablue.c,no.copyright
+benchmarks/interpreter/richards.c,no.copyright
+benchmarks/interpreter/sieve.c,no.copyright

vm/benchmarks/interpreter/richards.c renamed to sulong/benchmarks/interpreter/richards.c

@@ -398,9 +398,7 @@ def _open_for_reading(path):
 
 
 # Legacy bm suite
-mx_benchmark.add_bm_suite(mx_sulong_benchmarks.SulongBenchmarkSuite(False))
-# Polybench bm suite
-mx_benchmark.add_bm_suite(mx_sulong_benchmarks.SulongBenchmarkSuite(True))
+mx_benchmark.add_bm_suite(mx_sulong_benchmarks.SulongBenchmarkSuite())
 # LLVM unit tests suite
 mx_benchmark.add_bm_suite(mx_sulong_benchmarks.LLVMUnitTestsSuite())
 

vm/benchmarks/interpreter/sieve.c renamed to sulong/benchmarks/interpreter/sieve.c

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2016, 2023, Oracle and/or its affiliates.
+# Copyright (c) 2016, 2025, Oracle and/or its affiliates.
 #
 # All rights reserved.
 #
@@ -35,6 +35,7 @@
 import os
 from os.path import join, exists
 
+import mx_polybench
 import mx_subst
 from mx_benchmark import VmRegistry, java_vm_registry, Vm, GuestVm, VmBenchmarkSuite
 
@@ -63,25 +64,11 @@ def _parse_results_gen():
                     yield r
         return (x for x in _parse_results_gen())
 
-class PolybenchExcludeWarmupRule(mx_benchmark.StdOutRule):
-    """Rule that behaves as the StdOutRule, but skips input until a certain pattern."""
-
-    def __init__(self, *args, **kwargs):
-        self.startPattern = re.compile(kwargs.pop('startPattern'))
-        super(PolybenchExcludeWarmupRule, self).__init__(*args, **kwargs)
-
-    def parse(self, text):
-        m = self.startPattern.search(text)
-        if m:
-            return super(PolybenchExcludeWarmupRule, self).parse(text[m.end()+1:])
-        else:
-            return []
 
 class SulongBenchmarkSuite(VmBenchmarkSuite):
-    def __init__(self, use_polybench, *args, **kwargs):
+    def __init__(self, *args, **kwargs):
         super(SulongBenchmarkSuite, self).__init__(*args, **kwargs)
         self.bench_to_exec = {}
-        self.use_polybench = use_polybench
 
     def group(self):
         return 'Graal'
@@ -90,7 +77,7 @@ def subgroup(self):
         return 'sulong'
 
     def name(self):
-        return 'csuite-polybench' if self.use_polybench else 'csuite'
+        return 'csuite'
 
     def run(self, benchnames, bmSuiteArgs):
         vm = self.get_vm_registry().get_vm_from_suite_args(bmSuiteArgs)
@@ -116,9 +103,6 @@ def run(self, benchnames, bmSuiteArgs):
                 env = vm.prepare_env(env)
 
                 outName = vm.out_file()
-                if self.use_polybench:
-                    env['POLYBENCH'] = 'y'
-                    outName += '.so'
                 out = os.path.join(bench_out_dir, outName)
                 cmdline = ['make', '-f', '../Makefile', out]
                 if mx._opts.verbose:
@@ -155,25 +139,15 @@ def flakySkipPatterns(self, benchmarks, bmSuiteArgs):
         # preparatory benchmark is run using the llimul launcher and passing --multi-context-runs=0.
         # We can capture this argument here and instruct the benchmark infrastructure to ignore
         # the output of this benchmark.
-        if self.use_polybench:
-            if any(a == "store-aux-engine-cache" for a in bmSuiteArgs):
-                return [re.compile(r'.*', re.MULTILINE)]
-        else:
-            if any(a == "--multi-context-runs=0" for a in bmSuiteArgs):
-                return [re.compile(r'.*', re.MULTILINE)]
+        if any(a == "--multi-context-runs=0" for a in bmSuiteArgs):
+            return [re.compile(r'.*', re.MULTILINE)]
         return []
 
     def flakySuccessPatterns(self):
         # bzip2 is known to have a compiler error during OSR compilation, but peak numbers are still valid
         return [re.compile(r'Compilation of sendMTFValues<OSR@\d+> failed')]  # GR-38646
 
     def rules(self, out, benchmarks, bmSuiteArgs):
-        if self.use_polybench:
-            return self.polybenchRules(out, benchmarks, bmSuiteArgs)
-        else:
-            return self.legacyRules(out, benchmarks, bmSuiteArgs)
-
-    def legacyRules(self, out, benchmarks, bmSuiteArgs):
         return [
             SulongBenchmarkRule(
 		r'^run (?P<run>[\d]+) first [\d]+ warmup iterations (?P<benchmark>[\S]+):(?P<line>([ ,]+(?:\d+(?:\.\d+)?))+)',
@@ -243,33 +217,6 @@ def legacyRules(self, out, benchmarks, bmSuiteArgs):
             }),
         ]
 
-    def polybenchRules(self, output, benchmarks, bmSuiteArgs):
-        rules = [
-           mx_benchmark.StdOutRule(r"\[(?P<name>.*)\] iteration ([0-9]*): (?P<value>.*) (?P<unit>.*)", {
-               "bench-suite": "csuite",
-               "benchmark": benchmarks[0],
-               "metric.better": "lower",
-               "metric.name": "warmup",
-               "metric.unit": ("<unit>", str),
-               "metric.value": ("<value>", float),
-               "metric.type": "numeric",
-               "metric.score-function": "id",
-               "metric.iteration": ("$iteration", int),
-           }),
-           PolybenchExcludeWarmupRule(r"\[(?P<name>.*)\] iteration (?P<iteration>[0-9]*): (?P<value>.*) (?P<unit>.*)", {
-               "bench-suite": "csuite",
-               "benchmark": benchmarks[0],
-               "metric.better": "lower",
-               "metric.name": "time",
-               "metric.unit": ("<unit>", str),
-               "metric.value": ("<value>", float),
-               "metric.type": "numeric",
-               "metric.score-function": "id",
-               "metric.iteration": ("<iteration>", int),
-           }, startPattern=r"::: Running :::")
-        ]
-        return rules
-
     def _get_metric_name(self, bmSuiteArgs):
         metric = None
         for arg in bmSuiteArgs:
@@ -292,11 +239,7 @@ def workingDirectory(self, benchmarks, bmSuiteArgs):
         vm = self.get_vm_registry().get_vm_from_suite_args(bmSuiteArgs)
         assert isinstance(vm, CExecutionEnvironmentMixin)
 
-        if self.use_polybench and (any(a == "store-aux-engine-cache" for a in bmSuiteArgs) or any(a == "load-aux-engine-cache" for a in bmSuiteArgs)):
-            # When storing or loading an aux engine cache, the working directory must be the same (the cache for the source is selected by its URL)
-            return join(_benchmarksDirectory(), benchmarks[0], 'aux-engine-cache')
-        else:
-            return join(_benchmarksDirectory(), benchmarks[0], vm.bin_dir())
+        return join(_benchmarksDirectory(), benchmarks[0], vm.bin_dir())
 
     def createCommandLineArgs(self, benchmarks, bmSuiteArgs):
         if len(benchmarks) != 1:
@@ -307,15 +250,14 @@ def createCommandLineArgs(self, benchmarks, bmSuiteArgs):
         vmArgs = self.vmArgs(bmSuiteArgs)
         runArgs = self.runArgs(bmSuiteArgs)
         try:
-            if not self.use_polybench:
-                runArgs += ['--time', str(int(time.clock_gettime(time.CLOCK_REALTIME) * 1000000))]
+            runArgs += ['--time', str(int(time.clock_gettime(time.CLOCK_REALTIME) * 1000000))]
         except:
             # We can end up here in case the python version we're running on doesn't have clock_gettime or CLOCK_REALTIME.
             pass
         return vmArgs + [self.bench_to_exec[benchmarks[0]]] + runArgs
 
     def get_vm_registry(self):
-        return native_polybench_vm_registry if self.use_polybench else native_vm_registry
+        return native_vm_registry
 
 
 class CExecutionEnvironmentMixin(object):
@@ -540,93 +482,6 @@ def hosting_registry(self):
         return java_vm_registry
 
 
-class PolybenchVm(CExecutionEnvironmentMixin, GuestVm):
-
-    def __init__(self, config_name, options, templateOptions, host_vm=None):
-        super(PolybenchVm, self).__init__(host_vm)
-        self._config_name = config_name
-        self._options = options
-        self._templateOptions = templateOptions
-
-    def with_host_vm(self, host_vm):
-        return PolybenchVm(self._config_name, self._options, self._templateOptions, host_vm)
-
-    def config_name(self):
-        return self._config_name
-
-    def toolchain_name(self):
-        return "native"
-
-    def name(self):
-        return "sulong-polybench"
-
-    def launcherClass(self):
-        return "org.graalvm.polybench.PolyBenchLauncher"
-
-    def launcherName(self):
-        return "polybench"
-
-    def run(self, cwd, args):
-        bench_file = args[-1:]
-        bench_args = args[:-1]
-        launcher_args = ['--path'] + bench_file + self._options + bench_args
-        if hasattr(self.host_vm(), 'run_launcher'):
-            result = self.host_vm().run_launcher(self.launcherName(), launcher_args, cwd)
-        else:
-            def _filter_properties(args):
-                props = []
-                remaining_args = []
-                vm_prefix = "--vm.D"
-                for arg in args:
-                    if arg.startswith(vm_prefix):
-                        props.append('-D' + arg[len(vm_prefix):])
-                    else:
-                        remaining_args.append(arg)
-                return props, remaining_args
-
-            props, launcher_args = _filter_properties(launcher_args)
-            sulongCmdLine = self.launcher_vm_args() + \
-                            props + \
-                            [self.launcherClass(), '--path']  + bench_file + launcher_args
-            result = self.host_vm().run(cwd, sulongCmdLine)
-
-        ret_code, out, vm_dims = result
-        return ret_code, add_run_numbers(out), vm_dims
-
-    def prepare_env(self, env):
-        # if hasattr(self.host_vm(), 'run_launcher'):
-        #     import mx_sdk_vm_impl
-        #     env['CC'] = os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'jre', 'languages', 'llvm', self.toolchain_name(), 'bin', 'graalvm-{}-clang'.format(self.toolchain_name()))
-        # else:
-        # we always use the bootstrap toolchain since the toolchain is not installed by default in a graalvm
-        # change this if we can properly install components into a graalvm deployment
-        env['CC'] = mx_subst.path_substitutions.substitute('<toolchainGetToolPath:{},CC>'.format(self.toolchain_name()))
-        env['CXX'] = mx_subst.path_substitutions.substitute('<toolchainGetToolPath:{},CXX>'.format(self.toolchain_name()))
-        return env
-
-    def out_file(self):
-        return 'bench'
-
-    def opt_phases(self):
-        return []
-
-    def templateOptions(self):
-        return self._templateOptions
-
-    def launcher_vm_args(self):
-        return mx_sulong.getClasspathOptions(['POLYBENCH'])
-
-    def launcher_args(self, args):
-        launcher_args = [
-            '--experimental-options',
-            '--engine.CompilationFailureAction=ExitVM',
-            '--engine.TreatPerformanceWarningsAsErrors=call,instanceof,store',
-        ]
-        return launcher_args + args
-
-    def hosting_registry(self):
-        return java_vm_registry
-
 class LLVMUnitTestsSuite(VmBenchmarkSuite):
     def __init__(self, *args, **kwargs):
         super(LLVMUnitTestsSuite, self).__init__(*args, **kwargs)
@@ -755,25 +610,32 @@ def hosting_registry(self):
 native_vm_registry.add_vm(SulongVm('default-O2', [], cflags=['-O2', '-fno-vectorize', '-fno-slp-vectorize']), _suite, 10)
 native_vm_registry.add_vm(SulongVm('default-O3', [], cflags=['-O3', '-fno-vectorize', '-fno-slp-vectorize']), _suite, 10)
 
-native_polybench_vm_registry = VmRegistry("NativePolybench", known_host_registries=[java_vm_registry])
-native_polybench_vm_registry.add_vm(PolybenchVm('debug-aux-engine-cache',
-    ['--experimental-options', '--eval-source-only.0=true',
-     '--llvm.AOTCacheStore.0=true', '--llvm.AOTCacheLoad.0=false', '--engine.DebugCacheCompile.0=aot', '--engine.DebugCacheStore.0=true',
-     '--llvm.AOTCacheStore.1=false', '--llvm.AOTCacheLoad.1=true', '--engine.DebugCacheStore.1=false', '--engine.DebugCacheLoad.1=true',
-     '--engine.MultiTier=false', '--engine.CompileAOTOnCreate=false', '--engine.DebugCachePreinitializeContext=false',
-     '--engine.DebugTraceCache=true', '--multi-context-runs=2', '-w', '0', '-i', '10'], []), _suite, 10)
-native_polybench_vm_registry.add_vm(PolybenchVm('store-aux-engine-cache',
-    ['--experimental-options', '--multi-context-runs=1', '--eval-source-only=true',
-     '--llvm.AOTCacheStore=true', '--engine.CacheCompile=aot', '--engine.CachePreinitializeContext=false',
-     '--engine.TraceCache=true'], ['--engine.CacheStore=' + os.path.join(os.getcwd(), 'test-${benchmark}.image')]), _suite, 10)
-native_polybench_vm_registry.add_vm(PolybenchVm('load-aux-engine-cache',
-    ['--experimental-options', '--multi-context-runs=1',
-     '--llvm.AOTCacheLoad=true', '--engine.CachePreinitializeContext=false', '--engine.TraceCache=true',
-     '-w', '0', '-i', '10'], ['--engine.CacheLoad=' + os.path.join(os.getcwd(), 'test-${benchmark}.image')]), _suite, 10)
-native_polybench_vm_registry.add_vm(PolybenchVm('3-runs-exclusive-engine',
-    ['--multi-context-runs=3', '--shared-engine=false', '-w', '10', '-i', '10'], []), _suite, 10)
-native_polybench_vm_registry.add_vm(PolybenchVm('3-runs-shared-engine',
-    ['--multi-context-runs=3', '--shared-engine=true', '-w', '10', '-i', '10'], []), _suite, 10)
-
 lit_vm_registry = VmRegistry("Lit", known_host_registries=[java_vm_registry])
 lit_vm_registry.add_vm(LitVm('sulong-native', []), _suite, 10)
+
+mx_polybench.register_polybench_language(mx_suite=_suite, language="llvm", distributions=["LLVM_NATIVE_POM"])
+
+
+def sulong_polybench_runner(polybench_run: mx_polybench.PolybenchRunFunction, tags) -> None:
+    if "gate" in tags:
+        polybench_run(["--jvm", "interpreter/*.bc", "--experimental-options", "--engine.Compilation=false", "-w", "1", "-i", "1"])
+        polybench_run(["--native", "interpreter/*.bc", "--experimental-options", "--engine.Compilation=false", "-w", "1", "-i", "1"])
+    if "benchmark" in tags:
+        polybench_run(["--jvm", "interpreter/*.bc", "--experimental-options", "--engine.Compilation=false"])
+        polybench_run(["--native", "interpreter/*.bc", "--experimental-options", "--engine.Compilation=false"])
+        polybench_run(["--jvm", "interpreter/*.bc"])
+        polybench_run(["--native", "interpreter/*.bc"])
+        polybench_run(["--jvm", "interpreter/*.bc", "--metric=metaspace-memory"])
+        polybench_run(["--jvm", "interpreter/*.bc", "--metric=application-memory"])
+        polybench_run(["--jvm", "interpreter/*.bc", "--metric=allocated-bytes", "-w", "40", "-i", "10", "--experimental-options",
+                       "--engine.Compilation=false"])
+        polybench_run(["--native", "interpreter/*.bc", "--metric=allocated-bytes", "-w", "40", "-i", "10", "--experimental-options",
+                       "--engine.Compilation=false"])
+        polybench_run(["--jvm", "interpreter/*.bc", "--metric=allocated-bytes", "-w", "40", "-i", "10"])
+        polybench_run(["--native", "interpreter/*.bc", "--metric=allocated-bytes", "-w", "40", "-i", "10"])
+
+
+mx_polybench.register_polybench_benchmark_suite(mx_suite=_suite, name="sulong", languages=["llvm"],
+                                                benchmark_distribution="SULONG_POLYBENCH_BENCHMARKS",
+                                                benchmark_file_filter=".*bc", runner=sulong_polybench_runner,
+                                                tags={"gate", "benchmark"})