Further reduce the code in our benchmarks

Author: timfel

mx.graalpython/mx_graalpython_benchmark.py

@@ -23,6 +23,7 @@
 # OF THE POSSIBILITY OF SUCH DAMAGE.
 from __future__ import print_function
 
+import itertools
 import functools
 import statistics
 import sys
@@ -54,14 +55,7 @@
 # constants
 #
 # ----------------------------------------------------------------------------------------------------------------------
-ENV_PYPY_HOME = "PYPY_HOME"
-ENV_PYTHON3_HOME = "PYTHON3_HOME"
-ENV_VIRTUAL_ENV = "VIRTUAL_ENV"
-ENV_JYTHON_JAR = "JYTHON_JAR"
 VM_NAME_GRAALPYTHON = "graalpython"
-VM_NAME_CPYTHON = "cpython"
-VM_NAME_PYPY = "pypy"
-VM_NAME_JYTHON = "jython"
 VM_NAME_GRAALPYTHON_SVM = "graalpython-svm"
 GROUP_GRAAL = "Graal"
 SUBGROUP_GRAAL_PYTHON = "graalpython"
@@ -129,12 +123,11 @@ def _handle_var(key_value):
 #
 # ----------------------------------------------------------------------------------------------------------------------
 class AbstractPythonVm(OutputCapturingVm, ABC):
-    def __init__(self, name, config_name, options=None, env=None):
+    def __init__(self, name, config_name, options=None):
         super().__init__()
         self._name = name
         self._config_name = config_name
         self._options = options
-        self._env = env
 
     @property
     def options(self):
@@ -153,21 +146,21 @@ def interpreter(self):
     def post_process_command_line_args(self, args):
         return args
 
-    def run_vm(self, args, out=None, err=None, cwd=None, nonZeroIsFatal=False):
+    def run_vm(self, args, out=None, err=None, cwd=None, nonZeroIsFatal=False, env=None):
         mx.logv(shlex.join([self.interpreter] + args))
         return mx.run(
             [self.interpreter] + args,
             out=out,
             err=err,
             cwd=cwd,
             nonZeroIsFatal=nonZeroIsFatal,
-            env=self._env
+            env=env,
         )
 
 
 class AbstractPythonIterationsControlVm(AbstractPythonVm):
-    def __init__(self, name, config_name, options=None, env=None, iterations=None):
-        super().__init__(name, config_name, options=options, env=env)
+    def __init__(self, name, config_name, options=None, iterations=None):
+        super().__init__(name, config_name, options=options)
         try:
             self._iterations = int(iterations)
         except:
@@ -194,10 +187,8 @@ def run_vm(self, args, *splat, **kwargs):
 
 
 class CPythonVm(AbstractPythonIterationsControlVm):
-    PYTHON_INTERPRETER = "python3"
-
-    def __init__(self, config_name, options=None, env=None, virtualenv=None, iterations=0):
-        super().__init__(VM_NAME_CPYTHON, config_name, options=options, env=env, iterations=iterations)
+    def __init__(self, config_name, options=None, virtualenv=None, iterations=0):
+        super().__init__("cpython", config_name, options=options, iterations=iterations)
         self._virtualenv = virtualenv
 
     def override_iterations(self, requested_iterations):
@@ -206,41 +197,74 @@ def override_iterations(self, requested_iterations):
 
     @property
     def interpreter(self):
-        venv = self._virtualenv if self._virtualenv else mx.get_env(ENV_VIRTUAL_ENV)
-        if venv:
-            mx.log(f"CPythonVM virtualenv={venv}")
-            return os.path.join(venv, 'bin', CPythonVm.PYTHON_INTERPRETER)
-        home = mx.get_env(ENV_PYTHON3_HOME)
-        if home:
-            mx.log(f"CPythonVM python3 home={home}")
-            return os.path.join(home, CPythonVm.PYTHON_INTERPRETER)
-        return CPythonVm.PYTHON_INTERPRETER
+        candidates_pre = [
+            self._virtualenv,
+            mx.get_env("VIRTUAL_ENV"),
+            mx.get_env("PYTHON3_HOME"),
+        ]
+        candidates_suf = [
+            join("bin", "python3"),
+            join("bin", "python"),
+            "python3",
+            "python",
+            sys.executable,
+        ]
+        for p, s in itertools.product(candidates_pre, candidates_suf):
+            if os.path.exists(exe := os.path.join(p or "", s)):
+                mx.log(f"CPython VM {exe=}")
+                return exe
+        assert False, "sys.executable should really exist"
+
+    def run_vm(self, args, *splat, **kwargs):
+        for idx, arg in enumerate(args):
+            if "--vm.Xmx" in arg:
+                mx.warn(f"Ignoring {arg}, cannot restrict memory on CPython.")
+                args = args[:idx] + args[idx + 1 :]
+                break
+        return super().run_vm(args, *splat, **kwargs)
 
 
 class PyPyVm(AbstractPythonIterationsControlVm):
-    PYPY_INTERPRETER = "pypy3"
-
-    def __init__(self, config_name, options=None, env=None, virtualenv=None, iterations=0):
-        super().__init__(VM_NAME_PYPY, config_name, options=options, env=env, iterations=iterations)
+    def __init__(self, config_name, options=None, virtualenv=None, iterations=0):
+        super().__init__("pypy", config_name, options=options, iterations=iterations)
 
     def override_iterations(self, requested_iterations):
         # PyPy warms up much faster, half should be enough
         return int(requested_iterations / 2)
 
     @property
     def interpreter(self):
-        home = mx.get_env(ENV_PYPY_HOME)
-        if not home:
+        if home := mx.get_env("PYPY_HOME"):
+            exe = join(home, "bin", "pypy3")
+        else:
             try:
-                return subprocess.check_output("which %s" % PyPyVm.PYPY_INTERPRETER, shell=True).decode().strip()
+                exe = subprocess.check_output("which pypy3", shell=True).decode().strip()
             except OSError:
-                mx.abort("{} is not set!".format(ENV_PYPY_HOME))
-        return join(home, 'bin', PyPyVm.PYPY_INTERPRETER)
+                mx.abort("PYPY_HOME is not set!")
+        mx.log(f"PyPy {exe=}")
+        return exe
+
+    def run_vm(self, args, *splat, env=None, **kwargs):
+        env = env or os.environ.copy()
+        xmxArg = re.compile("--vm.Xmx([0-9]+)([kKgGmM])")
+        pypyGcMax = "8GB"
+        for idx, arg in enumerate(args):
+            if m := xmxArg.search(arg):
+                args = args[:idx] + args[idx + 1 :]
+                pypyGcMax = f"{m.group(1)}{m.group(2).upper()}B"
+                mx.log(f"Setting PYPY_GC_MAX={pypyGcMax} via {arg}")
+                break
+        else:
+            mx.log(
+                f"Setting PYPY_GC_MAX={pypyGcMax}, use --vm.Xmx argument to override it"
+            )
+        env["PYPY_GC_MAX"] = pypyGcMax
+        return super().run_vm(args, *splat, env=env, **kwargs)
 
 
 class GraalPythonVm(AbstractPythonIterationsControlVm):
-    def __init__(self, config_name, options=None, env=None, virtualenv=None, iterations=None, extra_polyglot_args=None):
-        super().__init__(VM_NAME_GRAALPYTHON, config_name, options=options, env=env, iterations=iterations)
+    def __init__(self, config_name, options=None, virtualenv=None, iterations=None, extra_polyglot_args=None):
+        super().__init__(VM_NAME_GRAALPYTHON, config_name, options=options, iterations=iterations)
         self._extra_polyglot_args = extra_polyglot_args or []
 
     @property
@@ -311,15 +335,14 @@ def get_extra_polyglot_args(self):
 
 class GraalPythonJavaDriverVm(GuestVm):
     def __init__(self, config_name=CONFIGURATION_DEFAULT, cp_suffix=None, distributions=None, cp_prefix=None,
-                 host_vm=None, extra_vm_args=None, extra_polyglot_args=None, env=None):
+                 host_vm=None, extra_vm_args=None, extra_polyglot_args=None):
         super().__init__(host_vm=host_vm)
         self._config_name = config_name
         self._distributions = distributions or ['GRAALPYTHON_BENCH']
         self._cp_suffix = cp_suffix
         self._cp_prefix = cp_prefix
         self._extra_vm_args = extra_vm_args
         self._extra_polyglot_args = extra_polyglot_args if isinstance(extra_polyglot_args, list) else []
-        self._env = env
 
     def name(self):
         return VM_NAME_GRAALPYTHON
@@ -341,30 +364,28 @@ def get_classpath(self):
     def with_host_vm(self, host_vm):
         return self.__class__(config_name=self._config_name, distributions=self._distributions,
                               cp_suffix=self._cp_suffix, cp_prefix=self._cp_prefix, host_vm=host_vm,
-                              extra_vm_args=self._extra_vm_args, extra_polyglot_args=self._extra_polyglot_args,
-                              env=self._env)
+                              extra_vm_args=self._extra_vm_args, extra_polyglot_args=self._extra_polyglot_args)
 
     def launcher_class(self):
         return 'com.oracle.graal.python.benchmarks.JavaBenchmarkDriver'
 
     def run(self, cwd, args):
         extra_polyglot_args = self.get_extra_polyglot_args()
         host_vm = self.host_vm()
-        with environ(self._env or {}):
-            cp = self.get_classpath()
-            jhm = mx.dependency("mx:JMH_1_21")
-            cp_deps = [
-                mx.distribution('GRAALPYTHON_BENCH', fatalIfMissing=True),
-                jhm,
-                mx.dependency("sdk:LAUNCHER_COMMON")
-            ] + jhm.deps
-            cp += [x.classpath_repr() for x in cp_deps]
-            java_args = ['-cp', ':'.join(cp)] + [self.launcher_class()]
-            out = mx.TeeOutputCapture(mx.OutputCapture())
-            code = host_vm.run_java(java_args + extra_polyglot_args + args, cwd=cwd, out=out, err=out)
-            out = out.underlying.data
-            dims = host_vm.dimensions(cwd, args, code, out)
-            return code, out, dims
+        cp = self.get_classpath()
+        jhm = mx.dependency("mx:JMH_1_21")
+        cp_deps = [
+            mx.distribution('GRAALPYTHON_BENCH', fatalIfMissing=True),
+            jhm,
+            mx.dependency("sdk:LAUNCHER_COMMON")
+        ] + jhm.deps
+        cp += [x.classpath_repr() for x in cp_deps]
+        java_args = ['-cp', ':'.join(cp)] + [self.launcher_class()]
+        out = mx.TeeOutputCapture(mx.OutputCapture())
+        code = host_vm.run_java(java_args + extra_polyglot_args + args, cwd=cwd, out=out, err=out)
+        out = out.underlying.data
+        dims = host_vm.dimensions(cwd, args, code, out)
+        return code, out, dims
 
     def get_extra_polyglot_args(self):
         return ["--experimental-options", "--python.MaxNativeMemory=%s" % (2**34), *self._extra_polyglot_args]
@@ -1001,6 +1022,9 @@ def register_vms(suite, sandboxed_options):
     # Other Python VMs:
     python_vm_registry.add_vm(CPythonVm(config_name=CONFIGURATION_DEFAULT), suite)
     python_vm_registry.add_vm(PyPyVm(config_name=CONFIGURATION_DEFAULT), suite)
+    # For continuity with old datapoints, provide CPython and PyPy with launcher config_name
+    python_vm_registry.add_vm(CPythonVm(config_name="launcher"), suite)
+    python_vm_registry.add_vm(PyPyVm(config_name="launcher"), suite)
 
     graalpy_vms = []
 

mx.graalpython/mx_graalpython_python_benchmarks.py

@@ -372,121 +372,6 @@ def parse(self, text: str) -> list:
         return r
 
 
-class GraalPyVm(mx_benchmark.GuestVm):
-    def __init__(self, config_name, options, host_vm=None):
-        super(GraalPyVm, self).__init__(host_vm=host_vm)
-        self._config_name = config_name
-        self._options = options
-
-    def name(self):
-        return "graalpython"
-
-    def config_name(self):
-        return self._config_name
-
-    def hosting_registry(self):
-        return mx_benchmark.java_vm_registry
-
-    def with_host_vm(self, host_vm):
-        return self.__class__(self.config_name(), self._options, host_vm)
-
-    def run(self, cwd, args):
-        for arg in args:
-            if "--vm.Xmx" in arg:
-                mx.log(f"Setting Xmx from {arg}")
-                break
-        else:
-            xmxArg = "--vm.Xmx8G"
-            mx.log(f"Setting Xmx as {xmxArg}")
-            args.insert(0, xmxArg)
-        try:
-            old_gp_arg = os.environ.get("GRAAL_PYTHON_ARGS")
-            if old_gp_arg:
-                os.environ["GRAAL_PYTHON_ARGS"] = old_gp_arg + " " + xmxArg
-            else:
-                os.environ["GRAAL_PYTHON_ARGS"] = xmxArg
-            old_java_opts = os.environ.get("JAVA_OPTS")
-            if old_java_opts:
-                os.environ["JAVA_OPTS"] = old_java_opts + " " + xmxArg.replace("--vm", "-")
-            else:
-                os.environ["JAVA_OPTS"] = xmxArg.replace("--vm.", "-")
-            mx.log("Running with `JAVA_OPTS={JAVA_OPTS}` and `GRAAL_PYTHON_ARGS={GRAAL_PYTHON_ARGS}`".format(**os.environ))
-            return self.host_vm().run_launcher("graalpy", self._options + args, cwd)
-        finally:
-            if old_java_opts:
-                os.environ["JAVA_OPTS"] = old_java_opts
-            else:
-                del os.environ["JAVA_OPTS"]
-            if old_gp_arg:
-                os.environ["GRAAL_PYTHON_ARGS"] = old_gp_arg
-            else:
-                del os.environ["GRAAL_PYTHON_ARGS"]
-
-
-class PyPyVm(mx_benchmark.Vm):
-    def config_name(self):
-        return "launcher"
-
-    def name(self):
-        return "pypy"
-
-    def interpreter(self):
-        home = mx.get_env("PYPY_HOME")
-        if not home:
-            try:
-                return (
-                    subprocess.check_output("which pypy3", shell=True).decode().strip()
-                )
-            except OSError:
-                mx.abort("{} is not set!".format("PYPY_HOME"))
-        return join(home, "bin", "pypy3")
-
-    def run(self, cwd, args):
-        env = os.environ.copy()
-        xmxArg = re.compile("--vm.Xmx([0-9]+)([kKgGmM])")
-        pypyGcMax = "8GB"
-        for idx, arg in enumerate(args):
-            if m := xmxArg.search(arg):
-                args = args[:idx] + args[idx + 1 :]
-                pypyGcMax = f"{m.group(1)}{m.group(2).upper()}B"
-                mx.log(f"Setting PYPY_GC_MAX={pypyGcMax} via {arg}")
-                break
-        else:
-            mx.log(
-                f"Setting PYPY_GC_MAX={pypyGcMax}, use --vm.Xmx argument to override it"
-            )
-        env["PYPY_GC_MAX"] = pypyGcMax
-        return mx.run([self.interpreter()] + args, cwd=cwd, env=env)
-
-
-class Python3Vm(mx_benchmark.Vm):
-    def config_name(self):
-        return "launcher"
-
-    def name(self):
-        return "cpython"
-
-    def interpreter(self):
-        home = mx.get_env("PYTHON3_HOME")
-        if not home:
-            return sys.executable
-        if exists(exe := join(home, "bin", "python3")):
-            return exe
-        elif exists(exe := join(home, "python3")):
-            return exe
-        elif exists(exe := join(home, "python")):
-            return exe
-        return join(home, "bin", "python")
-
-    def run(self, cwd, args):
-        for idx, arg in enumerate(args):
-            if "--vm.Xmx" in arg:
-                mx.warn(f"Ignoring {arg}, cannot restrict memory on CPython.")
-                args = args[:idx] + args[idx + 1 :]
-                break
-        return mx.run([self.interpreter()] + args, cwd=cwd)
-
-
 class WildcardList:
     """It is not easy to track for external suites which benchmarks are
     available, so we just return a wildcard list and assume the caller knows
@@ -924,16 +809,8 @@ def register_python_benchmarks():
     from mx_graalpython_benchmark import python_vm_registry as vm_registry
 
     python_vm_registry = vm_registry
-
     SUITE = mx.suite("graalpython")
 
-    python_vm_registry.add_vm(PyPyVm())
-    python_vm_registry.add_vm(Python3Vm())
-    for config_name, options, priority in [
-        ("launcher", [], 5),
-    ]:
-        python_vm_registry.add_vm(GraalPyVm(config_name, options), SUITE, priority)
-
     mx_benchmark.add_bm_suite(PyPerformanceSuite())
     mx_benchmark.add_bm_suite(PyPySuite())
     mx_benchmark.add_bm_suite(NumPySuite())