Add benchmark suite for (early/late)startup

Author: fangerer

mx.graalpython/mx_graalpython.py

@@ -55,11 +55,11 @@
 import mx_urlrewrites
 import mx_graalpython_bisect
 from mx_gate import Task
-from mx_graalpython_bench_param import PATH_MESO, BENCHMARKS, JBENCHMARKS
+from mx_graalpython_bench_param import PATH_MESO, BENCHMARKS, WARMUP_BENCHMARKS, JBENCHMARKS
 from mx_graalpython_benchmark import PythonBenchmarkSuite, python_vm_registry, CPythonVm, PyPyVm, JythonVm, GraalPythonVm, \
     CONFIGURATION_DEFAULT, CONFIGURATION_SANDBOXED, CONFIGURATION_NATIVE, \
     CONFIGURATION_DEFAULT_MULTI, CONFIGURATION_SANDBOXED_MULTI, CONFIGURATION_NATIVE_MULTI, \
-    PythonInteropBenchmarkSuite
+    PythonInteropBenchmarkSuite, PythonVmWarmupBenchmarkSuite
 
 
 if not sys.modules.get("__main__"):
@@ -1467,6 +1467,8 @@ def _register_vms(namespace):
 def _register_bench_suites(namespace):
     for py_bench_suite in PythonBenchmarkSuite.get_benchmark_suites(BENCHMARKS):
         mx_benchmark.add_bm_suite(py_bench_suite)
+    for py_bench_suite in PythonVmWarmupBenchmarkSuite.get_benchmark_suites(WARMUP_BENCHMARKS):
+        mx_benchmark.add_bm_suite(py_bench_suite)
     for java_bench_suite in PythonInteropBenchmarkSuite.get_benchmark_suites(JBENCHMARKS):
         mx_benchmark.add_bm_suite(java_bench_suite)
 

mx.graalpython/mx_graalpython_bench_param.py

@@ -36,6 +36,7 @@
 PATH_MICRO = os.path.join(_BASE_PATH, 'micro')
 PATH_MESO = os.path.join(_BASE_PATH, 'meso')
 PATH_MACRO = os.path.join(_BASE_PATH, 'macro')
+PATH_WARMUP = os.path.join(_BASE_PATH, 'warmup')
 PATH_INTEROP = os.path.join(_BASE_PATH, 'host_interop')
 
 # ----------------------------------------------------------------------------------------------------------------------
@@ -44,6 +45,7 @@
 #
 # ----------------------------------------------------------------------------------------------------------------------
 # the argument list contains both the harness and benchmark args
+ITER_100 = ['-i', '100']
 ITER_50 = ['-i', '50']
 ITER_35 = ['-i', '35']
 ITER_25 = ['-i', '25']
@@ -182,6 +184,14 @@ def _pickling_benchmarks(module='pickle'):
 }
 
 
+WARMUP_BENCHMARKS = {
+    'gcbench': ITER_100 + ["--startup=1,10,100"] + ['10'] ,
+    'binarytrees3': ITER_100 + ["--startup=1,10,100"] + ['18'],
+    #'binarytrees3': ["-i", "2", "--startup=1,10,100"] + ['18'],
+    'pads-integerpartitions': ITER_100  + ["--startup=1,10,100"] + ['700'],
+}
+
+
 INTEROP_BENCHMARKS = {
     'euler_java': ITER_10 + ['200'],
     'image-magix': ITER_10 + ['10000'],
@@ -212,6 +222,10 @@ def _pickling_benchmarks(module='pickle'):
     "interop": [PATH_INTEROP, INTEROP_BENCHMARKS],
 }
 
+WARMUP_BENCHMARKS = {
+    "warmup": [PATH_WARMUP, WARMUP_BENCHMARKS],
+}
+
 JBENCHMARKS = {
     "pyjava": [INTEROP_JAVA_BENCHMARKS],
 }

mx.graalpython/mx_graalpython_benchmark.py

@@ -284,7 +284,7 @@ def hosting_registry(self):
 
     def run(self, cwd, args):
         _check_vm_args(self.name(), args)
-        extra_polyglot_args = ["--experimental-options", "--python.MaxNativeMemory=%s" % (2**34)] + self._extra_polyglot_args
+        extra_polyglot_args = ["--experimental-options", "-snapshot-startup", "--python.MaxNativeMemory=%s" % (2**34)] + self._extra_polyglot_args
 
         host_vm = self.host_vm()
         if hasattr(host_vm, 'run_lang'): # this is a full GraalVM build
@@ -601,3 +601,58 @@ def createCommandLineArgs(self, benchmarks, bmSuiteArgs):
     def get_benchmark_suites(cls, benchmarks):
         assert isinstance(benchmarks, dict), "benchmarks must be a dict: {suite: {bench: args, ... }, ...}"
         return [cls(suite_name, suite_info[0]) for suite_name, suite_info in benchmarks.items()]
+
+
+class PythonVmWarmupBenchmarkSuite(PythonBenchmarkSuite):
+    def rules(self, output, benchmarks, bm_suite_args):
+        bench_name = self.get_bench_name(benchmarks)
+        arg = self.get_arg(bench_name)
+
+        return [
+            # startup (difference between start of VM to end of first iteration)
+            StdOutRule(
+                r"### STARTUP +at iteration: (?P<iteration>[0-9]+), +duration: (?P<time>[0-9]+(\.[0-9]+)?$)",
+                {
+                    "benchmark": '{}.{}'.format(self._name, bench_name),
+                    "metric.name": "startup",
+                    "metric.iteration": ("<iteration>", int),
+                    "metric.type": "numeric",
+                    "metric.value": ("<time>", float),
+                    "metric.unit": "s",
+                    "metric.score-function": "id",
+                    "metric.better": "lower",
+                    "config.run-flags": "".join(arg),
+                }
+            ),
+
+            StdOutRule(
+                r"### EARLY WARMUP +at iteration: (?P<iteration>[0-9]+), +duration: (?P<time>[0-9]+(\.[0-9]+)?$)",
+                {
+                    "benchmark": '{}.{}'.format(self._name, bench_name),
+                    "metric.name": "early-warmup",
+                    "metric.iteration": ("<iteration>", int),
+                    "metric.type": "numeric",
+                    "metric.value": ("<time>", float),
+                    "metric.unit": "s",
+                    "metric.score-function": "id",
+                    "metric.better": "lower",
+                    "config.run-flags": "".join(arg),
+                }
+            ),
+
+            StdOutRule(
+                r"### LATE WARMUP +at iteration: (?P<iteration>[0-9]+), +duration: (?P<time>[0-9]+(\.[0-9]+)?$)",
+                {
+                    "benchmark": '{}.{}'.format(self._name, bench_name),
+                    "metric.name": "late-warmup",
+                    "metric.iteration": ("<iteration>", int),
+                    "metric.type": "numeric",
+                    "metric.value": ("<time>", float),
+                    "metric.unit": "s",
+                    "metric.score-function": "id",
+                    "metric.better": "lower",
+                    "config.run-flags": "".join(arg),
+                }
+            ),
+        ]
+