Add bisect-benchmark command

Author: msimacek

bisect-benchmark.ini

@@ -0,0 +1,31 @@
+[bisect-benchmark]
+# This is the configuration file for bisecting benchmark jobs in the CI.
+# Usage:
+# - Create a temporary branch based on master (or the bad commit)
+# - Fill in this configuration file, commit the changes and push it
+# TODO job name
+# - Execute the $name job for the commit. When you click the "create PR" link,
+#   you can run jobs in commits UI. You need to wait a bit for the job
+#   enumerator to populate the job list. You don't need to actually create the
+#   PR.
+# - Wat for the jobs to finish. TODO
+
+# The command to build particular configuration. You can copy paste this from
+# the benchmark job log. Don't worry about sforceimports, the script takes care
+# of that
+build_command = mx --dy /compiler build
+
+# The commmand to run the particular benchmark on the configuration. You can
+# copy paste this from the benchmark job log. Make sure you replace the '*'
+# wildcard with a single benchmark, the script only supports bisecting a single
+# benchmark at a time
+benchmark_command = mx --dy /compiler benchmark micro:try-except-simple
+
+# The first known "bad" merge commit for bisection
+bad = 1234deadbeef
+
+# The last known "good" merge commit for bisection
+good = 1234cafebabe
+
+# Whether to checkout graal-enterprise
+enterprise = true

mx.graalpython/mx_graalpython.py

@@ -53,6 +53,7 @@
 import mx_sdk
 import mx_subst
 import mx_urlrewrites
+import mx_graalpython_bisect
 from mx_gate import Task
 from mx_graalpython_bench_param import PATH_MESO, BENCHMARKS, JBENCHMARKS
 from mx_graalpython_benchmark import PythonBenchmarkSuite, python_vm_registry, CPythonVm, PyPyVm, JythonVm, GraalPythonVm, \
@@ -1985,4 +1986,5 @@ def import_files(from_dir, to_dir):
     'graalpytest': [graalpytest, '[-h] [-v] [--python PYTHON] [-k TEST_PATTERN] [TESTS]'],
     'clean': [python_clean, ''],
     'python-update-hpy-import': [update_hpy_import_cmd, '[--no-pull] PATH_TO_HPY'],
+    'bisect-benchmark': [mx_graalpython_bisect.bisect_benchmark, ''],
 })

mx.graalpython/mx_graalpython_bisect.py

@@ -0,0 +1,204 @@
+import re
+import os
+import sys
+import argparse
+import shlex
+import types
+import configparser
+
+import mx
+
+
+SUITE = mx.suite('graalpython')
+
+
+def get_commit(suite, ref='HEAD'):
+    if not suite:
+        return None
+    return suite.vc.git_command(suite.vc_dir, ['rev-parse', ref], abortOnError=True).strip()
+
+
+def get_message(suite, commit):
+    return suite.vc.git_command(suite.vc_dir, ['log', '--format=%s', '-n', '1', commit]).strip()
+
+
+def run_bisect_benchmark(suite, bad, good, callback, downstreams, threshold=None):
+    git_dir = suite.vc_dir
+    commits = SUITE.vc.git_command(
+        git_dir,
+        ['log', '--merges', '--format=format:%H', f'{good}^..{bad}'],
+        abortOnError=True,
+    ).splitlines()
+    if not commits:
+        sys.exit("No merge commits found in the range. Did you swap good and bad?")
+    downstream_suite = downstreams.get(suite)
+    values = [None] * len(commits)
+    if threshold is None:
+        bad_index = 0
+        good_index = len(commits) - 1
+        values[bad_index] = callback(suite, bad)
+        downstream_bad = get_commit(downstream_suite)
+        values[good_index] = callback(suite, good)
+        downstream_good = get_commit(downstream_suite)
+        threshold = (values[bad_index] + values[good_index]) / 2
+        if values[good_index] * 1.03 > values[bad_index]:
+            sys.exit(
+                "Didn't detect a regression - less that 3% difference between good value "
+                f"{values[good_index]} and bad value {values[bad_index]}"
+            )
+    else:
+        bad_index = -1
+        good_index = len(commits)
+        downstream_bad = None
+        downstream_good = None
+    while True:
+        index = bad_index + ((good_index - bad_index) // 2)
+        if index == bad_index or index == good_index:
+            assert good_index - bad_index == 1
+            break
+        commit = commits[index]
+        values[index] = callback(suite, commit)
+        if values[index] < threshold:
+            good_index = index
+            downstream_good = get_commit(downstream_suite)
+        else:
+            bad_index = index
+            downstream_bad = get_commit(downstream_suite)
+    subresults = {}
+    if downstream_bad and downstream_good and downstream_bad != downstream_good:
+        subresult = run_bisect_benchmark(downstream_suite, downstream_bad, downstream_good, callback, downstreams, threshold)
+        subresults[bad_index] = subresult
+    return BisectResult(suite, commits, values, good_index, bad_index, subresults)
+
+
+class BisectResult:
+    def __init__(self, suite, commits, values, good_index, bad_index, subresults):
+        self.suite = suite
+        self.commits = commits
+        self.values = values
+        self.good_index = good_index
+        self.bad_index = bad_index
+        self.subresults = subresults
+
+    @property
+    def repo_name(self):
+        return os.path.basename(self.suite.vc_dir)
+
+    @property
+    def good_commit(self):
+        try:
+            return self.commits[self.good_index]
+        except IndexError:
+            return None
+
+    @property
+    def bad_commit(self):
+        try:
+            return self.commits[self.bad_index]
+        except IndexError:
+            return None
+
+    def visualize(self, level=1):
+        level_marker = '=' * level
+        print(f"{level_marker} {self.repo_name}")
+        for index, (commit, value) in enumerate(zip(self.commits, self.values)):
+            if value is not None:
+                print(f"{level_marker} {commit} {value:6.6} {get_message(self.suite, commit)}")
+            if self.subresults and index in self.subresults:
+                self.subresults[index].visualize(level + 1)
+
+    def summarize(self):
+        if self.bad_commit and self.good_commit:
+            for subresult in self.subresults.values():
+                if subresult.summarize():
+                    return True
+            print(f"Detected bad commit in {self.repo_name} repository:\n{self.bad_commit} {get_message(self.suite, self.bad_commit)}")
+            return True
+        return False
+
+
+def get_suite_py(commit):
+    suite_py = SUITE.vc.git_command(['show', f'{commit}:mx.graalpython/suite.py'], abortOnError=True)
+    namespace = {}
+    exec(suite_py, namespace, namespace)
+    return namespace['suite']
+
+
+def get_graal_commit(commit):
+    suite_py = get_suite_py(commit)
+    return [imp for imp in suite_py['imports'] if imp['name'] == 'sulong'][0]['version']
+
+
+def bisect_benchmark(argv):
+    if 'BISECT_BENCHMARK_CONFIG' in os.environ:
+        cp = configparser.ConfigParser()
+        cp.read(os.environ['BISECT_BENCHMARK_CONFIG'])
+        sec = cp['bisect-benchmark']
+        args = types.SimpleNamespace()
+        args.bad = sec['bad']
+        args.good = sec['good']
+        args.build_command = sec['build_command']
+        args.benchmark_command = sec['benchmark_command']
+        args.benchmark_criterion = sec.get('benchmark_criterion', 'BEST')
+        args.enterprise = sec.getboolean('enterprise', False)
+    else:
+        parser = argparse.ArgumentParser()
+        parser.add_mutually_exclusive_group()
+        parser.add_argument('bad')
+        parser.add_argument('good')
+        parser.add_argument('build_command')
+        parser.add_argument('benchmark_command')
+        parser.add_argument('--benchmark-criterion', default='BEST')
+        parser.add_argument('--enterprise', action='store_true')
+        args = parser.parse_args(argv)
+
+    vm_suite = mx.suite('vm')
+    downstreams = {
+        SUITE: vm_suite,
+    }
+    if args.enterprise:
+        downstreams[vm_suite] = mx.suite('vm-enterprise')
+
+    fetched_enterprise = False
+
+    def benchmark_callback(suite, commit):
+        nonlocal fetched_enterprise
+        suite.vc.update_to_branch(suite.vc_dir, commit)
+        mx.run_mx(['sforceimports'], suite=suite)
+        if args.enterprise and suite.name != 'vm-enterprise':
+            checkout_args = ['--dynamicimports', '/vm-enterprise', 'checkout-downstream', 'vm', 'vm-enterprise']
+            if fetched_enterprise:
+                checkout_args.append('--no-fetch')
+            mx.run_mx(checkout_args, out=mx.OutputCapture())
+            mx.run_mx(['--env', 'ee', 'sforceimports'], suite=mx.suite('vm-enterprise'))
+            fetched_enterprise = True
+        elif suite.name != 'vm':
+            mx.run_mx(['--env', 'ce', 'sforceimports'], suite=vm_suite)
+        suite.vc.update_to_branch(suite.vc_dir, commit)
+        mx.run_mx(['sforceimports'], suite=suite)
+        env = os.environ.copy()
+        if 'CI' not in os.environ:
+            env['MX_ALT_OUTPUT_ROOT'] = f'mxbuild-{commit}'
+        retcode = mx.run(shlex.split(args.build_command), env=env, nonZeroIsFatal=False)
+        if retcode:
+            sys.exit(f"Failed to execute the build command for {commit}")
+        output = mx.OutputCapture()
+        retcode = mx.run(shlex.split(args.benchmark_command), env=env, out=mx.TeeOutputCapture(output), nonZeroIsFatal=False)
+        if retcode:
+            sys.exit(f"Failed to execute benchmark for {commit}")
+        match = re.search(rf'{re.escape(args.benchmark_criterion)}.*duration: ([\d.]+)', output.data)
+        if not match:
+            sys.exit(f"Failed to get result from the benchmark")
+        return float(match.group(1))
+
+    bad = get_commit(SUITE, args.bad)
+    good = get_commit(SUITE, args.good)
+    result = run_bisect_benchmark(SUITE, bad, good, benchmark_callback, downstreams)
+    print()
+    result.visualize()
+    print()
+    result.summarize()
+    print()
+
+    if 'CI' not in os.environ:
+        print(f"You can rerun a benchmark for a particular commit using:\nMX_ALT_OUTPUT_ROOT=mxbuild-$commit {args.benchmark_command}")