add simple bench_harness utility script to be used for simple benchmarks

- suports number of iterations (for warmup analysis), special __warmup__ and __benchmark__ methods are used to run the actual benchmark

Author: cosminbasca

mx.graalpython/bench_harness.py

@@ -37,49 +37,92 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+import _io
 import argparse
+import os
 import sys
 from time import time
 
+_HRULE = '-'.join(['' for i in range(120)])
+ATTR_WARMUP = '__warmup__'
+ATTR_BENCHMARK = '__benchmark__'
 
-def _get_duration_message(i, name, duration):
-    return "iteration=%s, name=%s, duration=%s".format(i, name, duration)
 
+class BenchRunner(object):
+    def __init__(self, bench_file, bench_args=None, iterations=1, verbose=False):
+        if bench_args is None:
+            bench_args = []
+        self.bench_module = BenchRunner.get_bench_module(bench_file)
+        self.bench_args = bench_args
+        self.verbose = verbose
+        if isinstance(iterations, (list, tuple)):
+            iterations = iterations[0]
+        if isinstance(iterations, str):
+            iterations = int(iterations)
+        self.iterations = iterations
 
-def benchmark(name, iterations=1):
-    def fnbenchmark(func):
-        def wrapper(*args, **kwargs):
-            for i in range(iterations):
-                start = time()
-                func(*args, **kwargs)
-                duration = "%.3f\n" % (time() - start)
-                print(_get_duration_message(i, name, duration))
-        return wrapper
-    return fnbenchmark
+    @staticmethod
+    def get_bench_module(bench_file):
+        name = bench_file.rpartition("/")[2].partition(".")[0].replace('.py', '')
+        directory = bench_file.rpartition("/")[0]
+        pkg = []
+        while any(f.endswith("__init__.py") for f in os.listdir(directory)):
+            directory, slash, postfix = directory.rpartition("/")
+            pkg.insert(0, postfix)
+
+        if pkg:
+            sys.path.insert(0, directory)
+            bench_module = __import__(".".join(pkg + [name]))
+            for p in pkg[1:]:
+                bench_module = getattr(bench_module, p)
+            bench_module = getattr(bench_module, name)
+            return bench_module
+
+        else:
+            bench_module = type(sys)(name, bench_file)
+            with _io.FileIO(bench_file, "r") as f:
+                bench_module.__file__ = bench_file
+                exec(compile(f.readall(), bench_file, "exec"), bench_module.__dict__)
+                return bench_module
 
+    def _get_attr(self, attr_name):
+        if hasattr(self.bench_module, attr_name):
+            return getattr(self.bench_module, attr_name)
 
-def _call_if_defined(obj, attr_name):
-    if hasattr(obj, attr_name):
-        attr = getattr(obj, attr_name)
-        if hasattr(attr, '__call__'):
+    def _call_attr(self, attr_name):
+        attr = self._get_attr(attr_name)
+        if attr and hasattr(attr, '__call__'):
             attr()
 
+    def run(self):
+        if self.verbose:
+            print(_HRULE)
+            print(self.bench_module.__name__)
+            print(_HRULE)
+
+        print("### warming up ... ")
+        self._call_attr(ATTR_WARMUP)
+        print("### running benchmark ... ")
+
+        bench_func = self._get_attr(ATTR_BENCHMARK)
+        if bench_func and hasattr(bench_func, '__call__'):
+            for i in range(self.iterations):
+                start = time()
+                bench_func(*self.bench_args)
+                duration = "%.3f\n" % (time() - start)
+                print("### iteration={}, name={}, duration={}".format(i, self.bench_module.__name__, duration))
 
-ATTR_SETUP = '__setup__'
-ATTR_WARMUP = '__warmup__'
-ATTR_BENCHMARK = '__benchmark__'
 
+def run_benchmark(prog, args):
+    parser = argparse.ArgumentParser(prog=prog, description="Run specified benchmark.")
+    parser.add_argument("-v", "--verbose", help="Verbose output.", action="store_true")
+    parser.add_argument("-i", "--iterations", help="The number of iterations top run each benchmark.", default=1)
+    parser.add_argument("bench_file", metavar='BENCH', help="Path to the benchmark to execute.", nargs=1)
+    parser.add_argument("bench_args", metavar='ARGS', help="Path to the benchmarks to execute.", nargs='*', default=None)
 
-def run_benchmark(name):
-    parser = argparse.ArgumentParser(description="Run specified benchmark.")
-    parser.add_argument("-i", "--iterations", help="Number of iterations.", type=int)
+    args = parser.parse_args(args)
+    BenchRunner(args.bench_file[0], bench_args=args.bench_args, iterations=args.iterations, verbose=args.verbose).run()
 
-    args = parser.parse_args()
-    current_module = sys.modules[__name__]
-    _call_if_defined(current_module, ATTR_SETUP)
-    _call_if_defined(current_module, ATTR_WARMUP)
-    bench = getattr(current_module, ATTR_BENCHMARK)
-    if not bench:
-        raise ValueError('%s not defined for %s'.format(ATTR_BENCHMARK, name))
 
-    benchmark(name, iterations=args.iterations)(bench)()
+if __name__ == '__main__':
+    run_benchmark(sys.argv[0], sys.argv[1:])

mx.graalpython/mx_graalpython_benchmark.py

@@ -37,7 +37,7 @@
 # the graalpython suite
 #
 # ----------------------------------------------------------------------------------------------------------------------
-_truffle_python_suite = mx.suite("graalpython")
+_graalpython_suite = mx.suite("graalpython")
 
 # ----------------------------------------------------------------------------------------------------------------------
 #
@@ -158,7 +158,7 @@ def run(self, cwd, args):
         ]
 
         vm_args = [
-            "-Dpython.home=%s" % join(_truffle_python_suite.dir, "graalpython"),
+            "-Dpython.home=%s" % join(_graalpython_suite.dir, "graalpython"),
             '-cp',
             mx.classpath(["com.oracle.graal.python", "com.oracle.graal.python.shell"]),
             "com.oracle.graal.python.shell.GraalPythonMain"
@@ -183,7 +183,7 @@ class PythonBenchmarkSuite(VmBenchmarkSuite):
     def __init__(self, name):
         self._name = name
         self._bench_path, self._benchmarks = benchmarks_list[self._name]
-        self._bench_path = join(_truffle_python_suite.dir, self._bench_path)
+        self._bench_path = join(_graalpython_suite.dir, self._bench_path)
 
     def rules(self, output, benchmarks, bm_suite_args):
         bench_name = os.path.basename(os.path.splitext(benchmarks[0])[0])
@@ -259,6 +259,6 @@ def get_benchmark_suites(cls):
 #
 # ----------------------------------------------------------------------------------------------------------------------
 python_vm_registry = VmRegistry(PYTHON_VM_REGISTRY_NAME, known_host_registries=[java_vm_registry])
-python_vm_registry.add_vm(CPythonVm(CONFIGURATION_DEFAULT), _truffle_python_suite)
-python_vm_registry.add_vm(PyPyVm(CONFIGURATION_DEFAULT), _truffle_python_suite)
-python_vm_registry.add_vm(GraalPythonVm(), _truffle_python_suite, 10)
+python_vm_registry.add_vm(CPythonVm(CONFIGURATION_DEFAULT), _graalpython_suite)
+python_vm_registry.add_vm(PyPyVm(CONFIGURATION_DEFAULT), _graalpython_suite)
+python_vm_registry.add_vm(GraalPythonVm(), _graalpython_suite, 10)