[GR-26821] Add compiler warmup benchmarks.

PullRequest: graalpython/1359

Author: fangerer

ci.jsonnet

@@ -1 +1 @@
-{ "overlay": "90640c941cf82e5f7167b7c48a909e1d65b2b6d2" }
+{ "overlay": "cde0760728616ed04bef0f4a44ab2454efd457eb" }

graalpython/com.oracle.graal.python.benchmarks/python/harness.py

@@ -37,13 +37,27 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+from time import time
+
+# Capture module load time as soon as possible for other engines. The unit is seconds.
+_module_start_time = time()
+
 import _io
 import os
 import sys
 import types
-from time import time
-
 
+GRAALPYTHON = sys.implementation.name == "graalpython"
+
+# Try to use the timer with best accuracy. Unfortunately, 'monotonic_ns' is not available everywhere.
+if GRAALPYTHON:
+    from time import monotonic_ns
+    monotonic_best_accuracy = monotonic_ns
+    UNITS_PER_SECOND = 1e9
+else:
+    monotonic_best_accuracy = time
+    UNITS_PER_SECOND = 1.0
+    
 _HRULE = '-'.join(['' for i in range(80)])
 
 #: this function is used to pre-process the arguments as expected by the __benchmark__ and __setup__ entry points
@@ -58,6 +72,13 @@
 ATTR_TEARDOWN = '__teardown__'
 
 
+def get_seconds_since_startup(cur_time):
+    if GRAALPYTHON and __graalpython__.startup_nano != -1:
+        return (cur_time - __graalpython__.startup_nano) / UNITS_PER_SECOND
+    # note: the unit of _module_start_time is seconds
+    return cur_time / UNITS_PER_SECOND - _module_start_time
+
+
 # ----------------------------------------------------------------------------------------------------------------------
 #
 # the CUSUM method adapted for warmup detection within a given threshold (initial iterations)
@@ -195,7 +216,7 @@ def _as_int(value):
 
 
 class BenchRunner(object):
-    def __init__(self, bench_file, bench_args=None, iterations=1, warmup=-1, warmup_runs=0):
+    def __init__(self, bench_file, bench_args=None, iterations=1, warmup=-1, warmup_runs=0, startup=None):
         assert isinstance(iterations, int), \
             "BenchRunner iterations argument must be an int, got %s instead" % iterations
         assert isinstance(warmup, int), \
@@ -213,6 +234,7 @@ def __init__(self, bench_file, bench_args=None, iterations=1, warmup=-1, warmup_
         self.iterations = 1 if self._run_once else _iterations
         self.warmup_runs = warmup_runs if warmup_runs > 0 else 0
         self.warmup = warmup if warmup > 0 else -1
+        self.startup = startup
 
     @staticmethod
     def get_bench_module(bench_file):
@@ -271,7 +293,12 @@ def run(self):
         self._call_attr(ATTR_SETUP, *args)
         print("### start benchmark ... ")
 
+        report_startup = self.startup
+
         bench_func = self._get_attr(ATTR_BENCHMARK)
+        startup_s = -1.0
+        early_warmup_s = -1.0
+        late_warmup_s = -1.0
         durations = []
         if bench_func and hasattr(bench_func, '__call__'):
             if self.warmup_runs:
@@ -283,7 +310,14 @@ def run(self):
             for iteration in range(self.iterations):
                 start = time()
                 bench_func(*args)
+                cur_time = monotonic_best_accuracy()
                 duration = time() - start
+                if report_startup and startup_s < 0.0 and iteration == self.startup[0] - 1:
+                    startup_s = get_seconds_since_startup(cur_time)
+                if report_startup and early_warmup_s < 0.0 and iteration == self.startup[1] - 1:
+                    early_warmup_s = get_seconds_since_startup(cur_time)
+                if report_startup and late_warmup_s < 0 and iteration == self.startup[2] - 1:
+                    late_warmup_s = get_seconds_since_startup(cur_time)
                 durations.append(duration)
                 duration_str = "%.3f" % duration
                 self._call_attr(ATTR_CLEANUP, *args)
@@ -311,6 +345,11 @@ def run(self):
         print(_HRULE)
 
         # summary
+        # We can do that only on Graalpython
+        if report_startup:
+            print("### STARTUP at iteration: %d, duration: %.3f" % (self.startup[0], startup_s))
+            print("### EARLY WARMUP at iteration: %d, duration: %.3f" % (self.startup[1], early_warmup_s))
+            print("### LATE WARMUP at iteration: %d, duration: %.3f" % (self.startup[2], late_warmup_s))
         if self._run_once:
             print("### SINGLE RUN        duration: %.3f s" % durations[0])
         else:
@@ -333,6 +372,9 @@ def run(self):
             else:
                 print("### WARMUP iteration not specified or could not be detected")
 
+            if GRAALPYTHON and self.startup and __graalpython__.startup_nano == -1:
+                print("### NOTE: enable startup time snapshotting to increase accuracy.")
+
         print(_HRULE)
         print("### RAW DURATIONS: %s" % str(durations))
         print(_HRULE)
@@ -342,6 +384,7 @@ def run_benchmark(args):
     warmup = -1
     warmup_runs = 0
     iterations = 1
+    startup = None
     bench_file = None
     bench_args = []
     paths = []
@@ -367,6 +410,13 @@ def run_benchmark(args):
         elif arg.startswith("--warmup-runs"):
             warmup_runs = _as_int(arg.split("=")[1])
 
+        elif arg.startswith('--startup'):
+            try:
+                itrs = arg.split("=")[1].split(",")
+                startup = (int(itrs[0]), int(itrs[1]), int(itrs[2]))
+            except:
+                raise TypeError("incorrect argument; must be in form of '-s 1,10,100'")
+
         elif arg == '-p':
             i += 1
             paths = args[i].split(",")
@@ -379,6 +429,11 @@ def run_benchmark(args):
             bench_args.append(arg)
         i += 1
 
+    min_required_iterations = max(startup) if startup else 0
+    if startup and iterations < min_required_iterations:
+        print("### WARNING: you've specified less iterations than required to measure the startup. Overriding iterations with %d" % min_required_iterations)
+        iterations = min_required_iterations
+
     # set the paths if specified
     print(_HRULE)
     sys.path.append(os.path.split(bench_file)[0])
@@ -389,7 +444,7 @@ def run_benchmark(args):
     else:
         print("### no extra module search paths specified")
 
-    BenchRunner(bench_file, bench_args=bench_args, iterations=iterations, warmup=warmup, warmup_runs=warmup_runs).run()
+    BenchRunner(bench_file, bench_args=bench_args, iterations=iterations, warmup=warmup, warmup_runs=warmup_runs, startup=startup).run()
 
 
 if __name__ == '__main__':

graalpython/com.oracle.graal.python.benchmarks/python/warmup/binarytrees3.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# Copyright 2008-2010 Isaac Gouy
+# Copyright (c) 2013, 2014, Regents of the University of California
+# Copyright (c) 2017, 2020, Oracle and/or its affiliates.
+# All rights reserved.
+#
+# Revised BSD license
+#
+# This is a specific instance of the Open Source Initiative (OSI) BSD license
+# template http://www.opensource.org/licenses/bsd-license.php
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#   Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+#   Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+#   Neither the name of "The Computer Language Benchmarks Game" nor the name of
+#   "The Computer Language Shootout Benchmarks" nor the name "nanobench" nor the
+#   name "bencher" nor the names of its contributors may be used to endorse or
+#   promote products derived from this software without specific prior written
+#   permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# The Computer Language Benchmarks Game
+# http://shootout.alioth.debian.org/
+#
+# contributed by Antoine Pitrou
+# modified by Dominique Wahli
+# modified by Heinrich Acker
+
+
+def make_tree(item, depth):
+    if not depth: return item, None, None
+    item2 = item + item
+    depth -= 1
+    return item, make_tree(item2 - 1, depth), make_tree(item2, depth)
+
+
+def check_tree(xxx_todo_changeme):
+    (item, left, right) = xxx_todo_changeme
+    if not left: return item
+    return item + check_tree(left) - check_tree(right)
+
+
+def main(num):
+    min_depth = 4
+    max_depth = max(min_depth + 2, num)
+    stretch_depth = max_depth + 1
+
+    print("stretch tree of depth %d\t check:" % stretch_depth, check_tree(make_tree(0, stretch_depth)))
+
+    long_lived_tree = make_tree(0, max_depth)
+
+    iterations = 2**max_depth
+
+    for depth in range(min_depth, stretch_depth, 2):
+
+        check = 0
+        for i in range(1, iterations + 1):
+            check += check_tree(make_tree(i, depth)) + check_tree(make_tree(-i, depth))
+
+        print("%d\t trees of depth %d\t check:" % (iterations * 2, depth), check)
+        iterations //= 4
+
+    print("long lived tree of depth %d\t check:" % max_depth, check_tree(long_lived_tree))
+
+
+def __benchmark__(num=10):
+    main(num)