Report startup in harness.py

Author: fangerer

graalpython/com.oracle.graal.python.benchmarks/python/harness.py

@@ -196,7 +196,7 @@ def _as_int(value):
 
 
 class BenchRunner(object):
-    def __init__(self, bench_file, bench_args=None, iterations=1, warmup=-1, warmup_runs=0, startup=False):
+    def __init__(self, bench_file, bench_args=None, iterations=1, warmup=-1, warmup_runs=0, startup=None):
         assert isinstance(iterations, int), \
             "BenchRunner iterations argument must be an int, got %s instead" % iterations
         assert isinstance(warmup, int), \
@@ -276,25 +276,31 @@ def run(self):
         report_startup = GRAALPYTHON and self.startup and __graalpython__.startup_nano != -1
 
         bench_func = self._get_attr(ATTR_BENCHMARK)
-        startup = -1
+        startup_ns = -1
+        early_warmup_ns = -1
+        late_warmup_ns = -1
         durations = []
         if bench_func and hasattr(bench_func, '__call__'):
             if self.warmup_runs:
                 print("### (pre)warming up for %s iterations ... " % self.warmup_runs)
                 for _ in range(self.warmup_runs):
                     bench_func(*args)
                     cur_time_nano = monotonic_ns()
-                    if report_startup and startup == -1:
-                        startup = cur_time_nano - __graalpython__.startup_nano
+                    if report_startup and startup_ns == -1:
+                        startup_ns = cur_time_nano - __graalpython__.startup_nano
                     self._call_attr(ATTR_CLEANUP, *args)
 
             for iteration in range(self.iterations):
                 start = time()
                 bench_func(*args)
                 cur_time_nano = monotonic_ns()
                 duration = time() - start
-                if report_startup and startup == -1:
-                    startup = cur_time_nano - __graalpython__.startup_nano
+                if report_startup and startup_ns == -1 and iteration == self.startup[0] - 1:
+                    startup_ns = cur_time_nano - __graalpython__.startup_nano
+                if report_startup and early_warmup_ns == -1 and iteration == self.startup[1] - 1:
+                    early_warmup_ns = cur_time_nano - __graalpython__.startup_nano
+                if report_startup and late_warmup_ns == -1 and iteration == self.startup[2] - 1:
+                    late_warmup_ns = cur_time_nano - __graalpython__.startup_nano
                 durations.append(duration)
                 duration_str = "%.3f" % duration
                 self._call_attr(ATTR_CLEANUP, *args)
@@ -324,7 +330,9 @@ def run(self):
         # summary
         # We can do that only on Graalpython
         if report_startup:
-            print("### STARTUP           duration: %.3f s" % (startup / 10e9))
+            print("### STARTUP at iteration: %d, duration: %.3f" % (self.startup[0], startup_ns / 1e9))
+            print("### EARLY WARMUP at iteration: %d, duration: %.3f" % (self.startup[1], early_warmup_ns / 1e9))
+            print("### LATE WARMUP at iteration: %d, duration: %.3f" % (self.startup[2], late_warmup_ns / 1e9))
         if self._run_once:
             print("### SINGLE RUN        duration: %.3f s" % durations[0])
         else:
@@ -359,7 +367,7 @@ def run_benchmark(args):
     warmup = -1
     warmup_runs = 0
     iterations = 1
-    startup = False
+    startup = None
     bench_file = None
     bench_args = []
     paths = []
@@ -385,8 +393,12 @@ def run_benchmark(args):
         elif arg.startswith("--warmup-runs"):
             warmup_runs = _as_int(arg.split("=")[1])
 
-        elif arg == '-s' or arg == '--startup':
-            startup = True
+        elif arg.startswith('--startup'):
+            try:
+                itrs = arg.split("=")[1].split(",")
+                startup = (int(itrs[0]), int(itrs[1]), int(itrs[2]))
+            except:
+                raise TypeError("incorrect argument; must be in form of '-s 1,10,100'")
 
         elif arg == '-p':
             i += 1
@@ -400,6 +412,9 @@ def run_benchmark(args):
             bench_args.append(arg)
         i += 1
 
+    if startup and iterations < max(startup):
+        print("### WARNING: you've specified less iterations than required to measure the startup")
+
     # set the paths if specified
     print(_HRULE)
     sys.path.append(os.path.split(bench_file)[0])

graalpython/com.oracle.graal.python.benchmarks/python/warmup/binarytrees3.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# Copyright 2008-2010 Isaac Gouy
+# Copyright (c) 2013, 2014, Regents of the University of California
+# Copyright (c) 2017, 2020, Oracle and/or its affiliates.
+# All rights reserved.
+#
+# Revised BSD license
+#
+# This is a specific instance of the Open Source Initiative (OSI) BSD license
+# template http://www.opensource.org/licenses/bsd-license.php
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#   Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+#   Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+#   Neither the name of "The Computer Language Benchmarks Game" nor the name of
+#   "The Computer Language Shootout Benchmarks" nor the name "nanobench" nor the
+#   name "bencher" nor the names of its contributors may be used to endorse or
+#   promote products derived from this software without specific prior written
+#   permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# The Computer Language Benchmarks Game
+# http://shootout.alioth.debian.org/
+#
+# contributed by Antoine Pitrou
+# modified by Dominique Wahli
+# modified by Heinrich Acker
+
+
+def make_tree(item, depth):
+    if not depth: return item, None, None
+    item2 = item + item
+    depth -= 1
+    return item, make_tree(item2 - 1, depth), make_tree(item2, depth)
+
+
+def check_tree(xxx_todo_changeme):
+    (item, left, right) = xxx_todo_changeme
+    if not left: return item
+    return item + check_tree(left) - check_tree(right)
+
+
+def main(num):
+    min_depth = 4
+    max_depth = max(min_depth + 2, num)
+    stretch_depth = max_depth + 1
+
+    print("stretch tree of depth %d\t check:" % stretch_depth, check_tree(make_tree(0, stretch_depth)))
+
+    long_lived_tree = make_tree(0, max_depth)
+
+    iterations = 2**max_depth
+
+    for depth in range(min_depth, stretch_depth, 2):
+
+        check = 0
+        for i in range(1, iterations + 1):
+            check += check_tree(make_tree(i, depth)) + check_tree(make_tree(-i, depth))
+
+        print("%d\t trees of depth %d\t check:" % (iterations * 2, depth), check)
+        iterations //= 4
+
+    print("long lived tree of depth %d\t check:" % max_depth, check_tree(long_lived_tree))
+
+
+def __benchmark__(num=10):
+    main(num)