Add plaidml total compile time analysis

Author: Quincunx271

util/analyze/lib/compile_times.py

@@ -22,6 +22,15 @@ def sched_time(logs):
 
 _CPU2017_TIME_ELAPSED = re.compile(r"Elapsed compile for '(?P<bench>[^']+)': \S+ \((?P<elapsed>\d+)\)")
 _BACKUP_TIME_ELAPSED = re.compile(r'(?P<elapsed>\d+) total seconds elapsed')
+_PLAIDML_TIME_ELAPSED = re.compile(
+    r'Example finished, elapsed: (?P<elapsed>\S+)s \(compile\), (?P<exec>\S+)s \(execution\)')
+
+
+def plaidml_total_compile_time_seconds(logs):
+    try:
+        return sum(float(_PLAIDML_TIME_ELAPSED.search(bench.blocks[-1].raw_log)['elapsed']) for bench in logs.benchmarks)
+    except TypeError:
+        raise KeyError('Logs must contain "Example finished, elapsed:" output by the PlaidML benchmark suite')
 
 
 def total_compile_time_seconds(logs):
@@ -33,23 +42,34 @@ def total_compile_time_seconds(logs):
     if m:
         if len(m) != 1:
             logging.warning('Multiple CPU2017 elapsed time indicators. Using the first one out of: %s', m)
-        return m[0]['elapsed']
+        return int(m[0]['elapsed'])
 
     m = _BACKUP_TIME_ELAPSED.search(last_logs)
     assert m, \
         'Logs must contain "total seconds elapsed" output by the SPEC benchmark suite'
 
-    return m['elapsed']
+    return int(m['elapsed'])
+
+
+def total_compile_time_seconds_f(benchsuite):
+    return {
+        'spec': total_compile_time_seconds,
+        'plaidml': plaidml_total_compile_time_seconds
+    }[benchsuite]
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument('--variant', choices=('sched', 'total'),
+    parser.add_argument('--variant', choices=('sched', 'total', 'plaidml'),
                         help='Which timing variant to use')
     parser.add_argument('logs', help='The logs to analyze')
     args = analyze.parse_args(parser, 'logs')
 
-    fn = total_compile_time_seconds if args.variant == 'total' else sched_time
+    fn = {
+        'sched': sched_time,
+        'total': total_compile_time_seconds,
+        'plaidml': plaidml_total_compile_time_seconds,
+    }[args.variant]
     results = foreach_bench(fn, args.logs, combine=sum)
     writer = csv.DictWriter(sys.stdout, fieldnames=results.keys())
     writer.writeheader()

util/gt_analysis/gt_cmp.py

@@ -8,7 +8,6 @@
 from analyze.lib import block_stats, compile_times
 
 sched_time = compile_times.sched_time
-total_compile_time_seconds = compile_times.total_compile_time_seconds
 
 
 def blocks_enumerated_optimally(blocks):
@@ -87,9 +86,7 @@ def blk_relative_cost(nogt, gt) -> Tuple[int, int]:
     return no_sum, yes_sum
 
 
-def compute_stats(nogt: Logs, gt: Logs, *, pass_num: int):
-    TOTAL_BLOCKS = utils.count(nogt)
-
+def compute_stats(nogt: Logs, gt: Logs, *, pass_num: int, total_compile_time_seconds):
     nogt_all, gt_all = nogt, gt
 
     if pass_num is not None:
@@ -106,7 +103,7 @@ def compute_stats(nogt: Logs, gt: Logs, *, pass_num: int):
     nogt_rel, gt_rel = blk_relative_cost(nogt, gt)
 
     result = {
-        'Total Blocks in Benchsuite': TOTAL_BLOCKS,
+        'Total Blocks in Benchsuite': utils.count(nogt_all),
         'Num Blocks enumerated with & without GT': utils.count(nogt),
         'Num Blocks proved optimal just by GT': NUM_PROVED_OPTIMAL_WITHOUT_ENUMERATING,
 
@@ -158,7 +155,11 @@ def compute_stats(nogt: Logs, gt: Logs, *, pass_num: int):
     parser.add_argument('--pass-num', type=int, default=None, help='Which pass to analyze (default: all passes)')
     args = analyze.parse_args(parser, 'nogt', 'gt')
 
-    results = utils.foreach_bench(compute_stats, args.nogt, args.gt, pass_num=args.pass_num)
+    results = utils.foreach_bench(
+        compute_stats, args.nogt, args.gt,
+        pass_num=args.pass_num,
+        total_compile_time_seconds=compile_times.total_compile_time_seconds_f(args.benchsuite),
+    )
 
     writer = csv.DictWriter(sys.stdout,
                             fieldnames=['Benchmark'] + list(results['Total'].keys()))