Misc improvements to benchmarking infrastructure (#130)

This PR makes a couple minor improvements for the benchmarking code.
  - Actually checks that the tests being run pass.
  - Added more tests from the Chromium unittests and removed the
    chromium browser_tests that won't run headlessly.
  - Add a new utility, filter_tests.py, that runs through all of the
    tests in a test suite description and makes sure they pass.
  - Fixed list_gtests.py (single character fix, main didn't have a
    parameter before)
  - Add some info on the new utility to the documentation
  - Add debugging info in benchmark_report_converter for when there is
    a difference between two benchmark sets, it actually prints out
    what the exact difference is.
  - Remove advisor flag from list of required flags when building the
    LLVM test suite as it isn't required when the no compile flags are
    set.

Author: boomanaiden154

compiler_opt/benchmark/benchmark_chromium.py

@@ -57,10 +57,16 @@
 
 FLAGS = flags.FLAGS
 
+test_prefix = './compiler_opt/benchmark/chromium_test_descriptions/'
+
+test_description_files = [
+    'base_perftests.json', 'browser_tests.json', 'components_perftests.json',
+    'base_unittests.json', 'cc_unittests.json', 'components_unittests.json',
+    'content_unittests.json'
+]
+
 default_test_descriptions = [
-    './compiler_opt/tools/chromium_test_descriptions/base_perftests.json',
-    './compiler_opt/tools/chromium_test_descriptions/browser_tests.json',
-    './compiler_opt/tools/chromium_test_descriptions/components_perftests.json'
+    f'{test_prefix}{test_dsc}' for test_dsc in test_description_files
 ]
 
 flags.DEFINE_multi_string(

compiler_opt/benchmark/benchmark_llvm_test_suite.py

@@ -104,7 +104,6 @@
     'perf_counter', [], 'A perf counter to be used (may be defined more than'
     'once).')
 
-flags.mark_flag_as_required('advisor')
 flags.mark_flag_as_required('output_path')
 
 

compiler_opt/benchmark/benchmark_report.py

@@ -24,6 +24,8 @@
 from typing import List
 from typing import Tuple
 
+from absl import logging
+
 # For each benchmark, and for each counter, capture the recorded values.
 PerBenchmarkResults = Dict[str, Dict[str, List[float]]]
 
@@ -130,9 +132,16 @@ class BenchmarkComparison:
   """Analysis of 2 benchmark runs."""
 
   def __init__(self, base_report: BenchmarkReport, exp_report: BenchmarkReport):
+    base_names_set = set(base_report.names())
+    exp_names_set = set(exp_report.names())
     if base_report.suite_name() != exp_report.suite_name():
       raise ValueError('cannot compare different suites')
-    if set(base_report.names()) != set(exp_report.names()):
+    if base_names_set != exp_names_set:
+      diff_base_exp = base_names_set.difference(exp_names_set)
+      diff_exp_base = exp_names_set.difference(base_names_set)
+      diff_set = diff_base_exp.union(diff_exp_base)
+      logging.info('The following tests differ between the test suites: %s',
+                   diff_set)
       raise ValueError('suite runs have different benchmark names')
     if set(base_report.counters()) != set(exp_report.counters()):
       raise ValueError(

compiler_opt/benchmark/benchmark_report_converter.py

@@ -16,7 +16,7 @@
 
 To run:
 python3 \
-compiler_opt/benchmark/benchmark_report_counter.py \
+compiler_opt/benchmark/benchmark_report_converter.py \
   --base=/tmp/base_report.json \
   --exp=/tmp/exp_report.json \
   --counters=INSTRUCTIONS \

compiler_opt/benchmark/chromium_test_descriptions/base_unittests.json

@@ -0,0 +1,80 @@
+# coding=utf-8
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A script for filtering gtests based on whether or not they pass/fail
+
+Within Chromium specifically, there are some test executables that have a lot
+of tests that are excellent for benchmarking, but running these test suites
+in their entirety can sometimes be problematic as some subsets of the tests
+available in the executable might require certain hardware configurations
+such as an X configuration with working graphics drivers, and we would prefer
+to avoid those tests. This exectuable goes through an entire test suite
+description and returns another test suite description containing only tests
+that pass.
+
+Usage:
+PYTHONPATH=$PYTHONPATH:. python3 \
+  ./compiler_opt/benchmark/filter_tests.py \
+  --input_tests=./compiler_opt/benchmark/chromium_test_descriptions \
+    /browser_tests.json \
+  --output_tests=./browser_tests_filtered.json \
+  --num_threads=32 \
+  --executable_path=/chromium/src/out/Release/browser_tests
+"""
+
+import json
+import os
+
+from absl import flags
+from absl import app
+from absl import logging
+
+from compiler_opt.benchmark import gtest_executable_utils
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string('input_tests', '',
+                    'The path to the test description JSON to filter.')
+flags.DEFINE_string(
+    'output_tests', '',
+    'The path to the JSON file to place the output test suite '
+    'description.')
+flags.DEFINE_integer(
+    'num_threads', 1, 'The number of threads to use for running tests in '
+    'parallel.')
+flags.DEFINE_string(
+    'executable_path', '',
+    'The path to the Chromium build directory where all the '
+    'test executables are stored')
+
+
+def main(_):
+  if not os.path.exists(FLAGS.executable_path):
+    logging.fatal('Executable path does not exist.')
+  with open(FLAGS.input_tests, encoding='UTF-8') as test_description_file:
+    test_suite_description = json.load(test_description_file)
+    test_outputs = gtest_executable_utils.run_test_suite(
+        test_suite_description, FLAGS.executable_path, [], FLAGS.num_threads)
+    test_list = []
+    for test_output in test_outputs:
+      test_list.append(test_output['name'])
+    # copy the old test suite and just replace the tests array
+    new_test_suite_description = test_suite_description
+    new_test_suite_description['tests'] = test_list
+    with open(FLAGS.output_tests, 'w', encoding='UTF-8') as tests_output_file:
+      json.dump(new_test_suite_description, tests_output_file)
+
+
+if __name__ == '__main__':
+  app.run(main)