[libc++] Add a tool to produce historical libc++ benchmark results

This is extremely useful for analysis purposes like finding regressions.
The ability to run such historical analysis locally is extremely useful
for doing quick investigations that may involve non-mainstream libc++
configurations.

Author: ldionne

libcxx/utils/benchmark-historical

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+import argparse
+import logging
+import os
+import pathlib
+import subprocess
+import sys
+import tempfile
+
+PARENT_DIR = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
+
+def directory_path(string):
+    if os.path.isdir(string):
+        return pathlib.Path(string)
+    else:
+        raise NotADirectoryError(string)
+
+def whitespace_separated(stream):
+    """
+    Iterate over a stream, yielding whitespace-delimited elements.
+    """
+    for line in stream:
+        for element in line.split():
+            yield element
+
+def resolve_commit(git_repo, commit):
+    """
+    Resolve the full commit SHA from any tree-ish.
+    """
+    return subprocess.check_output(['git', '-C', git_repo, 'rev-parse', commit], text=True).strip()
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(
+        prog='benchmark-historical',
+        description='Run the libc++ benchmarks against the commits provided on standard input and store the results in '
+                    'LNT format in a directory. This makes it easy to generate historical benchmark results of libc++ '
+                    'for analysis purposes. This script\'s usage is optimized to be run on a set of commits and then '
+                    're-run on a potentially-overlapping set of commits, such as after pulling new commits with Git.')
+    parser.add_argument('--output', '-o', type=pathlib.Path, required=True,
+        help='Path to the directory where the resulting .lnt files are stored.')
+    parser.add_argument('--commit-list', type=argparse.FileType('r'), default=sys.stdin,
+        help='Path to a file containing a whitespace separated list of commits to test. '
+             'By default, this is read from standard input.')
+    parser.add_argument('--overwrite', action='store_true',
+        help='When the data for a commit already exists in the output directory, the tool normally skips it. '
+             'This option instructs the tool to generate the data and overwrite it in the output directory.')
+    parser.add_argument('lit_options', nargs=argparse.REMAINDER,
+        help='Optional arguments passed to lit when running the tests. Should be provided last and '
+             'separated from other arguments with a `--`.')
+    parser.add_argument('--git-repo', type=directory_path, default=pathlib.Path(os.getcwd()),
+        help='Optional path to the Git repository to use. By default, the current working directory is used.')
+    parser.add_argument('--dry-run', action='store_true',
+        help='Do not actually run anything, just print what would be done.')
+    args = parser.parse_args(argv)
+
+    logging.getLogger().setLevel(logging.INFO)
+
+    # Gather lit options
+    lit_options = []
+    if args.lit_options:
+        if args.lit_options[0] != '--':
+            raise ArgumentError('For clarity, Lit options must be separated from other options by --')
+        lit_options = args.lit_options[1:]
+
+    # Process commits one by one. Commits just need to be whitespace separated: we also handle
+    # the case where there is more than one commit per line.
+    for commit in whitespace_separated(args.commit_list):
+        commit = resolve_commit(args.git_repo, commit) # resolve e.g. HEAD to a real SHA
+
+        output_file = args.output / (commit + '.lnt')
+        if output_file.exists():
+            if args.overwrite:
+                logging.info(f'Will overwrite data for commit {commit} in {output_file}')
+            else:
+                logging.info(f'Data for commit {commit} already exists in {output_file}, skipping')
+                continue
+        else:
+            logging.info(f'Benchmarking commit {commit}')
+
+        with tempfile.TemporaryDirectory() as build_dir:
+            test_cmd = [PARENT_DIR / 'test-at-commit', '--git-repo', args.git_repo,
+                                                       '--build', build_dir,
+                                                       '--commit', commit]
+            test_cmd += ['--'] + lit_options
+
+            if args.dry_run:
+                pretty = ' '.join(str(a) for a in test_cmd)
+                logging.info(f'Running {pretty}')
+                continue
+
+            if subprocess.call(test_cmd) != 0:
+                logging.error(f'Failed to run the tests for commit {commit}, skipping')
+                continue
+
+            output_file.parent.mkdir(parents=True, exist_ok=True)
+            consolidate_cmd = [(PARENT_DIR / 'consolidate-benchmarks'), build_dir, '--output', output_file]
+            subprocess.check_call(consolidate_cmd)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

libcxx/utils/test-at-commit

@@ -1,13 +1,13 @@
 #!/usr/bin/env python3
 
 import argparse
-import logging
 import os
+import pathlib
 import subprocess
 import sys
 import tempfile
 
-PARENT_DIR = os.path.dirname(os.path.abspath(__file__))
+PARENT_DIR = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
 
 LIT_CONFIG_FILE = """
 #
@@ -37,7 +37,7 @@ libcxx.test.config.configure(
 
 def directory_path(string):
     if os.path.isdir(string):
-        return string
+        return pathlib.Path(string)
     else:
         raise NotADirectoryError(string)
 
@@ -50,14 +50,14 @@ def main(argv):
                     'performance data, bisect issues, and so on. '
                     'A current limitation of this script is that it assumes the arguments passed to CMake when '
                     'building the library.')
-    parser.add_argument('--build', '-B', type=str, required=True,
+    parser.add_argument('--build', '-B', type=pathlib.Path, required=True,
         help='Path to create the build directory for running the test suite at.')
     parser.add_argument('--commit', type=str, required=True,
         help='Commit to build libc++ at.')
     parser.add_argument('lit_options', nargs=argparse.REMAINDER,
         help='Optional arguments passed to lit when running the tests. Should be provided last and '
              'separated from other arguments with a `--`.')
-    parser.add_argument('--git-repo', type=directory_path, default=os.getcwd(),
+    parser.add_argument('--git-repo', type=directory_path, default=pathlib.Path(os.getcwd()),
         help='Optional path to the Git repository to use. By default, the current working directory is used.')
     args = parser.parse_args(argv)
 
@@ -70,26 +70,26 @@ def main(argv):
 
     with tempfile.TemporaryDirectory() as install_dir:
         # Build the library at the baseline
-        build_cmd = [os.path.join(PARENT_DIR, 'build-at-commit'), '--git-repo', args.git_repo,
-                                                                  '--install-dir', install_dir,
-                                                                  '--commit', args.commit]
+        build_cmd = [PARENT_DIR / 'build-at-commit', '--git-repo', args.git_repo,
+                                                     '--install-dir', install_dir,
+                                                     '--commit', args.commit]
         build_cmd += ['--', '-DCMAKE_BUILD_TYPE=RelWithDebInfo']
         subprocess.check_call(build_cmd)
 
         # Configure the test suite in the specified build directory
-        os.makedirs(args.build)
-        lit_cfg = os.path.abspath(os.path.join(args.build, 'temp_lit_cfg.cfg.in'))
+        args.build.mkdir(parents=True, exist_ok=True)
+        lit_cfg = (args.build / 'temp_lit_cfg.cfg.in').absolute()
         with open(lit_cfg, 'w') as f:
             f.write(LIT_CONFIG_FILE.format(INSTALL_ROOT=install_dir))
 
-        test_suite_cmd = ['cmake', '-B', args.build, '-S', os.path.join(args.git_repo, 'runtimes'), '-G', 'Ninja']
+        test_suite_cmd = ['cmake', '-B', args.build, '-S', args.git_repo / 'runtimes', '-G', 'Ninja']
         test_suite_cmd += ['-D', 'LLVM_ENABLE_RUNTIMES=libcxx;libcxxabi']
         test_suite_cmd += ['-D', 'LIBCXXABI_USE_LLVM_UNWINDER=OFF']
         test_suite_cmd += ['-D', f'LIBCXX_TEST_CONFIG={lit_cfg}']
         subprocess.check_call(test_suite_cmd)
 
         # Run the specified tests against the produced baseline installation
-        lit_cmd = [os.path.join(PARENT_DIR, 'libcxx-lit'), args.build] + lit_options
+        lit_cmd = [PARENT_DIR / 'libcxx-lit', args.build] + lit_options
         subprocess.check_call(lit_cmd)
 
 if __name__ == '__main__':