[libc++] Add scripts to build and test libc++ at a specified commit

[ldionne]

This is useful to perform historical analyses, bisections or establish a benchmarking baseline after making some changes on a branch. For example, one can run benchmarks against main and easily compare them to the results on the current feature branch with:

libcxx/utils/test-at-commit --commit $(git merge-base main HEAD) \
                            -B build/baseline -- <lit args>
libcxx/utils/libcxx-lit build/candidate <lit args>
libcxx/utils/compare-benchmarks \
    <(libcxx/utils/consolidate-benchmarks build/baseline) \
    <(libcxx/utils/consolidate-benchmarks build/candidate)

Doing this without these scripts would require checking out the desired baseline, setting up the build directory and running the tests manually. With these scripts, this can automatically be automated without dirtying the current checkout.

[llvmbot]

@llvm/pr-subscribers-libcxx

Author: Louis Dionne (ldionne)

Changes

This is useful to perform historical analyses, bisections or establish a benchmarking baseline after making some changes on a branch. For example, one can run benchmarks against main and easily compare them to the results on the current feature branch with:

libcxx/utils/test-at-commit --commit $(git merge-base main HEAD) \
                            -B build/baseline -- <lit args>
libcxx/utils/libcxx-lit build/candidate <lit args>
libcxx/utils/compare-benchmarks \
    <(libcxx/utils/consolidate-benchmarks build/baseline) \
    <(libcxx/utils/consolidate-benchmarks build/candidate)

Doing this without these scripts would require checking out the desired baseline, setting up the build directory and running the tests manually. With these scripts, this can automatically be automated without dirtying the current checkout.

---

Full diff: https://github.com/llvm/llvm-project/pull/158104.diff

2 Files Affected:

• (added) libcxx/utils/build-at-commit (+132)
• (added) libcxx/utils/test-at-commit (+94)

diff --git a/libcxx/utils/build-at-commit b/libcxx/utils/build-at-commit
new file mode 100755
index 0000000000000..8af7d1161f70a
--- /dev/null
+++ b/libcxx/utils/build-at-commit
@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import subprocess
+import sys
+import tempfile
+
+# Unofficial list of directories required to build libc++. This is a best guess that should work
+# when checking out the monorepo at most commits, but it's technically not guaranteed to work
+# (especially for much older commits).
+LIBCXX_REQUIRED_DIRECTORIES = [
+    'libcxx',
+    'libcxxabi',
+    'llvm/cmake',
+    'llvm/utils/llvm-lit',
+    'llvm/utils/lit',
+    'runtimes',
+    'cmake',
+    'third-party/benchmark',
+    'libc'
+]
+
+def directory_path(string):
+    if os.path.isdir(string):
+        return string
+    else:
+        raise NotADirectoryError(string)
+
+def resolve_commit(git_repo, commit):
+    """
+    Resolve the full commit SHA from any tree-ish.
+    """
+    return subprocess.check_output(['git', '-C', git_repo, 'rev-parse', commit], text=True).strip()
+
+def checkout_subdirectories(git_repo, commit, paths, destination):
+    """
+    Produce a copy of the specified Git-tracked files/directories at the given commit.
+    The resulting files and directories at placed at the given location.
+    """
+    with tempfile.TemporaryDirectory() as tmp:
+        tmpfile = os.path.join(tmp, 'archive.tar.gz')
+        git_archive = ['git', '-C', git_repo, 'archive', '--format', 'tar.gz', '--output', tmpfile, commit, '--'] + list(paths)
+        subprocess.check_call(git_archive)
+        os.makedirs(destination, exist_ok=True)
+        subprocess.check_call(['tar', '-x', '-z', '-f', tmpfile, '-C', destination])
+
+def build_libcxx(src_dir, build_dir, install_dir, cmake_options):
+    """
+    Build and install libc++ using the provided source, build and installation directories.
+    """
+    configure = ['cmake', '-S', os.path.join(src_dir, 'runtimes'), '-B', build_dir, '-G', 'Ninja']
+    configure += ['-D', 'LLVM_ENABLE_RUNTIMES=libcxx;libcxxabi']
+    configure += ['-D', f'CMAKE_INSTALL_PREFIX={install_dir}']
+    configure += ['-D', 'LIBCXXABI_USE_LLVM_UNWINDER=OFF']
+    configure += list(cmake_options)
+    subprocess.check_call(configure)
+
+    build = ['cmake', '--build', build_dir, '--target', 'install']
+    subprocess.check_call(build)
+
+def exists_in_commit(git_repo, commit, path):
+    """
+    Return whether the given path (file or directory) existed at the given commit.
+    """
+    cmd = ['git', '-C', git_repo, 'show', f'{commit}:{path}']
+    result = subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    return result == 0
+
+def main(argv):
+    parser = argparse.ArgumentParser(
+        prog='build-at-commit',
+        description='Attempt to build libc++ at the specified commit. '
+                    'This script checks out libc++ at the given commit and does a best effort attempt '
+                    'to build it and install it to the specified location. This can be useful when '
+                    'performing bisections or historical analyses of libc++ behavior, performance, etc. '
+                    'This may not work for some commits, for example commits where the library is broken '
+                    'or much older commits when the build process was different from today\'s build process.')
+    parser.add_argument('--commit', type=str, required=True,
+        help='Commit to checkout and build.')
+    parser.add_argument('--install-dir', type=str, required=True,
+        help='Path to install the library at. This is equivalent to the `CMAKE_INSTALL_PREFIX` '
+             'used when building.')
+    parser.add_argument('cmake_options', nargs=argparse.REMAINDER,
+        help='Optional arguments passed to CMake when configuring the build. Should be provided last and '
+             'separated from other arguments with a `--`.')
+    parser.add_argument('--git-repo', type=directory_path, default=os.getcwd(),
+        help='Optional path to the Git repository to use. By default, the current working directory is used.')
+    parser.add_argument('--tmp-src-dir', type=str, required=False,
+        help='Optional path to use for the ephemeral source checkout used to perform the build. '
+             'By default, a temporary directory is used and it is cleaned up after the build. '
+             'If a custom directory is specified, it is not cleaned up automatically.')
+    parser.add_argument('--tmp-build-dir', type=str, required=False,
+        help='Optional path to use for the ephemeral build directory used during the build.'
+             'By default, a temporary directory is used and it is cleaned up after the build. '
+             'If a custom directory is specified, it is not cleaned up automatically.')
+    args = parser.parse_args(argv)
+
+    # Gather CMake options
+    cmake_options = []
+    if args.cmake_options is not None:
+        if args.cmake_options[0] != '--':
+            raise ArgumentError('For clarity, CMake options must be separated from other options by --')
+        cmake_options = args.cmake_options[1:]
+
+    # Figure out which directories to check out at the given commit. We avoid checking
+    # out the whole monorepo as an optimization.
+    sha = resolve_commit(args.git_repo, args.commit)
+    checkout_dirs = [d for d in LIBCXX_REQUIRED_DIRECTORIES if exists_in_commit(args.git_repo, sha, d)]
+
+    tempdirs = []
+    if args.tmp_src_dir is not None:
+        src_dir = args.tmp_src_dir
+    else:
+        tempdirs.append(tempfile.TemporaryDirectory())
+        src_dir = tempdirs[-1].name
+
+    if args.tmp_build_dir is not None:
+        build_dir = args.tmp_build_dir
+    else:
+        tempdirs.append(tempfile.TemporaryDirectory())
+        build_dir = tempdirs[-1].name
+
+    try:
+        checkout_subdirectories(args.git_repo, sha, checkout_dirs, src_dir)
+        build_libcxx(src_dir, build_dir, args.install_dir, cmake_options)
+    finally:
+        for d in tempdirs:
+            d.cleanup()
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
diff --git a/libcxx/utils/test-at-commit b/libcxx/utils/test-at-commit
new file mode 100755
index 0000000000000..1ef1ec0c52815
--- /dev/null
+++ b/libcxx/utils/test-at-commit
@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+
+import argparse
+import logging
+import os
+import subprocess
+import sys
+import tempfile
+
+PARENT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+LIT_CONFIG_FILE = """
+#
+# This testing configuration handles running the test suite against a version
+# of libc++ installed at the given path.
+#
+
+lit_config.load_config(config, '@CMAKE_CURRENT_BINARY_DIR@/cmake-bridge.cfg')
+
+config.substitutions.append(('%{{flags}}',
+    '-pthread' + (' -isysroot {{}}'.format('@CMAKE_OSX_SYSROOT@') if '@CMAKE_OSX_SYSROOT@' else '')
+))
+config.substitutions.append(('%{{compile_flags}}', '-nostdinc++ -I {INSTALL_ROOT}/include/c++/v1 -I %{{libcxx-dir}}/test/support'))
+config.substitutions.append(('%{{link_flags}}', '-nostdlib++ -L {INSTALL_ROOT}/lib -Wl,-rpath,{INSTALL_ROOT}/lib -lc++'))
+config.substitutions.append(('%{{exec}}', '%{{executor}} --execdir %T -- '))
+
+import os, site
+site.addsitedir(os.path.join('@LIBCXX_SOURCE_DIR@', 'utils'))
+import libcxx.test.params, libcxx.test.config
+libcxx.test.config.configure(
+    libcxx.test.params.DEFAULT_PARAMETERS,
+    libcxx.test.features.DEFAULT_FEATURES,
+    config,
+    lit_config
+)
+"""
+
+def directory_path(string):
+    if os.path.isdir(string):
+        return string
+    else:
+        raise NotADirectoryError(string)
+
+def main(argv):
+    parser = argparse.ArgumentParser(
+        prog='test-at-commit',
+        description='Build libc++ at the specified commit and test it against the version of the test suite '
+                    'currently checked out in the specified Git repository. '
+                    'This makes it easier to perform historical analyses of libc++ behavior, gather historical '
+                    'performance data, bisect issues, and so on. '
+                    'A current limitation of this script is that it assumes the arguments passed to CMake when '
+                    'building the library.')
+    parser.add_argument('--build', '-B', type=str, required=True,
+        help='Path to create the build directory for running the test suite at.')
+    parser.add_argument('--commit', type=str, required=True,
+        help='Commit to build libc++ at.')
+    parser.add_argument('lit_options', nargs=argparse.REMAINDER,
+        help='Optional arguments passed to lit when running the tests. Should be provided last and '
+             'separated from other arguments with a `--`.')
+    parser.add_argument('--git-repo', type=directory_path, default=os.getcwd(),
+        help='Optional path to the Git repository to use. By default, the current working directory is used.')
+    args = parser.parse_args(argv)
+
+    # Gather lit options
+    lit_options = []
+    if args.lit_options is not None:
+        if args.lit_options[0] != '--':
+            raise ArgumentError('For clarity, Lit options must be separated from other options by --')
+        lit_options = args.lit_options[1:]
+
+    with tempfile.TemporaryDirectory() as install_dir:
+        # Build the library at the baseline
+        build_cmd = [os.path.join(PARENT_DIR, 'build-at-commit'), '--install-dir', install_dir, '--commit', args.commit]
+        build_cmd += ['--', '-DCMAKE_BUILD_TYPE=RelWithDebInfo']
+        subprocess.check_call(build_cmd)
+
+        # Configure the test suite in the specified build directory
+        os.makedirs(args.build)
+        lit_cfg = os.path.abspath(os.path.join(args.build, 'temp_lit_cfg.cfg.in'))
+        with open(lit_cfg, 'w') as f:
+            f.write(LIT_CONFIG_FILE.format(INSTALL_ROOT=install_dir))
+
+        test_suite_cmd = ['cmake', '-B', args.build, '-S', os.path.join(args.git_repo, 'runtimes'), '-G', 'Ninja']
+        test_suite_cmd += ['-D', 'LLVM_ENABLE_RUNTIMES=libcxx;libcxxabi']
+        test_suite_cmd += ['-D', 'LIBCXXABI_USE_LLVM_UNWINDER=OFF']
+        test_suite_cmd += ['-D', f'LIBCXX_TEST_CONFIG={lit_cfg}']
+        subprocess.check_call(test_suite_cmd)
+
+        # Run the specified tests against the produced baseline installation
+        lit_cmd = [os.path.join(PARENT_DIR, 'libcxx-lit'), args.build] + lit_options
+        subprocess.check_call(lit_cmd)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])