Add DebugifyBuilder for debug location coverage testing (#493)

This patch adds a new build factory for running tests using Debugify (see
"How To Update Debug Info" in the LLVM docs[0]) for the purposes of
detecting debug info errors as proposed on Discourse[1]. This builder is
very similar to the `TestSuiteBuilder`, but it adds some required CMake
flags to the LLVM and Test Suite builds, and adds an extra step where we
use a script within `llvm/utils` to evaluate the output of Debugify.

As part of implementing this, I had to make some small changes to the
`TestSuiteBuilder` to allow CMake flags to be passed for the Test Suite
build, as currently it only accepts flags for the LLVM build. These
changes should be a no-op for all existing builds; only by passing the
new `extra_test_suite_configure_args` parameter should this have any
effect.

[0] https://llvm.org/docs/HowToUpdateDebugInfo.html#test-original-debug-info-preservation-in-optimizations
[1] https://discourse.llvm.org/t/rfc-require-real-or-annotated-source-locations-on-all-instructions/86816

Author: SLTozer

zorg/buildbot/builders/DebugifyBuilder.py

@@ -0,0 +1,96 @@
+from buildbot.plugins import util
+from buildbot.steps.shell import ShellCommand
+from zorg.buildbot.builders import TestSuiteBuilder
+from zorg.buildbot.commands.CmakeCommand import CmakeCommand
+
+
+def addCheckDebugifyStep(f, debugify_output_path, compiler_dir=".", env={}):
+    script = util.Interpolate(
+        f"%(prop:builddir)s/{compiler_dir}/llvm/utils/llvm-original-di-preservation.py"
+    )
+    f.addStep(
+        ShellCommand(
+            name="check debugify output",
+            command=[
+                "python3",
+                script,
+                util.Interpolate(debugify_output_path),
+                "--acceptance-test",
+                "--reduce",
+            ],
+            description="check debugify output",
+            env=env,
+        )
+    )
+
+
+def getDebugifyBuildFactory(
+    depends_on_projects=None,
+    enable_runtimes="auto",
+    targets=None,
+    llvm_srcdir=None,
+    obj_dir=None,
+    checks=None,
+    install_dir=None,
+    clean=False,
+    test_suite_build_flags="-O2 -g -DNDEBUG",
+    extra_configure_args=None,
+    enable_origin_tracking=True,
+    extra_test_suite_configure_args=None,
+    env={},
+    **kwargs,
+):
+
+    # Make a local copy of the LLVM configure args, as we are going to modify that.
+    if extra_configure_args is not None:
+        llvm_cmake_args = extra_configure_args[:]
+    else:
+        llvm_cmake_args = list()
+
+    tracking_mode = "COVERAGE_AND_ORIGIN" if enable_origin_tracking else "COVERAGE"
+    CmakeCommand.applyRequiredOptions(llvm_cmake_args, [
+        ('-DLLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING=', tracking_mode)
+    ])
+
+    # This path will be passed through to util.Interpolate, so we leave it in this format.
+    debugify_output_path = f"%(prop:builddir)s/debugify-report.json"
+
+    # Make a local copy of the test suite configure args, as we are going to modify that.
+    if extra_test_suite_configure_args is not None:
+        test_suite_cmake_args = extra_test_suite_configure_args[:]
+    else:
+        test_suite_cmake_args = list()
+
+    CmakeCommand.applyDefaultOptions(test_suite_cmake_args, [
+        ('-DTEST_SUITE_SUBDIRS=', 'CTMark'),
+        ('-DTEST_SUITE_RUN_BENCHMARKS=', 'false'),
+        ('-DTEST_SUITE_COLLECT_CODE_SIZE=', 'false'),
+    ])
+    # The only configuration that currently makes sense for Debugify builds is optimized debug info builds; any build
+    # configuration adjustments can be made through the test_suite_build_flags arg.
+    build_flags = f'{test_suite_build_flags} -Xclang -fverify-debuginfo-preserve -Xclang -fverify-debuginfo-preserve-export={debugify_output_path} -mllvm --debugify-quiet -mllvm -debugify-level=locations'
+    CmakeCommand.applyRequiredOptions(test_suite_cmake_args, [
+        ('-DCMAKE_BUILD_TYPE=', 'RelWithDebInfo'),
+    ])
+    test_suite_cmake_args += [
+        util.Interpolate(f"-DCMAKE_C_FLAGS_RELWITHDEBINFO={build_flags}"),
+        util.Interpolate(f"-DCMAKE_CXX_FLAGS_RELWITHDEBINFO={build_flags}"),
+    ]
+
+    f = TestSuiteBuilder.getTestSuiteBuildFactory(
+        depends_on_projects=depends_on_projects,
+        enable_runtimes=enable_runtimes,
+        targets=targets,
+        llvm_srcdir=llvm_srcdir,
+        obj_dir=obj_dir,
+        checks=checks,
+        install_dir=install_dir,
+        clean=clean,
+        extra_configure_args=llvm_cmake_args,
+        extra_test_suite_configure_args=test_suite_cmake_args,
+        **kwargs
+    )
+
+    addCheckDebugifyStep(f, debugify_output_path, compiler_dir=f.monorepo_dir, env=env)
+
+    return f

zorg/buildbot/builders/TestSuiteBuilder.py

@@ -16,6 +16,7 @@ def addTestSuiteStep(
             compiler_dir = '.',
             env = None,
             lit_args = None,
+            extra_configure_args = None,
             **kwargs):
 
     # Set defaults
@@ -24,15 +25,19 @@ def addTestSuiteStep(
     if lit_args is None:
         lit_args = []
 
-    cc = util.Interpolate('-DCMAKE_C_COMPILER=' + '%(prop:builddir)s/'+compiler_dir+'/bin/clang')
-    cxx = util.Interpolate('-DCMAKE_CXX_COMPILER=' + '%(prop:builddir)s/'+compiler_dir+'/bin/clang++')
+    cc = util.Interpolate('-DCMAKE_C_COMPILER=%(prop:builddir)s/'+compiler_dir+'/bin/clang')
+    cxx = util.Interpolate('-DCMAKE_CXX_COMPILER=%(prop:builddir)s/'+compiler_dir+'/bin/clang++')
     lit = util.Interpolate('%(prop:builddir)s/' + compiler_dir + '/bin/llvm-lit')
     test_suite_base_dir = util.Interpolate('%(prop:builddir)s/' + 'test')
     test_suite_src_dir = util.Interpolate('%(prop:builddir)s/' + 'test/test-suite')
     test_suite_workdir = util.Interpolate('%(prop:builddir)s/' + 'test/build-test-suite')
-    cmake_lit_arg = util.Interpolate('-DTEST_SUITE_LIT:FILEPATH=' + '%(prop:builddir)s/' + compiler_dir + '/bin/llvm-lit')
+    cmake_lit_arg = util.Interpolate('-DTEST_SUITE_LIT:FILEPATH=%(prop:builddir)s/' + compiler_dir + '/bin/llvm-lit')
     # used for cmake building test-suite step
-    options = [cc, cxx, cmake_lit_arg]
+    if extra_configure_args is not None:
+        cmake_args = extra_configure_args[:]
+    else:
+        cmake_args = list()
+    cmake_args.extend([cc, cxx, cmake_lit_arg])
 
     # always clobber the build directory to test each new compiler
     f.addStep(ShellCommand(name='Clean Test Suite Build dir',
@@ -51,7 +56,7 @@ def addTestSuiteStep(
                            haltOnFailure=True,
                            description='Running cmake on Test Suite dir',
                            workdir=test_suite_workdir,
-                           options=options,
+                           options=cmake_args,
                            path=test_suite_src_dir,
                            generator='Ninja'))
 
@@ -80,6 +85,7 @@ def getTestSuiteBuildFactory(
            install_dir = None,
            clean = False,
            extra_configure_args = None,
+           extra_test_suite_configure_args = None,
            env = None,
            **kwargs):
 
@@ -109,6 +115,7 @@ def getTestSuiteBuildFactory(
            compiler_dir=f.obj_dir,
            env=env,
            lit_args=lit_args,
+           extra_configure_args=extra_test_suite_configure_args,
            **kwargs)
 
     return f