[DTLTO][LLD][ELF] Add support for Integrated Distributed ThinLTO

This patch introduces support for Integrated Distributed ThinLTO
(DTLTO) in ELF LLD.

DTLTO enables the distribution of ThinLTO backend compilations via
external distribution systems, such as Incredibuild, during the
traditional link step: https://llvm.org/docs/DTLTO.html.

It is expected that users will invoke DTLTO through the compiler
driver (e.g., Clang) rather than calling LLD directly. A Clang-side
interface for DTLTO will be added in a follow-up patch.

Note: Bitcode members of non-thin archives are not currently
supported. This will be addressed in a future change.

Testing:
- ELF LLD `lit` test coverage has been added, using a mock distributor
  to avoid requiring Clang.
- Cross-project `lit` tests cover integration with Clang.

For the design discussion of the DTLTO feature, see:
#126654

Author: bd1976bris

cross-project-tests/CMakeLists.txt

@@ -19,11 +19,12 @@ set(CROSS_PROJECT_TEST_DEPS
   FileCheck
   check-gdb-llvm-support
   count
-  llvm-dwarfdump
+  llvm-ar
   llvm-config
+  llvm-dwarfdump
   llvm-objdump
-  split-file
   not
+  split-file
   )
 
 if ("clang" IN_LIST LLVM_ENABLE_PROJECTS)
@@ -94,6 +95,13 @@ add_lit_testsuite(check-cross-amdgpu "Running AMDGPU cross-project tests"
   DEPENDS clang
   )
 
+# DTLTO tests.
+add_lit_testsuite(check-cross-dtlto "Running DTLTO cross-project tests"
+  ${CMAKE_CURRENT_BINARY_DIR}/dtlto
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS ${CROSS_PROJECT_TEST_DEPS}
+  )
+
 # Add check-cross-project-* targets.
 add_lit_testsuites(CROSS_PROJECT ${CMAKE_CURRENT_SOURCE_DIR}
   DEPENDS ${CROSS_PROJECT_TEST_DEPS}

cross-project-tests/dtlto/README.md

@@ -0,0 +1,3 @@
+Tests for DTLTO (Integrated Distributed ThinLTO) functionality.
+
+These are integration tests as DTLTO invokes `clang` for code-generation.

cross-project-tests/dtlto/archive-thin.test

@@ -0,0 +1,65 @@
+# REQUIRES: x86-registered-target,ld.lld,llvm-ar
+
+# Test that a DTLTO link succeeds and outputs the expected set of files
+# correctly when thin archives are present.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+RUN: %clang --target=x86_64-linux-gnu -flto=thin -c \
+RUN:   foo.c bar.c dog.c cat.c _start.c
+
+RUN: llvm-ar rcs foo.a foo.o --thin
+# Create this bitcode thin archive in a subdirectory to test the expansion of
+# the path to a bitcode file that is referenced using "..", e.g., in this case
+# "../bar.o".
+RUN: mkdir lib
+RUN: llvm-ar rcs lib/bar.a bar.o --thin
+# Create this bitcode thin archive with an absolute path entry containing "..".
+RUN: llvm-ar rcs dog.a %t/lib/../dog.o --thin
+RUN: llvm-ar rcs cat.a cat.o --thin
+RUN: llvm-ar rcs _start.a _start.o --thin
+
+RUN: mkdir %t/out && cd %t/out
+
+RUN: ld.lld %t/foo.a %t/lib/bar.a ../_start.a %t/cat.a \
+RUN:   --whole-archive ../dog.a \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
+RUN:   --thinlto-remote-compiler=%clang \
+RUN:   --save-temps
+
+# Check that the required output files have been created.
+RUN: ls | FileCheck %s --check-prefix=OUTPUTS \
+RUN:     --implicit-check-not=cat --implicit-check-not=foo
+
+# JSON jobs description.
+OUTPUTS-DAG: a.[[PID:[a-zA-Z0-9_]+]].dist-file.json
+
+# Individual summary index files.
+OUTPUTS-DAG:    dog.1.[[PID]].native.o.thinlto.bc{{$}}
+OUTPUTS-DAG: _start.2.[[PID]].native.o.thinlto.bc{{$}}
+OUTPUTS-DAG:    bar.3.[[PID]].native.o.thinlto.bc{{$}}
+
+# Native output object files.
+OUTPUTS-DAG:    dog.1.[[PID]].native.o{{$}}
+OUTPUTS-DAG: _start.2.[[PID]].native.o{{$}}
+OUTPUTS-DAG:    bar.3.[[PID]].native.o{{$}}
+
+#--- foo.c
+__attribute__((retain)) void foo() {}
+
+#--- bar.c
+extern void foo();
+__attribute__((retain)) void bar() { foo(); }
+
+#--- dog.c
+__attribute__((retain)) void dog() {}
+
+#--- cat.c
+__attribute__((retain)) void cat() {}
+
+#--- _start.c
+extern void bar();
+__attribute__((retain)) void _start() {
+  bar();
+}
+

cross-project-tests/dtlto/dtlto.c

@@ -0,0 +1,35 @@
+// REQUIRES: x86-registered-target,ld.lld
+
+/// Simple test that DTLTO works with a single input bitcode file and that
+/// --save-temps can be applied to the remote compilation.
+// RUN: rm -rf %t && mkdir %t && cd %t
+
+// RUN: %clang --target=x86_64-linux-gnu -c -flto=thin %s
+
+// RUN: ld.lld dtlto.o \
+// RUN:   --thinlto-distributor=%python \
+// RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
+// RUN:   --thinlto-remote-compiler=%clang \
+// RUN:   --thinlto-remote-compiler-arg=--save-temps
+
+/// Check that the required output files have been created.
+// RUN: ls | count 10
+// RUN: ls | FileCheck %s
+
+/// Produced by the bitcode compilation.
+// CHECK-DAG: {{^}}dtlto.o{{$}}
+
+/// Linked ELF.
+// CHECK-DAG: {{^}}a.out{{$}}
+
+/// --save-temps output for the backend compilation.
+// CHECK-DAG: {{^}}dtlto.s{{$}}
+// CHECK-DAG: {{^}}dtlto.s.0.preopt.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.1.promote.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.2.internalize.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.3.import.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.4.opt.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.5.precodegen.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.resolution.txt{{$}}
+
+int _start() { return 0; }

cross-project-tests/dtlto/lit.local.cfg

@@ -0,0 +1,2 @@
+if "clang" not in config.available_features:
+    config.unsupported = True

cross-project-tests/lit.cfg.py

@@ -19,7 +19,7 @@
 config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
 
 # suffixes: A list of file extensions to treat as test files.
-config.suffixes = [".c", ".cl", ".cpp", ".m"]
+config.suffixes = [".c", ".cl", ".cpp", ".m", ".test"]
 
 # excludes: A list of directories to exclude from the testsuite. The 'Inputs'
 # subdirectories contain auxiliary inputs for various tests in their parent
@@ -107,6 +107,8 @@ def get_required_attr(config, attr_name):
 if lldb_path is not None:
     config.available_features.add("lldb")
 
+if llvm_config.use_llvm_tool("llvm-ar"):
+    config.available_features.add("llvm-ar")
 
 def configure_dexter_substitutions():
     """Configure substitutions for host platform and return list of dependencies"""

lld/ELF/Config.h

@@ -249,6 +249,10 @@ struct Config {
   llvm::SmallVector<llvm::StringRef, 0> searchPaths;
   llvm::SmallVector<llvm::StringRef, 0> symbolOrderingFile;
   llvm::SmallVector<llvm::StringRef, 0> thinLTOModulesToCompile;
+  llvm::StringRef dtltoDistributor;
+  llvm::SmallVector<llvm::StringRef, 0> dtltoDistributorArgs;
+  llvm::StringRef dtltoCompiler;
+  llvm::SmallVector<llvm::StringRef, 0> dtltoCompilerArgs;
   llvm::SmallVector<llvm::StringRef, 0> undefined;
   llvm::SmallVector<SymbolVersion, 0> dynamicList;
   llvm::SmallVector<uint8_t, 0> buildIdVector;

lld/ELF/Driver.cpp

@@ -1341,6 +1341,11 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
       args.hasFlag(OPT_dependent_libraries, OPT_no_dependent_libraries, true);
   ctx.arg.disableVerify = args.hasArg(OPT_disable_verify);
   ctx.arg.discard = getDiscard(args);
+  ctx.arg.dtltoDistributor = args.getLastArgValue(OPT_thinlto_distributor_eq);
+  ctx.arg.dtltoDistributorArgs =
+      args::getStrings(args, OPT_thinlto_distributor_arg);
+  ctx.arg.dtltoCompiler = args.getLastArgValue(OPT_thinlto_compiler_eq);
+  ctx.arg.dtltoCompilerArgs = args::getStrings(args, OPT_thinlto_compiler_arg);
   ctx.arg.dwoDir = args.getLastArgValue(OPT_plugin_opt_dwo_dir_eq);
   ctx.arg.dynamicLinker = getDynamicLinker(ctx, args);
   ctx.arg.ehFrameHdr =

lld/ELF/InputFiles.cpp

@@ -20,6 +20,7 @@
 #include "llvm/ADT/CachedHashString.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/LTO/LTO.h"
+#include "llvm/Object/Archive.h"
 #include "llvm/Object/IRObjectFile.h"
 #include "llvm/Support/ARMAttributeParser.h"
 #include "llvm/Support/ARMBuildAttributes.h"
@@ -1739,6 +1740,33 @@ static uint8_t getOsAbi(const Triple &t) {
   }
 }
 
+// For DTLTO, bitcode member names must be valid paths to files on disk.
+// For thin archives, resolve `memberPath` relative to the archive's location.
+// Returns true if adjusted; false otherwise. Non-thin archives are unsupported.
+static bool dtltoAdjustMemberPathIfThinArchive(Ctx &ctx, StringRef archivePath,
+                                               std::string &memberPath) {
+  assert(!archivePath.empty() && !ctx.arg.dtltoDistributor.empty());
+
+  // Check if the archive file is a thin archive by reading its header.
+  auto bufferOrErr =
+      MemoryBuffer::getFileSlice(archivePath, sizeof(ThinArchiveMagic) - 1, 0);
+  if (std::error_code ec = bufferOrErr.getError()) {
+    ErrAlways(ctx) << "cannot open " << archivePath << ": " << ec.message();
+    return false;
+  }
+
+  if (!bufferOrErr->get()->getBuffer().starts_with(ThinArchiveMagic))
+    return false;
+
+  SmallString<64> resolvedPath;
+  if (path::is_relative(memberPath)) {
+    resolvedPath = path::parent_path(archivePath);
+    path::append(resolvedPath, memberPath);
+    memberPath = resolvedPath.str();
+  }
+  return true;
+}
+
 BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName,
                          uint64_t offsetInArchive, bool lazy)
     : InputFile(ctx, BitcodeKind, mb) {
@@ -1756,10 +1784,13 @@ BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName,
   // symbols later in the link stage). So we append file offset to make
   // filename unique.
   StringSaver &ss = ctx.saver;
-  StringRef name = archiveName.empty()
-                       ? ss.save(path)
-                       : ss.save(archiveName + "(" + path::filename(path) +
-                                 " at " + utostr(offsetInArchive) + ")");
+  StringRef name =
+      (archiveName.empty() ||
+       (!ctx.arg.dtltoDistributor.empty() &&
+        dtltoAdjustMemberPathIfThinArchive(ctx, archiveName, path)))
+          ? ss.save(path)
+          : ss.save(archiveName + "(" + path::filename(path) + " at " +
+                    utostr(offsetInArchive) + ")");
   MemoryBufferRef mbref(mb.getBuffer(), name);
 
   obj = CHECK2(lto::InputFile::create(mbref), this);

lld/ELF/LTO.cpp

@@ -180,6 +180,13 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) {
         std::string(ctx.arg.thinLTOPrefixReplaceNew),
         std::string(ctx.arg.thinLTOPrefixReplaceNativeObject),
         ctx.arg.thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite);
+  } else if (!ctx.arg.dtltoDistributor.empty()) {
+    backend = lto::createOutOfProcessThinBackend(
+        llvm::hardware_concurrency(ctx.arg.thinLTOJobs), onIndexWrite,
+        ctx.arg.thinLTOEmitIndexFiles, ctx.arg.thinLTOEmitImportsFiles,
+        ctx.arg.outputFile, ctx.arg.dtltoDistributor,
+        ctx.arg.dtltoDistributorArgs, ctx.arg.dtltoCompiler,
+        ctx.arg.dtltoCompilerArgs, !ctx.arg.saveTempsArgs.empty());
   } else {
     backend = lto::createInProcessThinBackend(
         llvm::heavyweight_hardware_concurrency(ctx.arg.thinLTOJobs),