Sync with llvm#127749

- Remove linker version.

- Make AddBuffer a default parameter for LTO::run() to minimize changes
  at call sites.

- Format code with clang-format, even if it deviates slightly from the
  surrounding style. I decided that CI passing for the PRs was more
  important than matching the existing style.

- Order distributor command-line options correctly.

- Remove dead code in llvm-lto2.cpp.

- Add docstrings to Python distributors.

- Update the Python mock.py distributor to validate JSON.

- Organize LLVM DTLTO tests by moving them into a dedicated "dtlto"
  directory, tidying existing tests and adding new ones.

- Implement fix for more than one LTO partition (spotted by MaskRay).

Author: bd1976bris

clang/lib/Driver/ToolChains/Gnu.cpp

@@ -547,7 +547,8 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
                            Twine(ToolChain.getDriver().getClangProgramPath())));
 
     for (auto A : Args.getAllArgValues(options::OPT_Xthinlto_distributor_EQ))
-      CmdArgs.push_back(Args.MakeArgString("-mllvm=-thinlto-distributor-arg=" + A));
+      CmdArgs.push_back(
+          Args.MakeArgString("-mllvm=-thinlto-distributor-arg=" + A));
   }
 
   if (Args.hasArg(options::OPT_Z_Xlinker__no_demangle))

clang/tools/clang-nvlink-wrapper/ClangNVLinkWrapper.cpp

@@ -645,7 +645,7 @@ Expected<SmallVector<StringRef>> getInput(const ArgList &Args) {
           std::make_unique<raw_fd_ostream>(FD, true));
     };
 
-    if (Error Err = LTOBackend.run(AddStream, /*AddBuffer=*/nullptr))
+    if (Error Err = LTOBackend.run(AddStream))
       return Err;
 
     if (Args.hasArg(OPT_lto_emit_llvm) || Args.hasArg(OPT_lto_emit_asm))

cross-project-tests/dtlto/dtlto-translate-options.ll

@@ -8,8 +8,8 @@
 ; RUN: opt -thinlto-bc x86_64-pc-windows-msvc.ll   -o x86_64-pc-windows-msvc.bc
 
 
-;; Check that any invalid arguments would cause a Clang error. This property is
-;; relied on by the actual testcases later in this test.
+;; Check that invalid arguments cause a Clang error. This property is relied on
+;; by the actual testcases later in this test.
 ; RUN: not %clang -x ir x86_64-unknown-linux-gnu.ll \
 ; RUN:     -invalid-incorrect-not-an-option 2>&1 | FileCheck %s --check-prefix=SANITY1
 ; SANITY1: unknown argument: '-invalid-incorrect-not-an-option'
@@ -25,7 +25,7 @@
 ; DEFINE:   @%{triple}.rsp %{extra_flags}
 
 
-;; Write common arguments to a response files.
+;; Write common arguments to response files.
 
 ; RUN: echo "x86_64-unknown-linux-gnu.bc -o x86_64-unknown-linux-gnu.o \
 ; RUN:       -dtlto \
@@ -63,7 +63,7 @@
 ; RUN: %{command}
 ; REDEFINE: %{distributor} = validate.py
 ; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=ON \
-; RUN:     --implicit-check-not=-no-pgo-warn-mismatch
+; RUN:   --implicit-check-not=-no-pgo-warn-mismatch
 ; ON-DAG: "-faddrsig"
 ; ON-DAG: "-ffunction-sections"
 ; ON-DAG: "-fdata-sections"
@@ -80,7 +80,7 @@
 ; OFF-NOT: --implicit-check-not=-no-pgo-warn-mismatch
 
 
-;; Check optimisation level.
+;; Check optimization level.
 
 ; RUN: llvm-lto2 run \
 ; RUN:   -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
@@ -124,7 +124,6 @@
 
 
 ;--- x86_64-unknown-linux-gnu.ll
-
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -134,7 +133,6 @@ entry:
 }
 
 ;--- x86_64-pc-windows-msvc.ll
-
 target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc"
 

lld/COFF/LTO.cpp

@@ -118,14 +118,11 @@ BitcodeCompiler::BitcodeCompiler(COFFLinkerContext &c) : ctx(c) {
   // Initialize ltoObj.
   lto::ThinBackend backend;
   if (!ctx.config.dtltoDistributor.empty()) {
-    StringRef version = getenv("LLD_VERSION"); // For testing only.
-    if (version.empty())
-      version = ctx.saver.save(getLLDVersion());
     backend = lto::createOutOfProcessThinBackend(
         llvm::heavyweight_hardware_concurrency(ctx.config.thinLTOJobs),
         /*OnWrite=*/nullptr,
         /*ShouldEmitIndexFiles=*/false,
-        /*ShouldEmitImportFiles=*/false, ctx.config.outputFile, version,
+        /*ShouldEmitImportFiles=*/false, ctx.config.outputFile,
         ctx.config.dtltoRemoteOptTool, ctx.config.dtltoDistributor,
         !ctx.config.saveTempsArgs.empty());
   } else if (ctx.config.thinLTOIndexOnly) {
@@ -210,7 +207,7 @@ std::vector<InputFile *> BitcodeCompiler::compile() {
         return std::make_unique<CachedFileStream>(
             std::make_unique<raw_svector_ostream>(buf[task].second));
       },
-      AddBuffer, cache));
+      cache, AddBuffer));
 
   // Emit empty index files for non-indexed files
   for (StringRef s : thinIndices) {

lld/ELF/LTO.cpp

@@ -188,13 +188,10 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) {
         std::string(ctx.arg.thinLTOPrefixReplaceNativeObject),
         ctx.arg.thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite);
   } else if (!ctx.arg.dtltoDistributor.empty() && !ctx.bitcodeFiles.empty()) {
-    StringRef version = getenv("LLD_VERSION"); // For testing only.
-    if (version.empty())
-      version = ctx.saver.save(getLLDVersion());
     backend = lto::createOutOfProcessThinBackend(
         llvm::heavyweight_hardware_concurrency(ctx.arg.thinLTOJobs),
         onIndexWrite, ctx.arg.thinLTOEmitIndexFiles,
-        ctx.arg.thinLTOEmitImportsFiles, ctx.arg.outputFile, version,
+        ctx.arg.thinLTOEmitImportsFiles, ctx.arg.outputFile,
         ctx.arg.dtltoRemoteOptTool, ctx.arg.dtltoDistributor,
         !ctx.arg.saveTempsArgs.empty());
   } else {
@@ -347,7 +344,7 @@ SmallVector<std::unique_ptr<InputFile>, 0> BitcodeCompiler::compile() {
                                 std::make_unique<raw_svector_ostream>(
                                     buf[task].second));
                           },
-                          AddBuffer, cache));
+                          cache, AddBuffer));
 
   // Emit empty index files for non-indexed files but not in single-module mode.
   if (ctx.arg.thinLTOModulesToCompile.empty()) {

lld/ELF/Options.td

@@ -1,4 +1,3 @@
-
 include "llvm/Option/OptParser.td"
 
 // Convenience classes for long options which only accept two dashes. For lld

lld/MachO/LTO.cpp

@@ -212,7 +212,7 @@ std::vector<ObjFile *> BitcodeCompiler::compile() {
           return std::make_unique<CachedFileStream>(
               std::make_unique<raw_svector_ostream>(buf[task]));
         },
-        AddBuffer, cache));
+        cache, AddBuffer));
 
   // Emit empty index files for non-indexed files
   for (StringRef s : thinIndices) {

lld/test/COFF/dtlto.test

@@ -26,7 +26,6 @@
 # NONE: distributor_args=[]
 
 # OPT:  "linker_output": "foo.exe"
-# OPT:  "linker_version": "LLD 1.0"
 # BOTH: "my_clang.exe"
 # BOTH:  "-O2"
 # OPT:  "bobbity=20"

lld/test/ELF/dtlto/dtlto.test

@@ -26,7 +26,6 @@
 # NONE: distributor_args=[]
 
 # OPT:       "linker_output": "a.out"
-# OPT:       "linker_version": "LLD 1.0"
 # BOTH:      "my_clang.exe"
 # OPT:       "-O3"
 # NONE:      "-O2"

lld/wasm/LTO.cpp

@@ -205,7 +205,7 @@ std::vector<StringRef> BitcodeCompiler::compile() {
         return std::make_unique<CachedFileStream>(
             std::make_unique<raw_svector_ostream>(buf[task].second));
       },
-      Addbuffer, cache));
+      cache, Addbuffer));
 
   // Emit empty index files for non-indexed files but not in single-module mode.
   for (StringRef s : thinIndices) {