[DTLTO][LLVM] Translate some LTO configuration state into clang options.

Intentionally minimal for now. Additional state translation will be added
in future commits.

Author: bd1976bris

cross-project-tests/CMakeLists.txt

@@ -19,11 +19,13 @@ set(CROSS_PROJECT_TEST_DEPS
   FileCheck
   check-gdb-llvm-support
   count
-  llvm-dwarfdump
   llvm-config
+  llvm-dwarfdump
+  llvm-lto2
   llvm-objdump
-  split-file
   not
+  opt
+  split-file
   )
 
 if ("clang" IN_LIST LLVM_ENABLE_PROJECTS)
@@ -94,6 +96,13 @@ add_lit_testsuite(check-cross-amdgpu "Running AMDGPU cross-project tests"
   DEPENDS clang
   )
 
+# DTLTO tests.
+add_lit_testsuite(check-cross-dtlto "Running DTLTO cross-project tests"
+  ${CMAKE_CURRENT_BINARY_DIR}/dtlto
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS ${CROSS_PROJECT_TEST_DEPS}
+  )
+
 # Add check-cross-project-* targets.
 add_lit_testsuites(CROSS_PROJECT ${CMAKE_CURRENT_SOURCE_DIR}
   DEPENDS ${CROSS_PROJECT_TEST_DEPS}

cross-project-tests/dtlto/README.md

@@ -0,0 +1,3 @@
+Tests for DTLTO (integrated distributed ThinLTO) functionality.
+
+These are integration tests as DTLTO invokes `clang` for code-generation.

cross-project-tests/dtlto/dtlto-translate-options.ll

@@ -0,0 +1,144 @@
+;; Check that the expected Clang arguments are generated by DTLTO for the 
+;; backend compilations and are accepted by Clang.
+
+; RUN: rm -rf %t && split-file %s %t && cd %t
+
+;; Generate bitcode files with a summary index.
+; RUN: opt -thinlto-bc x86_64-unknown-linux-gnu.ll -o x86_64-unknown-linux-gnu.bc
+; RUN: opt -thinlto-bc x86_64-pc-windows-msvc.ll   -o x86_64-pc-windows-msvc.bc
+
+
+;; Check that any invalid arguments would cause a Clang error. This property is
+;; relied on by the actual testcases later in this test.
+; RUN: not %clang -x ir x86_64-unknown-linux-gnu.ll \
+; RUN:     -invalid-incorrect-not-an-option 2>&1 | FileCheck %s --check-prefix=SANITY1
+; SANITY1: unknown argument: '-invalid-incorrect-not-an-option'
+
+
+;; Define a substitution used to simplify the testcases.
+; DEFINE: %{distributor} = dummy
+; DEFINE: %{extra_flags} = dummy
+; DEFINE: %{triple} = dummy
+; DEFINE: %{command} = llvm-lto2 run \
+; DEFINE:   -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/%{distributor} \
+; DEFINE:   -thinlto-remote-opt-tool-arg=-Wunused-command-line-argument \
+; DEFINE:   @%{triple}.rsp %{extra_flags}
+
+
+;; Write common arguments to a response files.
+
+; RUN: echo "x86_64-unknown-linux-gnu.bc -o x86_64-unknown-linux-gnu.o \
+; RUN:       -dtlto \
+; RUN:       -dtlto-remote-opt-tool=%clang \
+; RUN:       -thinlto-remote-opt-tool-arg=-Werror \
+; RUN:       -dtlto-distributor=%python \
+; RUN:       -r=x86_64-unknown-linux-gnu.bc,globalfunc1,plx" > x86_64-unknown-linux-gnu.rsp
+
+; RUN: echo "x86_64-pc-windows-msvc.bc -o x86_64-pc-windows-msvc.o \
+; RUN:       -dtlto \
+; RUN:       -dtlto-remote-opt-tool=%clang \
+; RUN:       -thinlto-remote-opt-tool-arg=-Werror \
+; RUN:       -thinlto-remote-opt-tool-arg=-Wno-override-module \
+; RUN:       -dtlto-distributor=%python \
+; RUN:       -r=x86_64-pc-windows-msvc.bc,globalfunc2,plx" > x86_64-pc-windows-msvc.rsp
+
+
+;; Check that boolean configuration states are translated as expected and Clang
+;; accepts them.
+
+; RUN: echo " \
+; RUN:   --addrsig=1 \
+; RUN:   -function-sections=1 \
+; RUN:   -data-sections=1" > on.rsp
+
+; RUN: echo " \
+; RUN:   --addrsig=0 \
+; RUN:   -function-sections=0 \
+; RUN:   -data-sections=0" > off.rsp
+
+;; Perform DTLTO with configuration state set.
+; REDEFINE: %{extra_flags} = @on.rsp
+; REDEFINE: %{distributor} = local.py
+; REDEFINE: %{triple} = x86_64-unknown-linux-gnu
+; RUN: %{command}
+; REDEFINE: %{distributor} = validate.py
+; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=ON \
+; RUN:     --implicit-check-not=-no-pgo-warn-mismatch
+; ON-DAG: "-faddrsig"
+; ON-DAG: "-ffunction-sections"
+; ON-DAG: "-fdata-sections"
+
+;; Perform DTLTO with configuration state unset.
+; REDEFINE: %{extra_flags} = @off.rsp
+; REDEFINE: %{distributor} = local.py
+; RUN: %{command}
+; REDEFINE: %{distributor} = validate.py
+; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=OFF
+; OFF-NOT: --implicit-check-not=--faddrsig
+; OFF-NOT: --implicit-check-not=--ffunction-sections
+; OFF-NOT: --implicit-check-not=--fdata-sections
+; OFF-NOT: --implicit-check-not=-no-pgo-warn-mismatch
+
+
+;; Check optimisation level.
+
+; RUN: llvm-lto2 run \
+; RUN:   -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
+; RUN:   @x86_64-unknown-linux-gnu.rsp \
+; RUN:   -O3
+
+; RUN: not llvm-lto2 run \
+; RUN:   -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \
+; RUN:   @x86_64-unknown-linux-gnu.rsp \
+; RUN:   -O3 2>&1 | FileCheck %s --check-prefix=OPTLEVEL
+; OPTLEVEL-DAG: "-O3"
+
+
+;; Check relocation model.
+
+; REDEFINE: %{extra_flags} = -relocation-model=pic
+; REDEFINE: %{distributor} = local.py
+; RUN: %{command}
+; REDEFINE: %{distributor} = validate.py
+; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=PIC
+; PIC: -fpic
+
+
+; REDEFINE: %{extra_flags} = -relocation-model=pic
+; REDEFINE: %{distributor} = local.py
+; REDEFINE: %{triple} = x86_64-pc-windows-msvc
+; RUN: %{command}
+; REDEFINE: %{distributor} = validate.py
+; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=NOPIC
+; REDEFINE: %{triple} = x86_64-unknown-linux-gnu
+; NOPIC-NOT: -fpic
+
+;; Check specifying a sample profile.
+; REDEFINE: %{extra_flags} = --lto-sample-profile-file="missing.profdata"
+; REDEFINE: %{distributor} = local.py
+; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=SAMPLE_PROFILE_ERR
+; SAMPLE_PROFILE_ERR: no such file or directory: 'missing.profdata'
+; REDEFINE: %{distributor} = validate.py
+; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=SAMPLE_PROFILE
+; SAMPLE_PROFILE-DAG: "-fprofile-sample-use=missing.profdata"
+
+
+;--- x86_64-unknown-linux-gnu.ll
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @globalfunc1() {
+entry:
+  ret void
+}
+
+;--- x86_64-pc-windows-msvc.ll
+
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+define void @globalfunc2() {
+entry:
+  ret void
+}

cross-project-tests/dtlto/lit.local.cfg

@@ -0,0 +1,2 @@
+if any(feature not in config.available_features for feature in ["clang", "llvm-lto2", "opt"]):
+    config.unsupported = True

cross-project-tests/lit.cfg.py

@@ -19,7 +19,7 @@
 config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
 
 # suffixes: A list of file extensions to treat as test files.
-config.suffixes = [".c", ".cl", ".cpp", ".m"]
+config.suffixes = [".c", ".cl", ".cpp", ".m", ".ll"]
 
 # excludes: A list of directories to exclude from the testsuite. The 'Inputs'
 # subdirectories contain auxiliary inputs for various tests in their parent
@@ -96,6 +96,9 @@ def get_required_attr(config, attr_name):
 if lldb_path is not None:
     config.available_features.add("lldb")
 
+for tool in ["llvm-lto2", "opt"]:
+    if llvm_config.use_llvm_tool(tool):
+        config.available_features.add(tool)
 
 def configure_dexter_substitutions():
     """Configure substitutions for host platform and return list of dependencies"""

llvm/lib/LTO/LTO.cpp

@@ -2291,6 +2291,62 @@ class OutOfProcessThinBackend : public CGThinBackend {
     return Error::success();
   }
 
+  // Derive a set of Clang options that will be shared/common for all DTLTO
+  // backend compilations. We are intentionally minimal here as these options
+  // must remain synchronized with the behavior of Clang. DTLTO does not support
+  // all the features available with in-process LTO. More features are expected
+  // to be added over time. Users can specify Clang options directly if a
+  // feature is not supported. Note that explicitly specified options that imply
+  // additional input or output file dependencies must be communicated to the
+  // distribution system, potentially by setting extra options on the
+  // distributor program.
+  // TODO: If this strategy of deriving options proves insufficient, alternative
+  // approaches should be considered, such as:
+  //   - A serialization/deserialization format for LTO configuration.
+  //   - Modifying LLD to be the tool that performs the backend compilations.
+  void buildCommonRemoteOptToolOptions() {
+    const lto::Config &C = Conf;
+    auto &Ops = CodegenOptions;
+    llvm::Triple TT{Jobs.front().Triple};
+
+    Ops.push_back(Saver.save("-O" + Twine(C.OptLevel)));
+
+    if (C.Options.EmitAddrsig)
+      Ops.push_back("-faddrsig");
+    if (C.Options.FunctionSections)
+      Ops.push_back("-ffunction-sections");
+    if (C.Options.DataSections)
+      Ops.push_back("-fdata-sections");
+
+    if (C.RelocModel == Reloc::PIC_)
+      // Clang doesn't have -fpic for all triples.
+      if (!TT.isOSBinFormatCOFF())
+        Ops.push_back("-fpic");
+
+    // Turn on/off warnings about profile cfg mismatch (default on)
+    // --lto-pgo-warn-mismatch.
+    if (!C.PGOWarnMismatch) {
+      Ops.push_back("-mllvm");
+      Ops.push_back("-no-pgo-warn-mismatch");
+    }
+
+    // Enable sample-based profile guided optimizations.
+    // Sample profile file path --lto-sample-profile=<value>.
+    if (!C.SampleProfile.empty()) {
+      Ops.push_back(
+          Saver.save("-fprofile-sample-use=" + Twine(C.SampleProfile)));
+      AdditionalInputs.insert(C.SampleProfile);
+    }
+
+    // We don't know which of options will be used by Clang.
+    Ops.push_back("-Wno-unused-command-line-argument");
+
+    // Forward any supplied options.
+    if (!ThinLTORemoteOptToolArgs.empty())
+      for (auto &a : ThinLTORemoteOptToolArgs)
+        Ops.push_back(a);
+  }
+
   // Generates a JSON file describing the backend compilations, for the
   // distributor.
   bool emitDistributorJson(StringRef DistributorJson) {
@@ -2327,7 +2383,7 @@ class OutOfProcessThinBackend : public CGThinBackend {
           JOS.value(Array{"summary_index", "-fthinlto-index=", 0});
           JOS.value(Saver.save("--target=" + Twine(Jobs.front().Triple)));
 
-          for (const auto &A : ThinLTORemoteOptToolArgs)
+          for (const auto &A : CodegenOptions)
             JOS.value(A);
         });
       });
@@ -2391,6 +2447,8 @@ class OutOfProcessThinBackend : public CGThinBackend {
       return make_error<StringError>(BCError + "all triples must be consistent",
                                      inconvertibleErrorCode());
 
+    buildCommonRemoteOptToolOptions();
+
     SString JsonFile = sys::path::parent_path(LinkerOutputFile);
     sys::path::append(JsonFile, sys::path::stem(LinkerOutputFile) + "." + UID +
                                     ".dist-file.json");

llvm/test/ThinLTO/X86/dtlto/json.test

@@ -43,6 +43,9 @@ CHECK-NEXT: "-fthinlto-index="
 CHECK-NEXT: 0
 CHECK-NEXT: ]
 CHECK:      "--target=x86_64-unknown-linux-gnu"
+CHECK:      "-O2",
+CHECK:      "-fpic"
+CHECK:      "-Wno-unused-command-line-argument"
 CHECK:      "--rota1=10"
 CHECK:      "--rota2=20"
 

llvm/utils/dtlto/local.py

@@ -0,0 +1,25 @@
+import subprocess
+import sys
+import json
+from pathlib import Path
+
+if __name__ == "__main__":
+    # Load the DTLTO information from the input JSON file.
+    data = json.loads(Path(sys.argv[-1]).read_bytes())
+
+    # Iterate over the jobs and execute the codegen tool.
+    for job in data["jobs"]:
+        jobargs = []
+        for arg in data["common"]["args"]:
+            if isinstance(arg, list):
+                # arg is a "template", into which an external filename is to be
+                # inserted. The first element of arg names an array of strings
+                # in the job. The remaining elements of arg are either indices
+                # into the array or literal strings.
+                files, rest = job[arg[0]], arg[1:]
+                jobargs.append(
+                    "".join(files[x] if isinstance(x, int) else x for x in rest)
+                )
+            else:
+                jobargs.append(arg)
+        subprocess.check_call(jobargs)