Skip to content

Commit 34fa9da

Browse files
author
git apple-llvm automerger
committed
Merge commit '54c87e7fdd12' from llvm.org/release/21.x into stable/21.x
2 parents 4ea3577 + 54c87e7 commit 34fa9da

File tree

10 files changed

+347
-4
lines changed

10 files changed

+347
-4
lines changed

‎cross-project-tests/dtlto/link-archive-thin.test

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
REQUIRES: lld-link
2+
3+
## Test that a DTLTO link succeeds and outputs the expected set of files
4+
## correctly when thin archives are present.
5+
6+
RUN: rm -rf %t && split-file %s %t && cd %t
7+
8+
## Compile bitcode. -O2 is required for cross-module importing.
9+
RUN: %clang -O2 --target=x86_64-pc-windows-msvc -flto=thin -c \
10+
RUN: foo.c bar.c dog.c cat.c start.c
11+
12+
## Generate thin archives.
13+
RUN: lld-link /lib /llvmlibthin /out:foo.lib foo.o
14+
## Create this bitcode thin archive in a subdirectory to test the expansion of
15+
## the path to a bitcode file that is referenced using "..", e.g., in this case
16+
## "../bar.o".
17+
RUN: mkdir lib
18+
RUN: lld-link /lib /llvmlibthin /out:lib/bar.lib bar.o
19+
## Create this bitcode thin archive with an absolute path entry containing "..".
20+
RUN: lld-link /lib /llvmlibthin /out:dog.lib %t/lib/../dog.o
21+
RUN: lld-link /lib /llvmlibthin /out:cat.lib cat.o
22+
RUN: lld-link /lib /llvmlibthin /out:start.lib start.o
23+
24+
## Link from a different directory to ensure that thin archive member paths are
25+
## resolved correctly relative to the archive locations.
26+
RUN: mkdir %t/out && cd %t/out
27+
RUN: lld-link /subsystem:console /machine:x64 /entry:start /out:my.exe \
28+
RUN: %t/foo.lib %t/lib/bar.lib ../start.lib %t/cat.lib \
29+
RUN: /includeoptional:dog ../dog.lib \
30+
RUN: -thinlto-distributor:%python \
31+
RUN: -thinlto-distributor-arg:%llvm_src_root/utils/dtlto/local.py \
32+
RUN: -thinlto-remote-compiler:%clang \
33+
RUN: /lldsavetemps
34+
35+
## Check that the required output files have been created.
36+
RUN: ls | FileCheck %s --check-prefix=OUTPUTS --implicit-check-not=cat
37+
38+
## JSON jobs description.
39+
OUTPUTS-DAG: my.[[PID:[a-zA-Z0-9_]+]].dist-file.json
40+
41+
## Individual summary index files.
42+
OUTPUTS-DAG: start.1.[[PID]].native.o.thinlto.bc{{$}}
43+
OUTPUTS-DAG: dog.2.[[PID]].native.o.thinlto.bc{{$}}
44+
OUTPUTS-DAG: foo.3.[[PID]].native.o.thinlto.bc{{$}}
45+
OUTPUTS-DAG: bar.4.[[PID]].native.o.thinlto.bc{{$}}
46+
47+
## Native output object files.
48+
OUTPUTS-DAG: start.1.[[PID]].native.o{{$}}
49+
OUTPUTS-DAG: dog.2.[[PID]].native.o{{$}}
50+
OUTPUTS-DAG: foo.3.[[PID]].native.o{{$}}
51+
OUTPUTS-DAG: bar.4.[[PID]].native.o{{$}}
52+
53+
54+
## It is important that cross-module inlining occurs for this test to show that Clang can
55+
## successfully load the bitcode file dependencies recorded in the summary indices.
56+
## Explicitly check that the expected importing has occurred.
57+
58+
RUN: llvm-dis start.1.*.native.o.thinlto.bc -o - | \
59+
RUN: FileCheck %s --check-prefixes=FOO,BAR,START
60+
61+
RUN: llvm-dis dog.2.*.native.o.thinlto.bc -o - | \
62+
RUN: FileCheck %s --check-prefixes=FOO,BAR,DOG,START
63+
64+
RUN: llvm-dis foo.3.*.native.o.thinlto.bc -o - | \
65+
RUN: FileCheck %s --check-prefixes=FOO,BAR,START
66+
67+
RUN: llvm-dis bar.4.*.native.o.thinlto.bc -o - | \
68+
RUN: FileCheck %s --check-prefixes=FOO,BAR,START
69+
70+
FOO-DAG: foo.o
71+
BAR-DAG: bar.o
72+
DOG-DAG: dog.o
73+
START-DAG: start.o
74+
75+
76+
#--- foo.c
77+
extern int bar(int), start(int);
78+
__attribute__((retain)) int foo(int x) { return x + bar(x) + start(x); }
79+
80+
#--- bar.c
81+
extern int foo(int), start(int);
82+
__attribute__((retain)) int bar(int x) { return x + foo(x) + start(x); }
83+
84+
#--- dog.c
85+
extern int foo(int), bar(int), start(int);
86+
__attribute__((retain)) int dog(int x) { return x + foo(x) + bar(x) + start(x); }
87+
88+
#--- cat.c
89+
__attribute__((retain)) void cat(int x) {}
90+
91+
#--- start.c
92+
extern int foo(int), bar(int);
93+
__attribute__((retain)) int start(int x) { return x + foo(x) + bar(x); }

‎cross-project-tests/dtlto/link-dtlto.c

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
// REQUIRES: lld-link
2+
3+
/// Simple test that DTLTO works with a single input bitcode file and that
4+
/// --save-temps can be applied to the remote compilation.
5+
6+
// RUN: rm -rf %t && mkdir %t && cd %t
7+
8+
// RUN: %clang --target=x86_64-pc-windows-msvc -c -flto=thin %s -o dtlto.obj
9+
10+
// RUN: lld-link /subsystem:console /entry:_start dtlto.obj \
11+
// RUN: -thinlto-distributor:%python \
12+
// RUN: -thinlto-distributor-arg:%llvm_src_root/utils/dtlto/local.py \
13+
// RUN: -thinlto-remote-compiler:%clang \
14+
// RUN: -thinlto-remote-compiler-arg:--save-temps
15+
16+
/// Check that the required output files have been created.
17+
// RUN: ls | sort | FileCheck %s
18+
19+
/// No files are expected before.
20+
// CHECK-NOT: {{.}}
21+
22+
/// Linked ELF.
23+
// CHECK: {{^}}dtlto.exe{{$}}
24+
25+
/// Produced by the bitcode compilation.
26+
// CHECK-NEXT: {{^}}dtlto.obj{{$}}
27+
28+
/// --save-temps output for the backend compilation.
29+
// CHECK-NEXT: {{^}}dtlto.s{{$}}
30+
// CHECK-NEXT: {{^}}dtlto.s.0.preopt.bc{{$}}
31+
// CHECK-NEXT: {{^}}dtlto.s.1.promote.bc{{$}}
32+
// CHECK-NEXT: {{^}}dtlto.s.2.internalize.bc{{$}}
33+
// CHECK-NEXT: {{^}}dtlto.s.3.import.bc{{$}}
34+
// CHECK-NEXT: {{^}}dtlto.s.4.opt.bc{{$}}
35+
// CHECK-NEXT: {{^}}dtlto.s.5.precodegen.bc{{$}}
36+
// CHECK-NEXT: {{^}}dtlto.s.resolution.txt{{$}}
37+
38+
/// No files are expected after.
39+
// CHECK-NOT: {{.}}
40+
41+
int _start() { return 0; }

‎cross-project-tests/lit.cfg.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
2020

2121
# suffixes: A list of file extensions to treat as test files.
22-
config.suffixes = [".c", ".cl", ".cpp", ".m"]
22+
config.suffixes = [".c", ".cl", ".cpp", ".m", ".test"]
2323

2424
# excludes: A list of directories to exclude from the testsuite. The 'Inputs'
2525
# subdirectories contain auxiliary inputs for various tests in their parent

‎lld/COFF/Config.h

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,18 @@ struct Configuration {
192192
// Used for /lldltocachepolicy=policy
193193
llvm::CachePruningPolicy ltoCachePolicy;
194194

195+
// Used for /thinlto-distributor:<path>
196+
StringRef dtltoDistributor;
197+
198+
// Used for /thinlto-distributor-arg:<arg>
199+
llvm::SmallVector<llvm::StringRef, 0> dtltoDistributorArgs;
200+
201+
// Used for /thinlto-remote-compiler:<path>
202+
StringRef dtltoCompiler;
203+
204+
// Used for /thinlto-remote-compiler-arg:<arg>
205+
llvm::SmallVector<llvm::StringRef, 0> dtltoCompilerArgs;
206+
195207
// Used for /opt:[no]ltodebugpassmanager
196208
bool ltoDebugPassManager = false;
197209

‎lld/COFF/Driver.cpp

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2088,6 +2088,23 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
20882088
Fatal(ctx) << "/manifestinput: requires /manifest:embed";
20892089
}
20902090

2091+
// Handle /thinlto-distributor:<path>
2092+
config->dtltoDistributor = args.getLastArgValue(OPT_thinlto_distributor);
2093+
2094+
// Handle /thinlto-distributor-arg:<arg>
2095+
for (auto *arg : args.filtered(OPT_thinlto_distributor_arg))
2096+
config->dtltoDistributorArgs.push_back(arg->getValue());
2097+
2098+
// Handle /thinlto-remote-compiler:<path>
2099+
config->dtltoCompiler = args.getLastArgValue(OPT_thinlto_compiler);
2100+
if (!config->dtltoDistributor.empty() && config->dtltoCompiler.empty())
2101+
Err(ctx) << "A value must be specified for /thinlto-remote-compiler if "
2102+
"/thinlto-distributor is specified.";
2103+
2104+
// Handle /thinlto-remote-compiler-arg:<arg>
2105+
for (auto *arg : args.filtered(OPT_thinlto_compiler_arg))
2106+
config->dtltoCompilerArgs.push_back(arg->getValue());
2107+
20912108
// Handle /dwodir
20922109
config->dwoDir = args.getLastArgValue(OPT_dwodir);
20932110

‎lld/COFF/LTO.cpp

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,16 @@ BitcodeCompiler::BitcodeCompiler(COFFLinkerContext &c) : ctx(c) {
110110

111111
// Initialize ltoObj.
112112
lto::ThinBackend backend;
113-
if (ctx.config.thinLTOIndexOnly) {
113+
if (!ctx.config.dtltoDistributor.empty()) {
114+
backend = lto::createOutOfProcessThinBackend(
115+
llvm::hardware_concurrency(ctx.config.thinLTOJobs),
116+
/*OnWrite=*/nullptr,
117+
/*ShouldEmitIndexFiles=*/false,
118+
/*ShouldEmitImportFiles=*/false, ctx.config.outputFile,
119+
ctx.config.dtltoDistributor, ctx.config.dtltoDistributorArgs,
120+
ctx.config.dtltoCompiler, ctx.config.dtltoCompilerArgs,
121+
!ctx.config.saveTempsArgs.empty());
122+
} else if (ctx.config.thinLTOIndexOnly) {
114123
auto OnIndexWrite = [&](StringRef S) { thinIndices.erase(S); };
115124
backend = lto::createWriteIndexesThinBackend(
116125
llvm::hardware_concurrency(ctx.config.thinLTOJobs),

‎lld/COFF/Options.td

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,17 @@ def thinlto_object_suffix_replace : P<
270270
def thinlto_prefix_replace: P<
271271
"thinlto-prefix-replace",
272272
"'old;new' replace old prefix with new prefix in ThinLTO outputs">;
273+
def thinlto_distributor : P<"thinlto-distributor",
274+
"Distributor to use for ThinLTO backend compilations. If specified, ThinLTO "
275+
"backend compilations will be distributed">;
276+
def thinlto_distributor_arg : P<"thinlto-distributor-arg",
277+
"Arguments to pass to the ThinLTO distributor">;
278+
def thinlto_compiler : P<"thinlto-remote-compiler",
279+
"Compiler for the ThinLTO distributor to invoke for ThinLTO backend "
280+
"compilations">;
281+
def thinlto_compiler_arg : P<"thinlto-remote-compiler-arg",
282+
"Compiler arguments for the ThinLTO distributor to pass for ThinLTO backend "
283+
"compilations">;
273284
def lto_obj_path : P<
274285
"lto-obj-path",
275286
"output native object for merged LTO unit to this path">;

‎lld/docs/DTLTO.rst

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,7 @@ during the traditional link step.
77

88
The implementation is documented here: https://llvm.org/docs/DTLTO.html.
99

10-
Currently, DTLTO is only supported in ELF LLD. Support will be added to other
11-
LLD flavours in the future.
10+
Currently, DTLTO is only supported in ELF and COFF LLD.
1211

1312
ELF LLD
1413
-------
@@ -40,3 +39,37 @@ The command-line interface is as follows:
4039
Some LLD LTO options (e.g., ``--lto-sample-profile=<file>``) are supported.
4140
Currently, other options are silently accepted but do not have the intended
4241
effect. Support for such options will be expanded in the future.
42+
43+
COFF LLD
44+
--------
45+
46+
The command-line interface is as follows:
47+
48+
- ``/thinlto-distributor:<path>``
49+
Specifies the file to execute as the distributor process. If specified,
50+
ThinLTO backend compilations will be distributed.
51+
52+
- ``/thinlto-remote-compiler:<path>``
53+
Specifies the path to the compiler that the distributor process will use for
54+
backend compilations. The compiler invoked must match the version of LLD.
55+
56+
- ``/thinlto-distributor-arg:<arg>``
57+
Specifies ``<arg>`` on the command line when invoking the distributor.
58+
Can be specified multiple times.
59+
60+
- ``/thinlto-remote-compiler-arg:<arg>``
61+
Appends ``<arg>`` to the remote compiler's command line.
62+
Can be specified multiple times.
63+
64+
Options that introduce extra input/output files may cause miscompilation if
65+
the distribution system does not automatically handle pushing/fetching them to
66+
remote nodes. In such cases, configure the distributor - possibly using
67+
``/thinlto-distributor-arg:`` - to manage these dependencies. See the
68+
distributor documentation for details.
69+
70+
Some LLD LTO options (e.g., ``/lto-sample-profile:<file>``) are supported.
71+
Currently, other options are silently accepted but do not have the intended
72+
effect. Support for such options could be expanded in the future.
73+
74+
Currently, there is no DTLTO command line interface supplied for ``clang-cl``,
75+
as users are expected to invoke LLD directly.

‎lld/test/COFF/dtlto/files.test

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
REQUIRES: x86
2+
3+
## Test that the LLD options /lldsavetemps and -thinlto-emit-imports-files
4+
## function correctly with DTLTO we also check that index files
5+
## (-thinlto-emit-index-files) are not emitted with DTLTO.
6+
7+
RUN: rm -rf %t && split-file %s %t && cd %t
8+
9+
RUN: sed 's/@t1/@t2/g' t1.ll > t2.ll
10+
11+
## Generate ThinLTO bitcode files. Note that t3.bc will not be used by the
12+
## linker.
13+
RUN: opt -thinlto-bc t1.ll -o t1.bc
14+
RUN: opt -thinlto-bc t2.ll -o t2.bc
15+
RUN: cp t1.bc t3.bc
16+
17+
## Generate object files for mock.py to return.
18+
RUN: llc t1.ll --filetype=obj -o t1.obj
19+
RUN: llc t2.ll --filetype=obj -o t2.obj
20+
21+
## Create response file containing shared ThinLTO linker arguments.
22+
## -start-lib/-end-lib is used to test the special case where unused lazy
23+
## bitcode inputs result in empty index/imports files.
24+
## Note that mock.py does not do any compilation; instead, it simply writes
25+
## the contents of the object files supplied on the command line into the
26+
## output object files in job order.
27+
RUN: echo "/entry:t1 /subsystem:console \
28+
RUN: t1.bc t2.bc -start-lib t3.bc -end-lib /out:my.exe \
29+
RUN: -thinlto-distributor:\"%python\" \
30+
RUN: -thinlto-distributor-arg:\"%llvm_src_root/utils/dtlto/mock.py\" \
31+
RUN: -thinlto-distributor-arg:t1.obj \
32+
RUN: -thinlto-distributor-arg:t2.obj \
33+
RUN: -thinlto-remote-compiler:fake.exe" > l.rsp
34+
35+
## Check that without extra flags, no index/imports files are produced and
36+
## backend temp files are removed.
37+
RUN: lld-link @l.rsp
38+
RUN: ls | FileCheck %s \
39+
RUN: --check-prefixes=NOBACKEND,NOOTHERS
40+
41+
## Check that with /lldsavetemps and -thinlto-emit-imports-files backend
42+
## tempoary files are retained and no index/imports files are produced.
43+
RUN: rm -f *.imports *.thinlto.bc
44+
RUN: lld-link @l.rsp /lldsavetemps -thinlto-emit-imports-files
45+
RUN: ls | sort | FileCheck %s \
46+
RUN: --check-prefixes=BACKEND,NOOTHERS
47+
48+
## JSON jobs description, retained with --save-temps.
49+
## Note that DTLTO temporary files include a PID component.
50+
NOBACKEND-NOT: {{^}}my.[[#]].dist-file.json{{$}}
51+
BACKEND: {{^}}my.[[#]].dist-file.json{{$}}
52+
53+
## Index/imports files for t1.bc.
54+
NOOTHERS-NOT: {{^}}t1.bc.imports{{$}}
55+
NOOTHERS-NOT: {{^}}t1.bc.thinlto.bc{{$}}
56+
57+
## Index/imports files for t2.bc.
58+
NOOTHERS-NOT: {{^}}t2.bc.imports{{$}}
59+
NOOTHERS-NOT: {{^}}t2.bc.thinlto.bc{{$}}
60+
61+
## Empty index/imports files for unused t3.bc.
62+
NOOTHERS-NOT: {{^}}t3.bc.imports{{$}}
63+
NOOTHERS-NOT: {{^}}t3.bc.thinlto.bc{{$}}
64+
65+
#--- t1.ll
66+
target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
67+
target triple = "x86_64-pc-windows-msvc"
68+
69+
define void @t1() {
70+
ret void
71+
}

0 commit comments

Comments
 (0)