Skip to content

Commit ea25797

Browse files
bd1976brisbd1976bris
committed
Test improvements and a bug fix.
- Addressed test review comments. - Improved content in the cross-project-tests/dtlto README. - Reorder the code so no operations on the jobs occur until all the index generation tasks have completed. This fixes a intermittent failure that could occur if an error occurred before all the index generation jobs were complete. As it was intermittent it wasn't caught by my testing and made it into the initial PR :( In a future revision we will want to introduce a more robust mechanism to clean up the DTLTO output. For now keep the implementation simple.
1 parent e4bd134 commit ea25797

File tree

10 files changed

+65
-65
lines changed

10 files changed

+65
-65
lines changed

‎clang/test/Driver/DTLTO/dtlto.c‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
// REQUIRES: lld
44

5-
// RUN: echo "-target x86_64-linux-gnu \
5+
// RUN: echo "--target=x86_64-linux-gnu \
66
// RUN: -Xdist distarg1 \
77
// RUN: -Xdist distarg2 \
88
// RUN: -fuse-ld=lld" > %t.rsp
@@ -36,8 +36,9 @@
3636

3737
/// Check the expected arguments are forwarded by default with only
3838
/// --thinlto-distributor=.
39-
// RUN: %clang -### -target x86_64-linux-gnu -fthinlto-distributor=dist.exe -fuse-ld=lld %s 2>&1 | \
40-
// RUN: FileCheck %s --check-prefixes=DEFAULT,NOMORE --implicit-check-not=warning
39+
// RUN: %clang --target=x86_64-linux-gnu -fthinlto-distributor=dist.exe \
40+
// RUN: -fuse-ld=lld -Werror -### %s 2>&1 | \
41+
// RUN: FileCheck %s --check-prefixes=DEFAULT,NOMORE
4142

4243
// DEFAULT: ld.lld
4344
// DEFAULT-SAME: "--thinlto-distributor=dist.exe"

‎cross-project-tests/dtlto/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Tests for DTLTO (integrated distributed ThinLTO) functionality.
2+
3+
These are integration tests as DTLTO invokes `clang` for code-generation.

‎cross-project-tests/dtlto/README.txt‎

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎cross-project-tests/dtlto/archive-thin.test‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@
22
## correctly when thin archives are present.
33

44
# RUN: rm -rf %t.dir && split-file %s %t.dir && cd %t.dir
5-
# RUN: %clang -target x86_64-linux-gnu -c foo.c -o foo.o
6-
# RUN: %clang -target x86_64-linux-gnu -c -flto=thin bar.c -o bar.o
7-
# RUN: %clang -target x86_64-linux-gnu -c -flto=thin dog.c -o dog.o
8-
# RUN: %clang -target x86_64-linux-gnu -c -flto=thin cat.c -o cat.o
9-
# RUN: %clang -target x86_64-linux-gnu -c -flto=thin _start.c -o _start.o
5+
# RUN: %clang --target=x86_64-linux-gnu -c foo.c -o foo.o
6+
# RUN: %clang --target=x86_64-linux-gnu -c -flto=thin bar.c -o bar.o
7+
# RUN: %clang --target=x86_64-linux-gnu -c -flto=thin dog.c -o dog.o
8+
# RUN: %clang --target=x86_64-linux-gnu -c -flto=thin cat.c -o cat.o
9+
# RUN: %clang --target=x86_64-linux-gnu -c -flto=thin _start.c -o _start.o
1010

1111
# RUN: llvm-ar rcs foo.a foo.o --thin
1212
## Create this bitcode thin archive in a sub-directory to test the expansion of
@@ -22,7 +22,7 @@
2222

2323
# RUN: mkdir %t.dir/out && cd %t.dir/out
2424

25-
# RUN: %clang -target x86_64-linux-gnu \
25+
# RUN: %clang --target=x86_64-linux-gnu \
2626
# RUN: %t.dir/foo.a %t.dir/lib/bar.a ../_start.a %t.dir/cat.a -Wl,--whole-archive,../dog.a \
2727
# RUN: -flto=thin \
2828
# RUN: -fthinlto-distributor=%python \

‎cross-project-tests/dtlto/dtlto.c‎

Lines changed: 25 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -6,44 +6,40 @@
66

77
// RUN: rm -rf %t && mkdir %t && cd %t
88

9-
// RUN: %clang -target x86_64-linux-gnu %s -shared -flto=thin \
9+
// RUN: %clang --target=x86_64-linux-gnu %s -shared -flto=thin \
1010
// RUN: -fthinlto-distributor=%python \
1111
// RUN: -Xdist %llvm_src_root/utils/dtlto/local.py \
1212
// RUN: --save-temps \
1313
// RUN: -fuse-ld=lld \
1414
// RUN: -nostdlib \
15-
// RUN: -nostartfiles \
1615
// RUN: -Wl,--save-temps \
1716
// RUN: -Werror
1817

1918
/// Check that the required output files have been created.
2019
// RUN: ls | count 13
21-
// RUN: ls | FileCheck %s --check-prefix=BITCODE
22-
// RUN: ls | FileCheck %s --check-prefix=BACKEND
23-
// RUN: ls | FileCheck %s --check-prefix=NATIVE
24-
// RUN: ls | FileCheck %s --check-prefix=LLD
25-
26-
/// Files produced by the bitcode compilation.
27-
// BITCODE: dtlto.bc
28-
// BITCODE: dtlto.i
29-
// BITCODE: dtlto.o
30-
31-
/// The DTLTO backend emits the jobs description JSON and a summary shard.
32-
// BACKEND: a.{{[0-9]+}}.dist-file.json
33-
// BACKEND: dtlto.{{[0-9]+}}.{{[0-9]+}}.native.o.thinlto.bc{{$}}
34-
35-
/// Native object output file for dtlto.o.
36-
// NATIVE: dtlto.{{[0-9]+}}.{{[0-9]+}}.native.o{{$}}
37-
/// linked ELF.
38-
// LLD: a.out{{$}}
39-
40-
/// save-temps incremental files for a.out.
41-
/// TODO: Perhaps we should suppress some of the linker hooks for DTLTO.
42-
// LLD: a.out.0.0.preopt.bc{{$}}
43-
// LLD: a.out.0.2.internalize.bc{{$}}
44-
// LLD: a.out.index.bc{{$}}
45-
// LLD: a.out.index.dot{{$}}
46-
// LLD: a.out.lto.dtlto.o{{$}}
47-
// LLD: a.out.resolution.txt{{$}}
20+
// RUN: ls | FileCheck %s
21+
22+
/// Produced by the bitcode compilation.
23+
// CHECK-DAG: {{^}}dtlto.bc{{$}}
24+
// CHECK-DAG: {{^}}dtlto.i{{$}}
25+
// CHECK-DAG: {{^}}dtlto.o{{$}}
26+
27+
/// A jobs description JSON and a summary shard is emitted for DTLTO.
28+
// CHECK-DAG: {{^}}a.[[#]].dist-file.json{{$}}
29+
// CHECK-DAG: {{^}}dtlto.[[#]].[[#]].native.o.thinlto.bc{{$}}
30+
31+
/// The backend compilation produces a native object output file for dtlto.o.
32+
// CHECK-DAG: dtlto.[[#]].[[#]].native.o{{$}}
33+
34+
/// Linked ELF.
35+
// CHECK-DAG: {{^}}a.out{{$}}
36+
37+
/// --save-temps incremental files for a.out.
38+
// CHECK-DAG: {{^}}a.out.lto.dtlto.o{{$}}
39+
// CHECK-DAG: {{^}}a.out.0.0.preopt.bc{{$}}
40+
// CHECK-DAG: {{^}}a.out.0.2.internalize.bc{{$}}
41+
// CHECK-DAG: {{^}}a.out.index.bc{{$}}
42+
// CHECK-DAG: {{^}}a.out.index.dot{{$}}
43+
// CHECK-DAG: {{^}}a.out.resolution.txt{{$}}
4844

4945
int _start() { return 0; }

‎lld/test/COFF/dtlto.test‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,6 @@
1515
# RUN: -mllvm:-thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \
1616
# RUN: --thinlto-remote-opt-tool=my_clang.exe" > l.rsp
1717

18-
## Command line arguments that should affect codegen.
19-
# RUN: echo "/lto-pgo-warn-mismatch:no \
20-
# RUN: /lto-sample-profile:foo.ll \
21-
# RUN: -mllvm:-thinlto-distributor-arg=bibbity=10 \
22-
# RUN: -mllvm:-thinlto-remote-opt-tool-arg=bobbity=20" > o.rsp
23-
2418
## Show that command line arguments have the desired effect when specified and
2519
## that the effect is not present otherwise.
2620
# RUN: not lld-link @l.rsp @o.rsp 2>&1 | FileCheck %s --check-prefixes=ERR,OPT,BOTH
@@ -40,6 +34,13 @@
4034

4135
# ERR: lld-link: error: DTLTO backend compilation: cannot open native object file:
4236

37+
## Command line arguments that should affect codegen.
38+
#--- o.rsp
39+
/lto-pgo-warn-mismatch:no
40+
/lto-sample-profile:foo.ll
41+
-mllvm:-thinlto-distributor-arg=bibbity=10
42+
-mllvm:-thinlto-remote-opt-tool-arg=bobbity=20
43+
4344
#--- foo.ll
4445
target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
4546
target triple = "x86_64-pc-windows-msvc"

‎lld/test/ELF/dtlto/dtlto.test‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,6 @@
1515
# RUN: -mllvm -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \
1616
# RUN: --thinlto-remote-opt-tool=my_clang.exe" > l.rsp
1717

18-
## Command line arguments that should affect codegen.
19-
# RUN: echo "--lto-O3 \
20-
# RUN: --lto-CGO2 \
21-
# RUN: --no-lto-pgo-warn-mismatch \
22-
# RUN: --lto-sample-profile=foo.ll \
23-
# RUN: -mllvm -thinlto-distributor-arg=bibbity=10 \
24-
# RUN: -mllvm -thinlto-remote-opt-tool-arg=bobbity=20" > o.rsp
25-
2618
## Show that command line arguments have the desired effect when specified and
2719
## that the effect is not present otherwise.
2820
# RUN: not ld.lld @l.rsp @o.rsp 2>&1 | FileCheck %s --check-prefixes=ERR,OPT,BOTH
@@ -43,6 +35,16 @@
4335

4436
# ERR: ld.lld: error: DTLTO backend compilation: cannot open native object file:
4537

38+
39+
## Command line arguments that should affect codegen.
40+
#--- o.rsp
41+
--lto-O3
42+
--lto-CGO2
43+
--no-lto-pgo-warn-mismatch
44+
--lto-sample-profile=foo.ll
45+
-mllvm -thinlto-distributor-arg=bibbity=10
46+
-mllvm -thinlto-remote-opt-tool-arg=bobbity=20
47+
4648
#--- foo.ll
4749
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
4850
target triple = "x86_64-unknown-linux-gnu"

‎lld/test/ELF/dtlto/imports.test‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
## Check that DTLTO handles imports files correctly.
44

5-
# RUN: rm -rf %t.dir && split-file %s %t.dir && cd %t.dir
5+
# RUN: rm -rf %t && split-file %s %t && cd %t
66

77
## Compile bitcode.
88
# RUN: opt -module-summary 0.ll -o 0.o -O2

‎llvm/docs/DTLTO.rst‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ Below is an example of a JSON job file for backend compilation of the module
147147
"-o", ["primary_output", 0],
148148
"-c", "-x", "ir", ["primary_input", 0],
149149
["summary_index", "-fthinlto-index=", 0],
150-
"-target", "x86_64-sie-ps5"
150+
"--target=x86_64-sie-ps5"
151151
]
152152
},
153153
"jobs": [
@@ -194,7 +194,7 @@ The example above generates the following backend compilation command for
194194
195195
/usr/local/clang -O3 -fprofile-sample-use=my.profdata \
196196
-o dtlto.1.51232.native.o -c -x ir dtlto.o \
197-
-fthinlto-index=dtlto.1.51232.native.o.thinlto.bc -target x86_64-sie-ps5
197+
-fthinlto-index=dtlto.1.51232.native.o.thinlto.bc --target=x86_64-sie-ps5
198198
199199
This expansion scheme allows the remote optimization tool to be changed without
200200
updating the distributors. For example, if the "args" field in the above example

‎llvm/lib/LTO/LTO.cpp‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2388,8 +2388,7 @@ class OutOfProcessThinBackend : public CGThinBackend {
23882388

23892389
// Reference to Job::SummaryIndexPath.
23902390
JOS.value(Array{"summary_index", "-fthinlto-index=", 0});
2391-
JOS.value("-target");
2392-
JOS.value(Jobs.front().Triple);
2391+
JOS.value(Saver.save("--target=" + Twine(Jobs.front().Triple)));
23932392
});
23942393
});
23952394
JOS.attributeArray("jobs", [&]() {
@@ -2426,6 +2425,12 @@ class OutOfProcessThinBackend : public CGThinBackend {
24262425
}
24272426

24282427
Error wait() override {
2428+
// Wait for the information on the required backend compilations to be
2429+
// gathered.
2430+
BackendThreadPool.wait();
2431+
if (Err)
2432+
return std::move(*Err);
2433+
24292434
auto CleanPerJobFiles = llvm::make_scope_exit([&] {
24302435
if (!SaveTemps)
24312436
for (auto &Job : Jobs) {
@@ -2448,12 +2453,6 @@ class OutOfProcessThinBackend : public CGThinBackend {
24482453

24492454
buildCommonRemoteOptToolOptions();
24502455

2451-
// Wait for the information on the required backend compilations to be
2452-
// gathered.
2453-
BackendThreadPool.wait();
2454-
if (Err)
2455-
return std::move(*Err);
2456-
24572456
SString JsonFile = sys::path::parent_path(LinkerOutputFile);
24582457
sys::path::append(JsonFile, sys::path::stem(LinkerOutputFile) + "." + UID +
24592458
".dist-file.json");

0 commit comments

Comments
 (0)