Skip to content

Commit 3ee54a6

Browse files
romanova-ekaterinaromanova-ekaterina
authored
[DTLTO] [LLVM] Initial DTLTO cache implementation (llvm#156433)
This patch implements DTLTO cache. DTLTO cache is implemented the same way as ThinLTO cache. In fact the same class Cache is used for both of them. Because parameters for codegen are different for DTLTO and ThinLTO (DTLTO codegen is done by invoking clang and its codegen parameters are not fully synchronized with codegen parameters used by LTO backend). The object files generated by DTLTO and ThinLTO might be different and shouldn't be mixed. If ThinLTO and DTLTO share the same cache directory, the cache file won't interfere with each other. I added a couple of test files in cross-project-test/dtlto directory, but if more tests are required for initial implementation, I could add them.
1 parent 82ba3f5 commit 3ee54a6

File tree

5 files changed

+371
-37
lines changed

5 files changed

+371
-37
lines changed

‎cross-project-tests/dtlto/dtlto-cache.test‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
REQUIRES: x86-registered-target, ld.lld
2+
3+
# Show that the ThinLTO cache works with DTLTO.
4+
5+
RUN: rm -rf %t && split-file %s %t && cd %t
6+
7+
# Compile source files into bitcode files.
8+
RUN: %clang -O2 --target=x86_64-linux-gnu -flto=thin -c foo.c main.c
9+
10+
# Execute the linker and check that the cache is populated.
11+
RUN: %clang -O2 --target=x86_64-linux-gnu -Werror -flto=thin -fuse-ld=lld -nostdlib -e main \
12+
RUN: main.o foo.o -o populate1.elf \
13+
RUN: -Wl,--thinlto-distributor=%python \
14+
RUN: -Wl,--thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
15+
RUN: -Wl,--thinlto-remote-compiler=%clang \
16+
RUN: -Wl,--thinlto-cache-dir=cache.dir \
17+
RUN: -Wl,--save-temps
18+
19+
# Check that there are two backend compilation jobs occurred.
20+
RUN: grep -wo args populate1.*.dist-file.json | wc -l | grep -qx 3
21+
RUN: ls cache.dir/llvmcache.timestamp
22+
RUN: ls cache.dir | count 3
23+
24+
# Execute the linker again and check that a fully populated cache is used correctly,
25+
# i.e., no additional cache entries are created for cache hits.
26+
RUN: %clang -O2 --target=x86_64-linux-gnu -Werror -flto=thin -fuse-ld=lld -nostdlib -e main \
27+
RUN: main.o foo.o -o populate2.elf \
28+
RUN: -Wl,--thinlto-distributor=%python \
29+
RUN: -Wl,--thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
30+
RUN: -Wl,--thinlto-remote-compiler=%clang \
31+
RUN: -Wl,--thinlto-cache-dir=cache.dir \
32+
RUN: -Wl,--save-temps
33+
34+
# Check that there are no backend compilation jobs occurred.
35+
RUN: grep -wo args populate2.*.dist-file.json | wc -l | grep -qx 1
36+
RUN: ls cache.dir | count 3
37+
38+
RUN: %clang -O0 --target=x86_64-linux-gnu -flto=thin -c foo.c -o foo.O0.o
39+
RUN: %clang -O0 --target=x86_64-linux-gnu -flto=thin -c main.c -o main.O0.o
40+
41+
# Execute the linker again and check that the cache is populated correctly when there
42+
# are no cache hits but there are existing cache entries.
43+
# As a side effect, this also verifies that the optimization level is considered when
44+
# evaluating the cache entry key.
45+
46+
RUN: %clang -O2 --target=x86_64-linux-gnu -Werror -flto=thin -fuse-ld=lld -nostdlib -e main \
47+
RUN: main.O0.o foo.O0.o -o populate3.elf \
48+
RUN: -Wl,--thinlto-distributor=%python \
49+
RUN: -Wl,--thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
50+
RUN: -Wl,--thinlto-remote-compiler=%clang \
51+
RUN: -Wl,--thinlto-cache-dir=cache.dir \
52+
RUN: -Wl,--save-temps
53+
54+
# Check that there are two new backend compilation jobs occurred.
55+
RUN: grep -wo args populate3.*.dist-file.json | wc -l | grep -qx 3
56+
RUN: ls cache.dir | count 5
57+
58+
RUN: %clang -O2 --target=x86_64-linux-gnu -flto=thin -c main-partial.c
59+
60+
# Execute the linker and check that everything works correctly with the partially populated cache;
61+
# One more cache entry should be generated after this run.
62+
63+
RUN: %clang -O2 --target=x86_64-linux-gnu -Werror -flto=thin -fuse-ld=lld -nostdlib -e main \
64+
RUN: main-partial.o foo.o -o main-partial.elf \
65+
RUN: -Wl,--thinlto-distributor=%python \
66+
RUN: -Wl,--thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
67+
RUN: -Wl,--thinlto-remote-compiler=%clang \
68+
RUN: -Wl,--thinlto-cache-dir=cache.dir \
69+
RUN: -Wl,--save-temps
70+
71+
# Check that there is one new backend compilation jobs occurred.
72+
RUN: grep -wo args main-partial.*.dist-file.json | wc -l | grep -qx 2
73+
RUN: ls cache.dir | count 6
74+
75+
#--- foo.c
76+
volatile int foo_int;
77+
__attribute__((retain)) int foo(int x) { return x + foo_int; }
78+
79+
#--- main.c
80+
extern int foo(int x);
81+
__attribute__((retain)) int main(int argc, char** argv) {
82+
return foo(argc);
83+
}
84+
85+
#--- main-partial.c
86+
extern int foo(int x);
87+
__attribute__((retain)) int main(int argc, char** argv) {
88+
return foo(argc+1);
89+
}

‎cross-project-tests/dtlto/dtlto-thinlto-cache.test‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
REQUIRES: x86-registered-target, ld.lld
2+
3+
# This test verifies that a cache populated by an in-process ThinLTO codegen is
4+
# not reused by an out-of-process (DTLTO) codegen and vice versa.
5+
6+
RUN: rm -rf %t && split-file %s %t && cd %t
7+
8+
# Compile source files into bitcode files.
9+
RUN: %clang -O2 --target=x86_64-linux-gnu -flto=thin -c foo.c main.c
10+
11+
# Execute the linker and check that in-process ThinLTO cache is populated.
12+
RUN: %clang -O2 --target=x86_64-linux-gnu -Werror -flto=thin -fuse-ld=lld -nostdlib -e main \
13+
RUN: main.o foo.o -o main.elf \
14+
RUN: -Wl,--thinlto-cache-dir=cache.dir \
15+
RUN: -Wl,--save-temps
16+
17+
RUN: ls cache.dir/llvmcache.timestamp
18+
RUN: ls cache.dir | count 3
19+
20+
# Execute the linker and check that out-of-process codegen (DTLTO) adds
21+
# additional entries to the cache, implying that in-process and
22+
# out-of-process codegens do not share cache entries.
23+
RUN: %clang -O2 --target=x86_64-linux-gnu -Werror -flto=thin -fuse-ld=lld -nostdlib -e main \
24+
RUN: main.o foo.o -o populate1.elf \
25+
RUN: -Wl,--thinlto-distributor=%python \
26+
RUN: -Wl,--thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
27+
RUN: -Wl,--thinlto-remote-compiler=%clang \
28+
RUN: -Wl,--thinlto-cache-dir=cache.dir \
29+
RUN: -Wl,--save-temps
30+
31+
# Check that there are two backend compilation jobs occurred.
32+
RUN: grep -wo args populate1.*.dist-file.json | wc -l | grep -qx 3
33+
RUN: ls cache.dir | count 5
34+
35+
# Clean up cache directory.
36+
RUN: rm -rf cache.dir
37+
38+
# Execute the linker and check that out-of-process (DTLTO) cache is populated.
39+
RUN: %clang -O2 --target=x86_64-linux-gnu -Werror -flto=thin -fuse-ld=lld -nostdlib -e main \
40+
RUN: main.o foo.o -o populate2.elf \
41+
RUN: -Wl,--thinlto-distributor=%python \
42+
RUN: -Wl,--thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
43+
RUN: -Wl,--thinlto-remote-compiler=%clang \
44+
RUN: -Wl,--thinlto-cache-dir=cache.dir \
45+
RUN: -Wl,--save-temps
46+
47+
# Check that there are two backend compilation jobs occurred.
48+
RUN: grep -wo args populate2.*.dist-file.json | wc -l | grep -qx 3
49+
RUN: ls cache.dir/llvmcache.timestamp
50+
RUN: ls cache.dir | count 3
51+
52+
# Execute the linker and check that in-process codegen adds additional entries
53+
# to the cache, implying that in-process and out-of-process codegens do
54+
# not share cache entries.
55+
RUN: %clang -O2 --target=x86_64-linux-gnu -Werror -flto=thin -fuse-ld=lld -nostdlib -e main \
56+
RUN: main.o foo.o -o main.elf \
57+
RUN: -Wl,--thinlto-cache-dir=cache.dir \
58+
RUN: -Wl,--save-temps
59+
60+
RUN: ls cache.dir | count 5
61+
62+
#--- foo.c
63+
volatile int foo_int;
64+
__attribute__((retain)) int foo(int x) { return x + foo_int; }
65+
66+
#--- main.c
67+
extern int foo(int x);
68+
__attribute__((retain)) int main(int argc, char** argv) {
69+
return foo(argc);
70+
}

‎llvm/include/llvm/LTO/Config.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,11 @@ struct Config {
9494
/// need to create copies, so it can set this field to false.
9595
bool KeepSymbolNameCopies = true;
9696

97+
/// This flag is used as one of parameters to calculate cache entries and to
98+
/// ensure that in-process cache and out-of-process (DTLTO) cache are
99+
/// distinguished.
100+
mutable bool Dtlto = 0;
101+
97102
/// Allows non-imported definitions to get the potentially more constraining
98103
/// visibility from the prevailing definition. FromPrevailing is the default
99104
/// because it works for many binary formats. ELF can use the more optimized

0 commit comments

Comments
 (0)