Skip to content

Commit e36bc7c

Browse files
authored
Merge branch 'llvm:main' into kcfi-salt
2 parents bbce666 + c73e5e3 commit e36bc7c

File tree

725 files changed

+22593
-10356
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

725 files changed

+22593
-10356
lines changed

.ci/compute_projects.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@
7777
DEPENDENT_RUNTIMES_TO_TEST = {
7878
"clang": {"compiler-rt"},
7979
"clang-tools-extra": {"libc"},
80+
"libc": {"libc"},
8081
".ci": {"compiler-rt", "libc"},
8182
}
8283
DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG = {

.ci/compute_projects_test.py

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ def test_top_level_file(self):
187187
self.assertEqual(env_variables["runtimes_check_targets"], "")
188188
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
189189

190-
def test_exclude_runtiems_in_projects(self):
190+
def test_exclude_libcxx_in_projects(self):
191191
env_variables = compute_projects.get_env_variables(
192192
["libcxx/CMakeLists.txt"], "Linux"
193193
)
@@ -197,6 +197,16 @@ def test_exclude_runtiems_in_projects(self):
197197
self.assertEqual(env_variables["runtimes_check_targets"], "")
198198
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
199199

200+
def test_include_libc_in_runtimes(self):
201+
env_variables = compute_projects.get_env_variables(
202+
["libc/CMakeLists.txt"], "Linux"
203+
)
204+
self.assertEqual(env_variables["projects_to_build"], "clang;lld")
205+
self.assertEqual(env_variables["project_check_targets"], "")
206+
self.assertEqual(env_variables["runtimes_to_build"], "libc")
207+
self.assertEqual(env_variables["runtimes_check_targets"], "check-libc")
208+
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
209+
200210
def test_exclude_docs(self):
201211
env_variables = compute_projects.get_env_variables(
202212
["llvm/docs/CIBestPractices.rst"], "Linux"

.github/new-prs-labeler.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -632,6 +632,10 @@ llvm:instcombine:
632632
- llvm/test/Transforms/InstCombine/**
633633
- llvm/test/Transforms/InstSimplify/**
634634

635+
llvm:vectorcombine:
636+
- llvm/lib/Transforms/Vectorize/VectorCombine.cpp
637+
- llvm/test/Transforms/VectorCombine/**
638+
635639
clangd:
636640
- clang-tools-extra/clangd/**
637641

bolt/lib/Passes/BinaryPasses.cpp

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1765,27 +1765,26 @@ Error PrintProgramStats::runOnFunctions(BinaryContext &BC) {
17651765

17661766
if (opts::ShowDensity) {
17671767
double Density = 0.0;
1768-
// Sorted by the density in descending order.
1769-
llvm::stable_sort(FuncDensityList,
1770-
[&](const std::pair<double, uint64_t> &A,
1771-
const std::pair<double, uint64_t> &B) {
1772-
if (A.first != B.first)
1773-
return A.first > B.first;
1774-
return A.second < B.second;
1775-
});
1768+
llvm::sort(FuncDensityList);
17761769

17771770
uint64_t AccumulatedSamples = 0;
1778-
uint32_t I = 0;
17791771
assert(opts::ProfileDensityCutOffHot <= 1000000 &&
17801772
"The cutoff value is greater than 1000000(100%)");
1781-
while (AccumulatedSamples <
1782-
TotalSampleCount *
1783-
static_cast<float>(opts::ProfileDensityCutOffHot) /
1784-
1000000 &&
1785-
I < FuncDensityList.size()) {
1786-
AccumulatedSamples += FuncDensityList[I].second;
1787-
Density = FuncDensityList[I].first;
1788-
I++;
1773+
// Subtract samples in zero-density functions (no fall-throughs) from
1774+
// TotalSampleCount (not used anywhere below).
1775+
for (const auto [CurDensity, CurSamples] : FuncDensityList) {
1776+
if (CurDensity != 0.0)
1777+
break;
1778+
TotalSampleCount -= CurSamples;
1779+
}
1780+
const uint64_t CutoffSampleCount =
1781+
1.f * TotalSampleCount * opts::ProfileDensityCutOffHot / 1000000;
1782+
// Process functions in decreasing density order
1783+
for (const auto [CurDensity, CurSamples] : llvm::reverse(FuncDensityList)) {
1784+
if (AccumulatedSamples >= CutoffSampleCount)
1785+
break;
1786+
AccumulatedSamples += CurSamples;
1787+
Density = CurDensity;
17891788
}
17901789
if (Density == 0.0) {
17911790
BC.errs() << "BOLT-WARNING: the output profile is empty or the "

bolt/lib/Passes/PAuthGadgetScanner.cpp

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1319,6 +1319,90 @@ shouldReportReturnGadget(const BinaryContext &BC, const MCInstReference &Inst,
13191319
return make_gadget_report(RetKind, Inst, *RetReg);
13201320
}
13211321

1322+
/// While BOLT already marks some of the branch instructions as tail calls,
1323+
/// this function tries to detect less obvious cases, assuming false positives
1324+
/// are acceptable as long as there are not too many of them.
1325+
///
1326+
/// It is possible that not all the instructions classified as tail calls by
1327+
/// this function are safe to be considered as such for the purpose of code
1328+
/// transformations performed by BOLT. The intention of this function is to
1329+
/// spot some of actually missed tail calls (and likely a number of unrelated
1330+
/// indirect branch instructions) as long as this doesn't increase the amount
1331+
/// of false positive reports unacceptably.
1332+
static bool shouldAnalyzeTailCallInst(const BinaryContext &BC,
1333+
const BinaryFunction &BF,
1334+
const MCInstReference &Inst) {
1335+
// Some BC.MIB->isXYZ(Inst) methods simply delegate to MCInstrDesc::isXYZ()
1336+
// (such as isBranch at the time of writing this comment), some don't (such
1337+
// as isCall). For that reason, call MCInstrDesc's methods explicitly when
1338+
// it is important.
1339+
const MCInstrDesc &Desc =
1340+
BC.MII->get(static_cast<const MCInst &>(Inst).getOpcode());
1341+
// Tail call should be a branch (but not necessarily an indirect one).
1342+
if (!Desc.isBranch())
1343+
return false;
1344+
1345+
// Always analyze the branches already marked as tail calls by BOLT.
1346+
if (BC.MIB->isTailCall(Inst))
1347+
return true;
1348+
1349+
// Try to also check the branches marked as "UNKNOWN CONTROL FLOW" - the
1350+
// below is a simplified condition from BinaryContext::printInstruction.
1351+
bool IsUnknownControlFlow =
1352+
BC.MIB->isIndirectBranch(Inst) && !BC.MIB->getJumpTable(Inst);
1353+
1354+
if (BF.hasCFG() && IsUnknownControlFlow)
1355+
return true;
1356+
1357+
return false;
1358+
}
1359+
1360+
static std::optional<PartialReport<MCPhysReg>>
1361+
shouldReportUnsafeTailCall(const BinaryContext &BC, const BinaryFunction &BF,
1362+
const MCInstReference &Inst, const SrcState &S) {
1363+
static const GadgetKind UntrustedLRKind(
1364+
"untrusted link register found before tail call");
1365+
1366+
if (!shouldAnalyzeTailCallInst(BC, BF, Inst))
1367+
return std::nullopt;
1368+
1369+
// Not only the set of registers returned by getTrustedLiveInRegs() can be
1370+
// seen as a reasonable target-independent _approximation_ of "the LR", these
1371+
// are *exactly* those registers used by SrcSafetyAnalysis to initialize the
1372+
// set of trusted registers on function entry.
1373+
// Thus, this function basically checks that the precondition expected to be
1374+
// imposed by a function call instruction (which is hardcoded into the target-
1375+
// specific getTrustedLiveInRegs() function) is also respected on tail calls.
1376+
SmallVector<MCPhysReg> RegsToCheck = BC.MIB->getTrustedLiveInRegs();
1377+
LLVM_DEBUG({
1378+
traceInst(BC, "Found tail call inst", Inst);
1379+
traceRegMask(BC, "Trusted regs", S.TrustedRegs);
1380+
});
1381+
1382+
// In musl on AArch64, the _start function sets LR to zero and calls the next
1383+
// stage initialization function at the end, something along these lines:
1384+
//
1385+
// _start:
1386+
// mov x30, #0
1387+
// ; ... other initialization ...
1388+
// b _start_c ; performs "exit" system call at some point
1389+
//
1390+
// As this would produce a false positive for every executable linked with
1391+
// such libc, ignore tail calls performed by ELF entry function.
1392+
if (BC.StartFunctionAddress &&
1393+
*BC.StartFunctionAddress == Inst.getFunction()->getAddress()) {
1394+
LLVM_DEBUG({ dbgs() << " Skipping tail call in ELF entry function.\n"; });
1395+
return std::nullopt;
1396+
}
1397+
1398+
// Returns at most one report per instruction - this is probably OK...
1399+
for (auto Reg : RegsToCheck)
1400+
if (!S.TrustedRegs[Reg])
1401+
return make_gadget_report(UntrustedLRKind, Inst, Reg);
1402+
1403+
return std::nullopt;
1404+
}
1405+
13221406
static std::optional<PartialReport<MCPhysReg>>
13231407
shouldReportCallGadget(const BinaryContext &BC, const MCInstReference &Inst,
13241408
const SrcState &S) {
@@ -1478,6 +1562,9 @@ void FunctionAnalysisContext::findUnsafeUses(
14781562
if (PacRetGadgetsOnly)
14791563
return;
14801564

1565+
if (auto Report = shouldReportUnsafeTailCall(BC, BF, Inst, S))
1566+
Reports.push_back(*Report);
1567+
14811568
if (auto Report = shouldReportCallGadget(BC, Inst, S))
14821569
Reports.push_back(*Report);
14831570
if (auto Report = shouldReportSigningOracle(BC, Inst, S))

bolt/lib/Rewrite/RewriteInstance.cpp

Lines changed: 15 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -626,6 +626,9 @@ Error RewriteInstance::discoverStorage() {
626626
NextAvailableAddress += BC->PageAlign;
627627
}
628628

629+
NewTextSegmentAddress = NextAvailableAddress;
630+
NewTextSegmentOffset = NextAvailableOffset;
631+
629632
if (!opts::UseGnuStack && !BC->IsLinuxKernel) {
630633
// This is where the black magic happens. Creating PHDR table in a segment
631634
// other than that containing ELF header is tricky. Some loaders and/or
@@ -652,6 +655,8 @@ Error RewriteInstance::discoverStorage() {
652655

653656
PHDRTableAddress = NextAvailableAddress;
654657
PHDRTableOffset = NextAvailableOffset;
658+
NewTextSegmentAddress = NextAvailableAddress;
659+
NewTextSegmentOffset = NextAvailableOffset;
655660

656661
// Reserve space for 3 extra pheaders.
657662
unsigned Phnum = Obj.getHeader().e_phnum;
@@ -664,14 +669,12 @@ Error RewriteInstance::discoverStorage() {
664669

665670
NextAvailableAddress += Phnum * sizeof(ELF64LEPhdrTy);
666671
NextAvailableOffset += Phnum * sizeof(ELF64LEPhdrTy);
667-
}
668672

669-
// Align at cache line.
670-
NextAvailableAddress = alignTo(NextAvailableAddress, 64);
671-
NextAvailableOffset = alignTo(NextAvailableOffset, 64);
673+
// Align at cache line.
674+
NextAvailableAddress = alignTo(NextAvailableAddress, 64);
675+
NextAvailableOffset = alignTo(NextAvailableOffset, 64);
676+
}
672677

673-
NewTextSegmentAddress = NextAvailableAddress;
674-
NewTextSegmentOffset = NextAvailableOffset;
675678
BC->LayoutStartAddress = NextAvailableAddress;
676679

677680
// Tools such as objcopy can strip section contents but leave header
@@ -4133,13 +4136,8 @@ void RewriteInstance::mapAllocatableSections(
41334136
}
41344137

41354138
if (SType == ST_READONLY) {
4136-
if (PHDRTableAddress) {
4137-
// Segment size includes the size of the PHDR area.
4138-
NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress;
4139-
} else if (NewTextSegmentAddress) {
4140-
// Existing PHDR table would be updated.
4139+
if (NewTextSegmentAddress)
41414140
NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
4142-
}
41434141
} else if (SType == ST_READWRITE) {
41444142
NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
41454143
// Restore NextAvailableAddress if no new writable sections
@@ -4186,9 +4184,7 @@ void RewriteInstance::patchELFPHDRTable() {
41864184
// NOTE Currently .eh_frame_hdr appends to the last segment, recalculate
41874185
// last segments size based on the NextAvailableAddress variable.
41884186
if (!NewWritableSegmentSize) {
4189-
if (PHDRTableAddress)
4190-
NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress;
4191-
else if (NewTextSegmentAddress)
4187+
if (NewTextSegmentAddress)
41924188
NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
41934189
} else {
41944190
NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
@@ -4201,15 +4197,9 @@ void RewriteInstance::patchELFPHDRTable() {
42014197
SmallVector<ELF64LEPhdrTy, 3> NewPhdrs;
42024198
ELF64LEPhdrTy NewPhdr;
42034199
NewPhdr.p_type = ELF::PT_LOAD;
4204-
if (PHDRTableAddress) {
4205-
NewPhdr.p_offset = PHDRTableOffset;
4206-
NewPhdr.p_vaddr = PHDRTableAddress;
4207-
NewPhdr.p_paddr = PHDRTableAddress;
4208-
} else {
4209-
NewPhdr.p_offset = NewTextSegmentOffset;
4210-
NewPhdr.p_vaddr = NewTextSegmentAddress;
4211-
NewPhdr.p_paddr = NewTextSegmentAddress;
4212-
}
4200+
NewPhdr.p_offset = NewTextSegmentOffset;
4201+
NewPhdr.p_vaddr = NewTextSegmentAddress;
4202+
NewPhdr.p_paddr = NewTextSegmentAddress;
42134203
NewPhdr.p_filesz = NewTextSegmentSize;
42144204
NewPhdr.p_memsz = NewTextSegmentSize;
42154205
NewPhdr.p_flags = ELF::PF_X | ELF::PF_R;
@@ -4270,7 +4260,7 @@ void RewriteInstance::patchELFPHDRTable() {
42704260
};
42714261

42724262
auto writeNewSegmentPhdrs = [&]() {
4273-
if (PHDRTableAddress || NewTextSegmentSize) {
4263+
if (NewTextSegmentSize) {
42744264
SmallVector<ELF64LE::Phdr, 3> NewPhdrs = createNewPhdrs();
42754265
OS.write(reinterpret_cast<const char *>(NewPhdrs.data()),
42764266
sizeof(ELF64LE::Phdr) * NewPhdrs.size());

bolt/test/X86/zero-density.s

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
## Check that trampoline functions are excluded from density computation.
2+
3+
# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o
4+
# RUN: ld.lld %t.o -o %t
5+
# RUN: link_fdata %s %t %t.preagg PREAGG
6+
# RUN: llvm-strip -NLjmp %t
7+
# RUN: perf2bolt %t -p %t.preagg --pa -o %t.fdata | FileCheck %s
8+
# CHECK: Functions with density >= {{.*}} account for 99.00% total sample counts.
9+
# CHECK-NOT: the output profile is empty or the --profile-density-cutoff-hot option is set too low.
10+
11+
.text
12+
.globl trampoline
13+
trampoline:
14+
mov main,%rax
15+
jmpq *%rax
16+
.size trampoline,.-trampoline
17+
# PREAGG: f #trampoline# #trampoline# 2
18+
19+
.globl main
20+
main:
21+
.cfi_startproc
22+
vmovaps %zmm31,%zmm3
23+
24+
add $0x4,%r9
25+
add $0x40,%r10
26+
dec %r14
27+
Ljmp:
28+
jne main
29+
# PREAGG: T #Ljmp# #main# #Ljmp# 10
30+
ret
31+
.cfi_endproc
32+
.size main,.-main

0 commit comments

Comments
 (0)