Skip to content

Commit 0c4c05e

Browse files
ronliebronlieb
authored
merge main into amd-staging (#543)
2 parents bf50425 + f1e4266 commit 0c4c05e

File tree

13 files changed

+286
-73
lines changed

13 files changed

+286
-73
lines changed

‎bolt/include/bolt/Rewrite/MetadataRewriters.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,14 @@ class BinaryContext;
1919

2020
// The list of rewriter build functions.
2121

22-
std::unique_ptr<MetadataRewriter> createLinuxKernelRewriter(BinaryContext &);
23-
2422
std::unique_ptr<MetadataRewriter> createBuildIDRewriter(BinaryContext &);
2523

24+
std::unique_ptr<MetadataRewriter> createLinuxKernelRewriter(BinaryContext &);
25+
2626
std::unique_ptr<MetadataRewriter> createPseudoProbeRewriter(BinaryContext &);
2727

28+
std::unique_ptr<MetadataRewriter> createRSeqRewriter(BinaryContext &);
29+
2830
std::unique_ptr<MetadataRewriter> createSDTRewriter(BinaryContext &);
2931

3032
std::unique_ptr<MetadataRewriter> createGNUPropertyRewriter(BinaryContext &);

‎bolt/lib/Rewrite/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ add_llvm_library(LLVMBOLTRewrite
2424
BuildIDRewriter.cpp
2525
PseudoProbeRewriter.cpp
2626
RewriteInstance.cpp
27+
RSeqRewriter.cpp
2728
SDTRewriter.cpp
2829
GNUPropertyRewriter.cpp
2930

‎bolt/lib/Rewrite/RSeqRewriter.cpp‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
//===- bolt/Rewrite/RSeqRewriter.cpp --------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
//
9+
// Basic support for restartable sequences used by tcmalloc. Prevent critical
10+
// section overrides by ignoring optimizations in containing functions.
11+
//
12+
// References:
13+
// * https://google.github.io/tcmalloc/rseq.html
14+
// * tcmalloc/internal/percpu_rseq_x86_64.S
15+
//
16+
//===----------------------------------------------------------------------===//
17+
18+
#include "bolt/Core/BinaryFunction.h"
19+
#include "bolt/Rewrite/MetadataRewriter.h"
20+
#include "bolt/Rewrite/MetadataRewriters.h"
21+
#include "llvm/Support/Errc.h"
22+
23+
using namespace llvm;
24+
using namespace bolt;
25+
26+
namespace {
27+
28+
class RSeqRewriter final : public MetadataRewriter {
29+
public:
30+
RSeqRewriter(StringRef Name, BinaryContext &BC)
31+
: MetadataRewriter(Name, BC) {}
32+
33+
Error preCFGInitializer() override {
34+
for (const BinarySection &Section : BC.allocatableSections()) {
35+
if (Section.getName() != "__rseq_cs")
36+
continue;
37+
38+
auto handleRelocation = [&](const Relocation &Rel, bool IsDynamic) {
39+
BinaryFunction *BF = nullptr;
40+
if (Rel.Symbol)
41+
BF = BC.getFunctionForSymbol(Rel.Symbol);
42+
else if (Relocation::isRelative(Rel.Type))
43+
BF = BC.getBinaryFunctionContainingAddress(Rel.Addend);
44+
45+
if (!BF) {
46+
BC.errs() << "BOLT-WARNING: no function found matching "
47+
<< (IsDynamic ? "dynamic " : "")
48+
<< "relocation in __rseq_cs\n";
49+
} else if (!BF->isIgnored()) {
50+
BC.outs() << "BOLT-INFO: restartable sequence reference detected in "
51+
<< *BF << ". Function will not be optimized\n";
52+
BF->setIgnored();
53+
}
54+
};
55+
56+
for (const Relocation &Rel : Section.dynamicRelocations())
57+
handleRelocation(Rel, /*IsDynamic*/ true);
58+
59+
for (const Relocation &Rel : Section.relocations())
60+
handleRelocation(Rel, /*IsDynamic*/ false);
61+
}
62+
63+
return Error::success();
64+
}
65+
};
66+
67+
} // namespace
68+
69+
std::unique_ptr<MetadataRewriter>
70+
llvm::bolt::createRSeqRewriter(BinaryContext &BC) {
71+
return std::make_unique<RSeqRewriter>("rseq-cs-rewriter", BC);
72+
}

‎bolt/lib/Rewrite/RewriteInstance.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3346,6 +3346,8 @@ void RewriteInstance::initializeMetadataManager() {
33463346

33473347
MetadataManager.registerRewriter(createPseudoProbeRewriter(*BC));
33483348

3349+
MetadataManager.registerRewriter(createRSeqRewriter(*BC));
3350+
33493351
MetadataManager.registerRewriter(createSDTRewriter(*BC));
33503352

33513353
MetadataManager.registerRewriter(createGNUPropertyRewriter(*BC));

‎bolt/test/X86/rseq.s‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
## Check that llvm-bolt avoids optimization of functions referenced from
2+
## __rseq_cs section, i.e. containing critical sections and abort handlers used
3+
## by restartable sequences in tcmalloc.
4+
5+
# RUN: %clang %cflags %s -o %t -nostdlib -no-pie -Wl,-q
6+
# RUN: llvm-bolt %t -o %t.bolt --print-cfg 2>&1 | FileCheck %s
7+
# RUN: %clang %cflags %s -o %t.pie -nostdlib -pie -Wl,-q
8+
# RUN: llvm-bolt %t.pie -o %t.pie.bolt 2>&1 | FileCheck %s
9+
10+
# CHECK: restartable sequence reference detected in _start
11+
# CHECK: restartable sequence reference detected in __rseq_abort
12+
13+
## Force relocations against .text
14+
.text
15+
.reloc 0, R_X86_64_NONE
16+
17+
.global _start
18+
.type _start, %function
19+
_start:
20+
pushq %rbp
21+
mov %rsp, %rbp
22+
.L1:
23+
pop %rbp
24+
.L2:
25+
retq
26+
.size _start, .-_start
27+
28+
.section __rseq_abort, "ax"
29+
## Signature for rseq abort IP. Unmarked in the symbol table.
30+
.byte 0x0f, 0x1f, 0x05
31+
.long 0x42424242
32+
.L3:
33+
jmp .L2
34+
35+
.section __rseq_cs, "aw"
36+
.balign 32
37+
.quad .L1
38+
.quad .L3

‎lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/TestLocationsAfterRebuild.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
import lldb
88
import lldbsuite.test.lldbutil as lldbutil
99
from lldbsuite.test.lldbtest import *
10-
from lldbsuite.test.decorators import skipIfWindows
10+
from lldbsuite.test.decorators import skipIfWindows, skipIfRemote
1111
import os
1212

1313

@@ -19,6 +19,7 @@ class TestLocationsAfterRebuild(TestBase):
1919

2020
# On Windows we cannot remove a file that lldb is debugging.
2121
@skipIfWindows
22+
@skipIfRemote
2223
def test_remaining_location_spec(self):
2324
"""If we rebuild a couple of times some of the old locations
2425
get removed. Make sure the command-line breakpoint id

‎llvm/lib/Transforms/InstCombine/InstructionCombining.cpp‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5627,8 +5627,15 @@ bool InstCombinerImpl::run() {
56275627

56285628
for (Use &U : I->uses()) {
56295629
User *User = U.getUser();
5630-
if (User->isDroppable())
5631-
continue;
5630+
if (User->isDroppable()) {
5631+
// Do not sink if there are dereferenceable assumes that would be
5632+
// removed.
5633+
auto II = dyn_cast<IntrinsicInst>(User);
5634+
if (II->getIntrinsicID() != Intrinsic::assume ||
5635+
!II->getOperandBundle("dereferenceable"))
5636+
continue;
5637+
}
5638+
56325639
if (NumUsers > MaxSinkNumUsers)
56335640
return std::nullopt;
56345641

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8157,9 +8157,10 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
81578157
return new VPWidenSelectRecipe(*cast<SelectInst>(Instr), R->operands());
81588158

81598159
if (Instruction::isCast(VPI->getOpcode())) {
8160+
auto *CastR = cast<VPInstructionWithType>(R);
81608161
auto *CI = cast<CastInst>(Instr);
81618162
return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(0),
8162-
CI->getType(), *CI);
8163+
CastR->getResultType(), *CI);
81638164
}
81648165

81658166
return tryToWiden(VPI);

‎llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "llvm/Analysis/LoopInfo.h"
2121
#include "llvm/Analysis/LoopIterator.h"
2222
#include "llvm/Analysis/ScalarEvolution.h"
23+
#include "llvm/IR/InstrTypes.h"
2324
#include "llvm/IR/MDBuilder.h"
2425

2526
#define DEBUG_TYPE "vplan"
@@ -233,10 +234,15 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
233234
for (Value *Op : Inst->operands())
234235
VPOperands.push_back(getOrCreateVPOperand(Op));
235236

236-
// Build VPInstruction for any arbitrary Instruction without specific
237-
// representation in VPlan.
238-
NewR = cast<VPInstruction>(
239-
VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst));
237+
if (auto *CI = dyn_cast<CastInst>(Inst)) {
238+
NewR = VPIRBuilder.createScalarCast(CI->getOpcode(), VPOperands[0],
239+
CI->getType(), CI->getDebugLoc());
240+
NewR->setUnderlyingValue(CI);
241+
} else {
242+
// Build VPInstruction for any arbitrary Instruction without specific
243+
// representation in VPlan.
244+
NewR = VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst);
245+
}
240246
}
241247

242248
IRDef2VPValue[Inst] = NewR;

‎llvm/test/Transforms/InstCombine/sink-dereferenceable-assume.ll‎

Lines changed: 98 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,44 @@
11
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
22
; RUN: opt -p instcombine -S %s | FileCheck %s
33

4-
define i64 @test_sink_with_dereferenceable_assume(ptr %p, ptr %q, i1 %cond) {
5-
; CHECK-LABEL: define i64 @test_sink_with_dereferenceable_assume(
6-
; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[COND:%.*]]) {
4+
define i64 @test_dereferenceable_assume(ptr %p, ptr %q, i1 %c.0) {
5+
; CHECK-LABEL: define i64 @test_dereferenceable_assume(
6+
; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C_0:%.*]]) {
77
; CHECK-NEXT: [[ENTRY:.*:]]
8-
; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
8+
; CHECK-NEXT: [[P_INT:%.*]] = ptrtoint ptr [[P]] to i64
9+
; CHECK-NEXT: [[Q_INT:%.*]] = ptrtoint ptr [[Q]] to i64
10+
; CHECK-NEXT: [[DIFF:%.*]] = sub i64 [[Q_INT]], [[P_INT]]
11+
; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[P]], i64 [[DIFF]]) ]
12+
; CHECK-NEXT: br i1 [[C_0]], label %[[THEN:.*]], label %[[ELSE:.*]]
13+
; CHECK: [[THEN]]:
14+
; CHECK-NEXT: ret i64 [[DIFF]]
15+
; CHECK: [[ELSE]]:
16+
; CHECK-NEXT: ret i64 0
17+
;
18+
entry:
19+
%p_int = ptrtoint ptr %p to i64
20+
%q_int = ptrtoint ptr %q to i64
21+
%diff = sub i64 %q_int, %p_int
22+
call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %p, i64 %diff) ]
23+
br i1 %c.0, label %then, label %else
24+
25+
then:
26+
ret i64 %diff
27+
28+
else:
29+
ret i64 0
30+
}
31+
32+
define i64 @test_sink_with_dereferenceable_assume_same_block_as_user(ptr %p, ptr %q, i1 %c.0) {
33+
; CHECK-LABEL: define i64 @test_sink_with_dereferenceable_assume_same_block_as_user(
34+
; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C_0:%.*]]) {
35+
; CHECK-NEXT: [[ENTRY:.*:]]
36+
; CHECK-NEXT: br i1 [[C_0]], label %[[THEN:.*]], label %[[ELSE:.*]]
937
; CHECK: [[THEN]]:
1038
; CHECK-NEXT: [[Q_INT:%.*]] = ptrtoint ptr [[Q]] to i64
1139
; CHECK-NEXT: [[P_INT:%.*]] = ptrtoint ptr [[P]] to i64
1240
; CHECK-NEXT: [[DIFF:%.*]] = sub i64 [[Q_INT]], [[P_INT]]
41+
; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[P]], i64 [[DIFF]]) ]
1342
; CHECK-NEXT: ret i64 [[DIFF]]
1443
; CHECK: [[ELSE]]:
1544
; CHECK-NEXT: ret i64 0
@@ -18,14 +47,77 @@ entry:
1847
%p_int = ptrtoint ptr %p to i64
1948
%q_int = ptrtoint ptr %q to i64
2049
%diff = sub i64 %q_int, %p_int
21-
call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %p, i64 %diff) ]
22-
br i1 %cond, label %then, label %else
50+
br i1 %c.0, label %then, label %else
2351

2452
then:
53+
call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %p, i64 %diff) ]
2554
ret i64 %diff
2655

2756
else:
2857
ret i64 0
2958
}
3059

60+
define i64 @test_sink_with_multiple_users_dominated_by_deref(ptr %p, ptr %q, i1 %c.0, i1 %c.1) {
61+
; CHECK-LABEL: define i64 @test_sink_with_multiple_users_dominated_by_deref(
62+
; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C_0:%.*]], i1 [[C_1:%.*]]) {
63+
; CHECK-NEXT: [[ENTRY:.*:]]
64+
; CHECK-NEXT: [[P_INT:%.*]] = ptrtoint ptr [[P]] to i64
65+
; CHECK-NEXT: [[Q_INT:%.*]] = ptrtoint ptr [[Q]] to i64
66+
; CHECK-NEXT: [[DIFF:%.*]] = sub i64 [[Q_INT]], [[P_INT]]
67+
; CHECK-NEXT: br i1 [[C_0]], label %[[THEN:.*]], label %[[ELSE:.*]]
68+
; CHECK: [[THEN]]:
69+
; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[P]], i64 [[DIFF]]) ]
70+
; CHECK-NEXT: br i1 [[C_1]], label %[[THEN_2:.*]], label %[[ELSE]]
71+
; CHECK: [[THEN_2]]:
72+
; CHECK-NEXT: [[DOUBLED:%.*]] = shl i64 [[DIFF]], 1
73+
; CHECK-NEXT: ret i64 [[DOUBLED]]
74+
; CHECK: [[ELSE]]:
75+
; CHECK-NEXT: ret i64 0
76+
;
77+
entry:
78+
%p_int = ptrtoint ptr %p to i64
79+
%q_int = ptrtoint ptr %q to i64
80+
%diff = sub i64 %q_int, %p_int
81+
br i1 %c.0, label %then, label %else
82+
83+
then:
84+
call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %p, i64 %diff) ]
85+
br i1 %c.1, label %then.2, label %else
86+
87+
then.2:
88+
%doubled = mul i64 %diff, 2
89+
ret i64 %doubled
90+
91+
else:
92+
ret i64 0
93+
}
94+
95+
define i64 @test_deref_user_does_not_dominate_other_user(ptr %p, ptr %q, i1 %c.0) {
96+
; CHECK-LABEL: define i64 @test_deref_user_does_not_dominate_other_user(
97+
; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C_0:%.*]]) {
98+
; CHECK-NEXT: [[ENTRY:.*:]]
99+
; CHECK-NEXT: [[P_INT:%.*]] = ptrtoint ptr [[P]] to i64
100+
; CHECK-NEXT: [[Q_INT:%.*]] = ptrtoint ptr [[Q]] to i64
101+
; CHECK-NEXT: [[DIFF:%.*]] = sub i64 [[Q_INT]], [[P_INT]]
102+
; CHECK-NEXT: br i1 [[C_0]], label %[[MIDDLE:.*]], label %[[EXIT:.*]]
103+
; CHECK: [[MIDDLE]]:
104+
; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[P]], i64 [[DIFF]]) ]
105+
; CHECK-NEXT: br label %[[EXIT]]
106+
; CHECK: [[EXIT]]:
107+
; CHECK-NEXT: ret i64 [[DIFF]]
108+
;
109+
entry:
110+
%p_int = ptrtoint ptr %p to i64
111+
%q_int = ptrtoint ptr %q to i64
112+
%diff = sub i64 %q_int, %p_int
113+
br i1 %c.0, label %middle, label %exit
114+
115+
middle:
116+
call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %p, i64 %diff) ]
117+
br label %exit
118+
119+
exit:
120+
ret i64 %diff
121+
}
122+
31123
declare void @llvm.assume(i1 noundef)

0 commit comments

Comments
 (0)