Skip to content

Commit 48d344d

Browse files
boomanaiden154boomanaiden154
authored
Merge branch 'main' into remove-deprecated-string-triples
2 parents 2591107 + 03cb514 commit 48d344d

File tree

255 files changed

+2156
-12102
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

255 files changed

+2156
-12102
lines changed

‎llvm/include/llvm/IR/Instructions.h‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
#include "llvm/IR/Instruction.h"
3333
#include "llvm/IR/Intrinsics.h"
3434
#include "llvm/IR/OperandTraits.h"
35+
#include "llvm/IR/ProfDataUtils.h"
3536
#include "llvm/IR/Use.h"
3637
#include "llvm/IR/User.h"
3738
#include "llvm/Support/AtomicOrdering.h"
@@ -3536,8 +3537,6 @@ class SwitchInstProfUpdateWrapper {
35363537
bool Changed = false;
35373538

35383539
protected:
3539-
LLVM_ABI MDNode *buildProfBranchWeightsMD();
3540-
35413540
LLVM_ABI void init();
35423541

35433542
public:
@@ -3549,8 +3548,8 @@ class SwitchInstProfUpdateWrapper {
35493548
SwitchInstProfUpdateWrapper(SwitchInst &SI) : SI(SI) { init(); }
35503549

35513550
~SwitchInstProfUpdateWrapper() {
3552-
if (Changed)
3553-
SI.setMetadata(LLVMContext::MD_prof, buildProfBranchWeightsMD());
3551+
if (Changed && Weights.has_value() && Weights->size() >= 2)
3552+
setBranchWeights(SI, Weights.value(), /*IsExpected=*/false);
35543553
}
35553554

35563555
/// Delegate the call to the underlying SwitchInst::removeCase() and remove

‎llvm/include/llvm/IR/ProfDataUtils.h‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,13 @@ LLVM_ABI bool extractProfTotalWeight(const Instruction &I,
145145
/// \param Weights an array of weights to set on instruction I.
146146
/// \param IsExpected were these weights added from an llvm.expect* intrinsic.
147147
LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
148-
bool IsExpected);
148+
bool IsExpected, bool ElideAllZero = false);
149+
150+
/// Variant of `setBranchWeights` where the `Weights` will be fit first to
151+
/// uint32_t by shifting right.
152+
LLVM_ABI void setFittedBranchWeights(Instruction &I, ArrayRef<uint64_t> Weights,
153+
bool IsExpected,
154+
bool ElideAllZero = false);
149155

150156
/// downscale the given weights preserving the ratio. If the maximum value is
151157
/// not already known and not provided via \param KnownMaxCount , it will be

‎llvm/lib/Analysis/ValueTracking.cpp‎

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -7651,25 +7651,26 @@ static bool isGuaranteedNotToBeUndefOrPoison(
76517651
return true;
76527652
}
76537653

7654-
if (const auto *PN = dyn_cast<PHINode>(V)) {
7655-
unsigned Num = PN->getNumIncomingValues();
7656-
bool IsWellDefined = true;
7657-
for (unsigned i = 0; i < Num; ++i) {
7658-
if (PN == PN->getIncomingValue(i))
7659-
continue;
7660-
auto *TI = PN->getIncomingBlock(i)->getTerminator();
7661-
if (!isGuaranteedNotToBeUndefOrPoison(PN->getIncomingValue(i), AC, TI,
7662-
DT, Depth + 1, Kind)) {
7663-
IsWellDefined = false;
7664-
break;
7654+
if (!::canCreateUndefOrPoison(Opr, Kind,
7655+
/*ConsiderFlagsAndMetadata=*/true)) {
7656+
if (const auto *PN = dyn_cast<PHINode>(V)) {
7657+
unsigned Num = PN->getNumIncomingValues();
7658+
bool IsWellDefined = true;
7659+
for (unsigned i = 0; i < Num; ++i) {
7660+
if (PN == PN->getIncomingValue(i))
7661+
continue;
7662+
auto *TI = PN->getIncomingBlock(i)->getTerminator();
7663+
if (!isGuaranteedNotToBeUndefOrPoison(PN->getIncomingValue(i), AC, TI,
7664+
DT, Depth + 1, Kind)) {
7665+
IsWellDefined = false;
7666+
break;
7667+
}
76657668
}
7666-
}
7667-
if (IsWellDefined)
7669+
if (IsWellDefined)
7670+
return true;
7671+
} else if (all_of(Opr->operands(), OpCheck))
76687672
return true;
7669-
} else if (!::canCreateUndefOrPoison(Opr, Kind,
7670-
/*ConsiderFlagsAndMetadata*/ true) &&
7671-
all_of(Opr->operands(), OpCheck))
7672-
return true;
7673+
}
76737674
}
76747675

76757676
if (auto *I = dyn_cast<LoadInst>(V))

‎llvm/lib/IR/Instructions.cpp‎

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -4141,23 +4141,6 @@ void SwitchInst::growOperands() {
41414141
growHungoffUses(ReservedSpace);
41424142
}
41434143

4144-
MDNode *SwitchInstProfUpdateWrapper::buildProfBranchWeightsMD() {
4145-
assert(Changed && "called only if metadata has changed");
4146-
4147-
if (!Weights)
4148-
return nullptr;
4149-
4150-
assert(SI.getNumSuccessors() == Weights->size() &&
4151-
"num of prof branch_weights must accord with num of successors");
4152-
4153-
bool AllZeroes = all_of(*Weights, [](uint32_t W) { return W == 0; });
4154-
4155-
if (AllZeroes || Weights->size() < 2)
4156-
return nullptr;
4157-
4158-
return MDBuilder(SI.getParent()->getContext()).createBranchWeights(*Weights);
4159-
}
4160-
41614144
void SwitchInstProfUpdateWrapper::init() {
41624145
MDNode *ProfileData = getBranchWeightMDNode(SI);
41634146
if (!ProfileData)

‎llvm/lib/IR/ProfDataUtils.cpp‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,15 @@
1212

1313
#include "llvm/IR/ProfDataUtils.h"
1414

15+
#include "llvm/ADT/STLExtras.h"
1516
#include "llvm/ADT/SmallVector.h"
1617
#include "llvm/IR/Constants.h"
1718
#include "llvm/IR/Function.h"
1819
#include "llvm/IR/Instructions.h"
1920
#include "llvm/IR/LLVMContext.h"
2021
#include "llvm/IR/MDBuilder.h"
2122
#include "llvm/IR/Metadata.h"
23+
#include "llvm/Support/CommandLine.h"
2224

2325
using namespace llvm;
2426

@@ -84,10 +86,31 @@ static void extractFromBranchWeightMD(const MDNode *ProfileData,
8486
}
8587
}
8688

89+
/// Push the weights right to fit in uint32_t.
90+
static SmallVector<uint32_t> fitWeights(ArrayRef<uint64_t> Weights) {
91+
SmallVector<uint32_t> Ret;
92+
Ret.reserve(Weights.size());
93+
uint64_t Max = *llvm::max_element(Weights);
94+
if (Max > UINT_MAX) {
95+
unsigned Offset = 32 - llvm::countl_zero(Max);
96+
for (const uint64_t &Value : Weights)
97+
Ret.push_back(static_cast<uint32_t>(Value >> Offset));
98+
} else {
99+
append_range(Ret, Weights);
100+
}
101+
return Ret;
102+
}
103+
87104
} // namespace
88105

89106
namespace llvm {
90-
107+
cl::opt<bool> ElideAllZeroBranchWeights("elide-all-zero-branch-weights",
108+
#if defined(LLVM_ENABLE_PROFCHECK)
109+
cl::init(false)
110+
#else
111+
cl::init(true)
112+
#endif
113+
);
91114
const char *MDProfLabels::BranchWeights = "branch_weights";
92115
const char *MDProfLabels::ExpectedBranchWeights = "expected";
93116
const char *MDProfLabels::ValueProfile = "VP";
@@ -282,12 +305,23 @@ bool hasExplicitlyUnknownBranchWeights(const Instruction &I) {
282305
}
283306

284307
void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
285-
bool IsExpected) {
308+
bool IsExpected, bool ElideAllZero) {
309+
if ((ElideAllZeroBranchWeights && ElideAllZero) &&
310+
llvm::all_of(Weights, [](uint32_t V) { return V == 0; })) {
311+
I.setMetadata(LLVMContext::MD_prof, nullptr);
312+
return;
313+
}
314+
286315
MDBuilder MDB(I.getContext());
287316
MDNode *BranchWeights = MDB.createBranchWeights(Weights, IsExpected);
288317
I.setMetadata(LLVMContext::MD_prof, BranchWeights);
289318
}
290319

320+
void setFittedBranchWeights(Instruction &I, ArrayRef<uint64_t> Weights,
321+
bool IsExpected, bool ElideAllZero) {
322+
setBranchWeights(I, fitWeights(Weights), IsExpected, ElideAllZero);
323+
}
324+
291325
SmallVector<uint32_t> downscaleWeights(ArrayRef<uint64_t> Weights,
292326
std::optional<uint64_t> KnownMaxCount) {
293327
uint64_t MaxCount = KnownMaxCount.has_value() ? KnownMaxCount.value()

‎llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ class RISCVExpandPseudo : public MachineFunctionPass {
4646
MachineBasicBlock::iterator &NextMBBI);
4747
bool expandCCOp(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
4848
MachineBasicBlock::iterator &NextMBBI);
49+
bool expandCCOpToCMov(MachineBasicBlock &MBB,
50+
MachineBasicBlock::iterator MBBI);
4951
bool expandVMSET_VMCLR(MachineBasicBlock &MBB,
5052
MachineBasicBlock::iterator MBBI, unsigned Opcode);
5153
bool expandMV_FPR16INX(MachineBasicBlock &MBB,
@@ -178,6 +180,9 @@ bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
178180
bool RISCVExpandPseudo::expandCCOp(MachineBasicBlock &MBB,
179181
MachineBasicBlock::iterator MBBI,
180182
MachineBasicBlock::iterator &NextMBBI) {
183+
// First try expanding to a Conditional Move rather than a branch+mv
184+
if (expandCCOpToCMov(MBB, MBBI))
185+
return true;
181186

182187
MachineFunction *MF = MBB.getParent();
183188
MachineInstr &MI = *MBBI;
@@ -277,6 +282,86 @@ bool RISCVExpandPseudo::expandCCOp(MachineBasicBlock &MBB,
277282
return true;
278283
}
279284

285+
bool RISCVExpandPseudo::expandCCOpToCMov(MachineBasicBlock &MBB,
286+
MachineBasicBlock::iterator MBBI) {
287+
MachineInstr &MI = *MBBI;
288+
DebugLoc DL = MI.getDebugLoc();
289+
290+
if (MI.getOpcode() != RISCV::PseudoCCMOVGPR &&
291+
MI.getOpcode() != RISCV::PseudoCCMOVGPRNoX0)
292+
return false;
293+
294+
if (!STI->hasVendorXqcicm())
295+
return false;
296+
297+
// FIXME: Would be wonderful to support LHS=X0, but not very easy.
298+
if (MI.getOperand(1).getReg() == RISCV::X0 ||
299+
MI.getOperand(4).getReg() == RISCV::X0 ||
300+
MI.getOperand(5).getReg() == RISCV::X0)
301+
return false;
302+
303+
auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
304+
305+
unsigned CMovOpcode, CMovIOpcode;
306+
switch (CC) {
307+
default:
308+
llvm_unreachable("Unhandled CC");
309+
case RISCVCC::COND_EQ:
310+
CMovOpcode = RISCV::QC_MVEQ;
311+
CMovIOpcode = RISCV::QC_MVEQI;
312+
break;
313+
case RISCVCC::COND_NE:
314+
CMovOpcode = RISCV::QC_MVNE;
315+
CMovIOpcode = RISCV::QC_MVNEI;
316+
break;
317+
case RISCVCC::COND_LT:
318+
CMovOpcode = RISCV::QC_MVLT;
319+
CMovIOpcode = RISCV::QC_MVLTI;
320+
break;
321+
case RISCVCC::COND_GE:
322+
CMovOpcode = RISCV::QC_MVGE;
323+
CMovIOpcode = RISCV::QC_MVGEI;
324+
break;
325+
case RISCVCC::COND_LTU:
326+
CMovOpcode = RISCV::QC_MVLTU;
327+
CMovIOpcode = RISCV::QC_MVLTUI;
328+
break;
329+
case RISCVCC::COND_GEU:
330+
CMovOpcode = RISCV::QC_MVGEU;
331+
CMovIOpcode = RISCV::QC_MVGEUI;
332+
break;
333+
}
334+
335+
if (MI.getOperand(2).getReg() == RISCV::X0) {
336+
// $dst = PseudoCCMOVGPR $lhs, X0, $cc, $falsev (=$dst), $truev
337+
// $dst = PseudoCCMOVGPRNoX0 $lhs, X0, $cc, $falsev (=$dst), $truev
338+
// =>
339+
// $dst = QC_MVccI $falsev (=$dst), $lhs, 0, $truev
340+
BuildMI(MBB, MBBI, DL, TII->get(CMovIOpcode))
341+
.addDef(MI.getOperand(0).getReg())
342+
.addReg(MI.getOperand(4).getReg())
343+
.addReg(MI.getOperand(1).getReg())
344+
.addImm(0)
345+
.addReg(MI.getOperand(5).getReg());
346+
347+
MI.eraseFromParent();
348+
return true;
349+
}
350+
351+
// $dst = PseudoCCMOVGPR $lhs, $rhs, $cc, $falsev (=$dst), $truev
352+
// $dst = PseudoCCMOVGPRNoX0 $lhs, $rhs, $cc, $falsev (=$dst), $truev
353+
// =>
354+
// $dst = QC_MVcc $falsev (=$dst), $lhs, $rhs, $truev
355+
BuildMI(MBB, MBBI, DL, TII->get(CMovOpcode))
356+
.addDef(MI.getOperand(0).getReg())
357+
.addReg(MI.getOperand(4).getReg())
358+
.addReg(MI.getOperand(1).getReg())
359+
.addReg(MI.getOperand(2).getReg())
360+
.addReg(MI.getOperand(5).getReg());
361+
MI.eraseFromParent();
362+
return true;
363+
}
364+
280365
bool RISCVExpandPseudo::expandVMSET_VMCLR(MachineBasicBlock &MBB,
281366
MachineBasicBlock::iterator MBBI,
282367
unsigned Opcode) {

‎llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1350,6 +1350,10 @@ class QCIMVCCIPat<CondCode Cond, QCIMVCCI Inst, DAGOperand InTyImm>
13501350
: Pat<(i32 (riscv_selectcc (i32 GPRNoX0:$rs1), InTyImm:$imm, Cond, (i32 GPRNoX0:$rs3), (i32 GPRNoX0:$rd))),
13511351
(Inst GPRNoX0:$rd, GPRNoX0:$rs1, InTyImm:$imm, GPRNoX0:$rs3)>;
13521352

1353+
class QCIMVCCIZeroPat<CondCode Cond, QCIMVCCI Inst>
1354+
: Pat<(i32 (riscv_selectcc (i32 GPRNoX0:$rs1), (i32 0), Cond, (i32 GPRNoX0:$rs3), (i32 GPRNoX0:$rd))),
1355+
(Inst GPRNoX0:$rd, GPRNoX0:$rs1, 0, GPRNoX0:$rs3)>;
1356+
13531357
class QCISELECTCCIPat<CondCode Cond, QCISELECTCCI Inst>
13541358
: Pat<(i32 (riscv_selectcc (i32 GPRNoX0:$rd), simm5:$imm, Cond, (i32 GPRNoX0:$rs2), (i32 GPRNoX0:$rs3))),
13551359
(Inst GPRNoX0:$rd, simm5:$imm, GPRNoX0:$rs2, GPRNoX0:$rs3)>;
@@ -1538,27 +1542,32 @@ def: Pat<(i32 (ctlz (not (i32 GPR:$rs1)))), (QC_CLO GPR:$rs1)>;
15381542
let Predicates = [HasVendorXqciint, IsRV32] in
15391543
def : Pat<(riscv_mileaveret_glue), (QC_C_MILEAVERET)>;
15401544

1541-
let Predicates = [HasVendorXqcicm, IsRV32] in {
1542-
// (SELECT X, Y, Z) is canonicalised to `(riscv_selectcc x, 0, NE, y, z)`.
1543-
// This exists to prioritise over the `Select_GPR_Using_CC_GPR` pattern.
1544-
def : Pat<(i32 (riscv_selectcc (i32 GPRNoX0:$rs1), (i32 0), SETNE, (i32 GPRNoX0:$rs3), (i32 GPRNoX0:$rd))),
1545-
(QC_MVNEI GPRNoX0:$rd, GPRNoX0:$rs1, 0, GPRNoX0:$rs3)>;
1546-
def : Pat<(i32 (riscv_selectcc (i32 GPRNoX0:$rs1), (i32 0), SETEQ, (i32 GPRNoX0:$rs3), (i32 GPRNoX0:$rd))),
1547-
(QC_MVEQI GPRNoX0:$rd, GPRNoX0:$rs1, 0, GPRNoX0:$rs3)>;
1548-
1545+
let Predicates = [HasVendorXqcicm, NoShortForwardBranchOpt, IsRV32] in {
15491546
def : QCIMVCCPat<SETEQ, QC_MVEQ>;
15501547
def : QCIMVCCPat<SETNE, QC_MVNE>;
15511548
def : QCIMVCCPat<SETLT, QC_MVLT>;
15521549
def : QCIMVCCPat<SETULT, QC_MVLTU>;
15531550
def : QCIMVCCPat<SETGE, QC_MVGE>;
15541551
def : QCIMVCCPat<SETUGE, QC_MVGEU>;
15551552

1556-
def : QCIMVCCIPat<SETEQ, QC_MVEQI, simm5>;
1557-
def : QCIMVCCIPat<SETNE, QC_MVNEI, simm5>;
1558-
def : QCIMVCCIPat<SETLT, QC_MVLTI, simm5>;
1559-
def : QCIMVCCIPat<SETULT, QC_MVLTUI, uimm5>;
1560-
def : QCIMVCCIPat<SETGE, QC_MVGEI, simm5>;
1561-
def : QCIMVCCIPat<SETUGE, QC_MVGEUI, uimm5>;
1553+
// These exist to prioritise over the `Select_GPR_Using_CC_GPR` pattern for X0.
1554+
def : QCIMVCCIZeroPat<SETEQ, QC_MVEQI>;
1555+
def : QCIMVCCIZeroPat<SETNE, QC_MVNEI>;
1556+
def : QCIMVCCIZeroPat<SETLT, QC_MVLTI>;
1557+
def : QCIMVCCIZeroPat<SETULT, QC_MVLTUI>;
1558+
def : QCIMVCCIZeroPat<SETGE, QC_MVGEI>;
1559+
def : QCIMVCCIZeroPat<SETUGE, QC_MVGEUI>;
1560+
}
1561+
1562+
let Predicates = [HasVendorXqcicm, IsRV32] in {
1563+
// These all use *imm5nonzero because we want to use PseudoCCMOVGPR with X0 when SFB is enabled.
1564+
// When SFB is not enabled, the `QCIMVCCIZeroPat`s above will be used if RHS=0.
1565+
def : QCIMVCCIPat<SETEQ, QC_MVEQI, simm5nonzero>;
1566+
def : QCIMVCCIPat<SETNE, QC_MVNEI, simm5nonzero>;
1567+
def : QCIMVCCIPat<SETLT, QC_MVLTI, simm5nonzero>;
1568+
def : QCIMVCCIPat<SETULT, QC_MVLTUI, uimm5nonzero>;
1569+
def : QCIMVCCIPat<SETGE, QC_MVGEI, simm5nonzero>;
1570+
def : QCIMVCCIPat<SETUGE, QC_MVGEUI, uimm5nonzero>;
15621571
}
15631572

15641573
let Predicates = [HasVendorXqcicli, IsRV32] in {

‎llvm/lib/Transforms/IPO/SampleProfile.cpp‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1664,8 +1664,9 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) {
16641664
else if (OverwriteExistingWeights)
16651665
I.setMetadata(LLVMContext::MD_prof, nullptr);
16661666
} else if (!isa<IntrinsicInst>(&I)) {
1667-
setBranchWeights(I, {static_cast<uint32_t>(BlockWeights[BB])},
1668-
/*IsExpected=*/false);
1667+
setBranchWeights(
1668+
I, ArrayRef<uint32_t>{static_cast<uint32_t>(BlockWeights[BB])},
1669+
/*IsExpected=*/false);
16691670
}
16701671
}
16711672
} else if (OverwriteExistingWeights || ProfileSampleBlockAccurate) {
@@ -1676,7 +1677,8 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) {
16761677
if (cast<CallBase>(I).isIndirectCall()) {
16771678
I.setMetadata(LLVMContext::MD_prof, nullptr);
16781679
} else {
1679-
setBranchWeights(I, {uint32_t(0)}, /*IsExpected=*/false);
1680+
setBranchWeights(I, ArrayRef<uint32_t>{uint32_t(0)},
1681+
/*IsExpected=*/false);
16801682
}
16811683
}
16821684
}

‎llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -672,8 +672,8 @@ CallBase &llvm::pgo::promoteIndirectCall(CallBase &CB, Function *DirectCallee,
672672
createBranchWeights(CB.getContext(), Count, TotalCount - Count));
673673

674674
if (AttachProfToDirectCall)
675-
setBranchWeights(NewInst, {static_cast<uint32_t>(Count)},
676-
/*IsExpected=*/false);
675+
setFittedBranchWeights(NewInst, {Count},
676+
/*IsExpected=*/false);
677677

678678
using namespace ore;
679679

0 commit comments

Comments
 (0)