Skip to content

Commit 988fab7

Browse files
rlavaeerlavaee
committed
Fix everything
1 parent d0ab0b5 commit 988fab7

File tree

9 files changed

+127
-88
lines changed

9 files changed

+127
-88
lines changed

‎llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ class BasicBlockSectionsProfileReader {
102102
SmallVector<PrefetchHint>
103103
getPrefetchHintsForFunction(StringRef FuncName) const;
104104

105-
DenseMap<UniqueBBID, SmallVector<unsigned>> getPrefetchTargetsForFunction(StringRef FuncName) const;
105+
SmallVector<BBPosition> getPrefetchTargetsForFunction(StringRef FuncName) const;
106106

107107
private:
108108
StringRef getAliasName(StringRef FuncName) const {
@@ -215,7 +215,7 @@ class BasicBlockSectionsProfileReaderWrapperPass : public ImmutablePass {
215215
SmallVector<PrefetchHint>
216216
getPrefetchHintsForFunction(StringRef FuncName) const;
217217

218-
DenseMap<UniqueBBID, SmallVector<unsigned>> getPrefetchTargetsForFunction(StringRef FuncName) const;
218+
SmallVector<BBPosition> getPrefetchTargetsForFunction(StringRef FuncName) const;
219219

220220
// Initializes the FunctionNameToDIFilename map for the current module and
221221
// then reads the profile for the matching functions.

‎llvm/include/llvm/CodeGen/MachineBasicBlock.h‎

Lines changed: 5 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -100,12 +100,6 @@ template <> struct DenseMapInfo<MBBSectionID> {
100100
}
101101
};
102102

103-
struct PrefetchTarget {
104-
StringRef TargetFunction;
105-
UniqueBBID TargetBBID;
106-
unsigned TargetBBOffset;
107-
};
108-
109103
template <> struct ilist_traits<MachineInstr> {
110104
private:
111105
friend class MachineBasicBlock; // Set by the owning MachineBasicBlock.
@@ -219,8 +213,6 @@ class MachineBasicBlock
219213
/// basic block sections and basic block labels.
220214
std::optional<UniqueBBID> BBID;
221215

222-
SmallVector<unsigned> PrefetchTargets;
223-
224216
/// With basic block sections, this stores the Section ID of the basic block.
225217
MBBSectionID SectionID{0};
226218

@@ -237,7 +229,7 @@ class MachineBasicBlock
237229
/// is only computed once and is cached.
238230
mutable MCSymbol *CachedMCSymbol = nullptr;
239231

240-
mutable SmallVector<MCSymbol *, 4> CallInstSymbols;
232+
SmallVector<unsigned> PrefetchTargetIndexes;
241233

242234
/// Cached MCSymbol for this block (used if IsEHContTarget).
243235
mutable MCSymbol *CachedEHContMCSymbol = nullptr;
@@ -720,12 +712,12 @@ class MachineBasicBlock
720712

721713
std::optional<UniqueBBID> getBBID() const { return BBID; }
722714

723-
const SmallVector<unsigned> &getPrefetchTargets() const {
724-
return PrefetchTargets;
715+
const SmallVector<unsigned> &getPrefetchTargetIndexes() const {
716+
return PrefetchTargetIndexes;
725717
}
726718

727-
void setPrefetchTargets(const SmallVector<unsigned> &V) {
728-
PrefetchTargets = V;
719+
void setPrefetchTargetIndexes(const SmallVector<unsigned> &V) {
720+
PrefetchTargetIndexes = V;
729721
}
730722

731723
/// Returns the section ID of this basic block.
@@ -1293,12 +1285,6 @@ class MachineBasicBlock
12931285
/// Return the MCSymbol for this basic block.
12941286
LLVM_ABI MCSymbol *getSymbol() const;
12951287

1296-
MCSymbol *getCallInstSymbol(unsigned CallInstNumber) const;
1297-
1298-
const SmallVector<MCSymbol *, 4>& getCallInstSymbols() const {
1299-
return CallInstSymbols;
1300-
}
1301-
13021288
/// Return the Windows EH Continuation Symbol for this basic block.
13031289
LLVM_ABI MCSymbol *getEHContSymbol() const;
13041290

‎llvm/include/llvm/CodeGen/Passes.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ LLVM_ABI MachineFunctionPass *createBasicBlockSectionsPass();
6969

7070
LLVM_ABI MachineFunctionPass *createBasicBlockPathCloningPass();
7171

72+
LLVM_ABI MachineFunctionPass *createInsertCodePrefetchPass();
73+
7274
/// createMachineBlockHashInfoPass - This pass computes basic block hashes.
7375
LLVM_ABI MachineFunctionPass *createMachineBlockHashInfoPass();
7476

‎llvm/include/llvm/InitializePasses.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ LLVM_ABI void initializeAssignmentTrackingAnalysisPass(PassRegistry &);
5656
LLVM_ABI void initializeAssumptionCacheTrackerPass(PassRegistry &);
5757
LLVM_ABI void initializeAtomicExpandLegacyPass(PassRegistry &);
5858
LLVM_ABI void initializeBasicBlockPathCloningPass(PassRegistry &);
59+
LLVM_ABI void initializeInsertCodePrefetchPass(PassRegistry &);
5960
LLVM_ABI void
6061
initializeBasicBlockSectionsProfileReaderWrapperPassPass(PassRegistry &);
6162
LLVM_ABI void initializeBasicBlockSectionsPass(PassRegistry &);

‎llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp‎

Lines changed: 19 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -179,11 +179,6 @@ static cl::opt<bool> EmitJumpTableSizesSection(
179179
cl::desc("Emit a section containing jump table addresses and sizes"),
180180
cl::Hidden, cl::init(false));
181181

182-
static cl::opt<bool> InsertNoopsForPrefetch(
183-
"insert-noops-for-prefetch",
184-
cl::desc("Whether to insert noops instead of prefetches."), cl::init(false),
185-
cl::Hidden);
186-
187182
// This isn't turned on by default, since several of the scheduling models are
188183
// not completely accurate, and we don't want to be misleading.
189184
static cl::opt<bool> PrintLatency(
@@ -485,7 +480,6 @@ void AsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
485480
AU.addRequired<MachineBranchProbabilityInfoWrapperPass>();
486481
if (EmitBBHash)
487482
AU.addRequired<MachineBlockHashInfo>();
488-
AU.addUsedIfAvailable<BasicBlockSectionsProfileReaderWrapperPass>();
489483
}
490484

491485
bool AsmPrinter::doInitialization(Module &M) {
@@ -1988,44 +1982,29 @@ void AsmPrinter::emitFunctionBody() {
19881982

19891983
FunctionCallGraphInfo FuncCGInfo;
19901984
const auto &CallSitesInfoMap = MF->getCallSitesInfo();
1991-
DenseMap<UniqueBBID, SmallVector<unsigned>> FunctionPrefetchTargets;
1992-
if (auto *BBSPRPass =
1993-
getAnalysisIfAvailable<BasicBlockSectionsProfileReaderWrapperPass>()) {
1994-
FunctionPrefetchTargets = BBSPRPass->getBBSPR().getPrefetchTargetsForFunction(MF->getName());
1995-
}
19961985

19971986
for (auto &MBB : *MF) {
1998-
1999-
SmallVector<unsigned> BBPrefetchTargets;
2000-
= FunctionPrefetchTargets.lookup(MBB.g);
2001-
int NextPrefetchTargetIndex = MBB.getPrefetchTargets().empty() ? -1 : 0;
20021987
// Print a label for the basic block.
20031988
emitBasicBlockStart(MBB);
20041989
DenseMap<StringRef, unsigned> MnemonicCounts;
2005-
unsigned NumCallsInBlock = 0;
2006-
for (auto &MI : MBB) {
2007-
if (NextPrefetchTargetIndex != -1 &&
2008-
NumCallsInBlock >= MBB.getPrefetchTargets()[NextPrefetchTargetIndex]) {
20091990

2010-
MCSymbol *PrefetchTargetSymbol = OutContext.getOrCreateSymbol(
1991+
SmallVector<unsigned> PrefetchTargets = MBB.getPrefetchTargetIndexes();
1992+
auto PrefetchTargetIt = PrefetchTargets.begin();
1993+
unsigned NumCalls = 0;
1994+
auto EmitPrefetchTargetSymbolIfNeeded = [&]() {
1995+
if (PrefetchTargetIt == PrefetchTargets.end() || NumCalls < *PrefetchTargetIt)
1996+
return;
1997+
MCSymbol *PrefetchTargetSymbol = OutContext.getOrCreateSymbol(
20111998
Twine("__llvm_prefetch_target_") + MF->getName() + Twine("_") + utostr(MBB.getBBID()->BaseID) +
20121999
Twine("_") +
2013-
utostr(MBB.getPrefetchTargets()[NextPrefetchTargetIndex]));
2014-
if (MF->getFunction().isWeakForLinker()) {
2015-
OutStreamer->emitSymbolAttribute(PrefetchTargetSymbol, MCSA_Weak);
2016-
errs() << "Emitting weak symbol: " << PrefetchTargetSymbol->getName() << "\n";
2017-
} else {
2018-
OutStreamer->emitSymbolAttribute(PrefetchTargetSymbol, MCSA_Global);
2019-
errs() << "Emitting global symbol: " << PrefetchTargetSymbol->getName() << "\n";
2020-
}
2021-
// OutStreamer->emitSymbolAttribute(PrefetchTargetSymbol, MCSA_Extern);
2022-
// errs() << "Emitting symbol: " << PrefetchTargetSymbol->getName() << "\n";
2000+
utostr(*PrefetchTargetIt));
2001+
OutStreamer->emitSymbolAttribute(PrefetchTargetSymbol, MF->getFunction().isWeakForLinker() ? MCSA_Weak : MCSA_Global);
20232002
OutStreamer->emitLabel(PrefetchTargetSymbol);
2024-
++NextPrefetchTargetIndex;
2025-
if (NextPrefetchTargetIndex >=
2026-
static_cast<int>(MBB.getPrefetchTargets().size()))
2027-
NextPrefetchTargetIndex = -1;
2028-
}
2003+
++PrefetchTargetIt;
2004+
};
2005+
2006+
for (auto &MI : MBB) {
2007+
EmitPrefetchTargetSymbolIfNeeded();
20292008
// Print the assembly for the instruction.
20302009
if (!MI.isPosition() && !MI.isImplicitDef() && !MI.isKill() &&
20312010
!MI.isDebugInstr()) {
@@ -2163,8 +2142,11 @@ void AsmPrinter::emitFunctionBody() {
21632142
break;
21642143
}
21652144

2166-
if (MI.isCall() && MF->getTarget().Options.BBAddrMap)
2145+
if (MI.isCall()) {
2146+
if (MF->getTarget().Options.BBAddrMap)
21672147
OutStreamer->emitLabel(createCallsiteEndSymbol(MBB));
2148+
++NumCalls;
2149+
}
21682150

21692151
if (TM.Options.EmitCallGraphSection && MI.isCall())
21702152
handleCallsiteForCallgraph(FuncCGInfo, CallSitesInfoMap, MI);
@@ -2176,24 +2158,7 @@ void AsmPrinter::emitFunctionBody() {
21762158
for (auto &Handler : Handlers)
21772159
Handler->endInstruction();
21782160
}
2179-
while (NextPrefetchTargetIndex != -1) {
2180-
MCSymbol *PrefetchTargetSymbol = OutContext.getOrCreateSymbol(
2181-
Twine("__llvm_prefetch_target_") + MF->getName() + Twine("_") + utostr(MBB.getBBID()->BaseID) +
2182-
Twine("_") +
2183-
utostr(MBB.getPrefetchTargets()[NextPrefetchTargetIndex]));
2184-
if (MF->getFunction().hasWeakLinkage()) {
2185-
OutStreamer->emitSymbolAttribute(PrefetchTargetSymbol, MCSA_WeakDefinition);
2186-
} else {
2187-
OutStreamer->emitSymbolAttribute(PrefetchTargetSymbol, MCSA_Global);
2188-
}
2189-
OutStreamer->emitSymbolAttribute(PrefetchTargetSymbol, MCSA_Extern);
2190-
OutStreamer->emitLabel(PrefetchTargetSymbol);
2191-
++NextPrefetchTargetIndex;
2192-
if (NextPrefetchTargetIndex >=
2193-
static_cast<int>(MBB.getPrefetchTargets().size()))
2194-
NextPrefetchTargetIndex = -1;
2195-
}
2196-
2161+
EmitPrefetchTargetSymbolIfNeeded();
21972162

21982163
// We must emit temporary symbol for the end of this basic block, if either
21992164
// we have BBLabels enabled or if this basic blocks marks the end of a

‎llvm/lib/CodeGen/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ add_llvm_component_library(LLVMCodeGen
7979
IndirectBrExpandPass.cpp
8080
InitUndef.cpp
8181
InlineSpiller.cpp
82+
InsertCodePrefetch.cpp
8283
InterferenceCache.cpp
8384
InterleavedAccessPass.cpp
8485
InterleavedLoadCombinePass.cpp

‎llvm/lib/CodeGen/InsertCodePrefetch.cpp‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
//===-- InsertCodePrefetch.cpp ---=========-----------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
//
9+
/// \file
10+
/// Prefetch insertion pass implementation.
11+
//===----------------------------------------------------------------------===//
12+
/// Prefetch insertion pass.
13+
//===----------------------------------------------------------------------===//
14+
15+
#include "llvm/ADT/SmallVector.h"
16+
#include "llvm/ADT/StringExtras.h"
17+
#include "llvm/ADT/StringRef.h"
18+
#include "llvm/CodeGen/BasicBlockSectionUtils.h"
19+
#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
20+
#include "llvm/CodeGen/MachineBasicBlock.h"
21+
#include "llvm/CodeGen/MachineFunction.h"
22+
#include "llvm/CodeGen/MachineFunctionPass.h"
23+
#include "llvm/CodeGen/MachineOperand.h"
24+
#include "llvm/CodeGen/Passes.h"
25+
#include "llvm/CodeGen/TargetInstrInfo.h"
26+
#include "llvm/InitializePasses.h"
27+
#include "llvm/MC/MCContext.h"
28+
#include "llvm/Target/TargetMachine.h"
29+
30+
using namespace llvm;
31+
#define DEBUG_TYPE "prefetchinsertion"
32+
33+
namespace {
34+
class InsertCodePrefetch : public MachineFunctionPass {
35+
public:
36+
static char ID;
37+
38+
InsertCodePrefetch() : MachineFunctionPass(ID) {
39+
initializeInsertCodePrefetchPass(*PassRegistry::getPassRegistry());
40+
}
41+
42+
StringRef getPassName() const override { return "X86 Cide Prefetch Inserter Pass"; }
43+
44+
void getAnalysisUsage(AnalysisUsage &AU) const override;
45+
46+
/// Identify basic blocks that need separate sections and prepare to emit them
47+
/// accordingly.
48+
bool runOnMachineFunction(MachineFunction &MF) override;
49+
};
50+
51+
} // end anonymous namespace
52+
53+
//===----------------------------------------------------------------------===//
54+
// Implementation
55+
//===----------------------------------------------------------------------===//
56+
57+
char InsertCodePrefetch::ID = 0;
58+
INITIALIZE_PASS_BEGIN(
59+
InsertCodePrefetch, DEBUG_TYPE,
60+
"Reads prefetch", true,
61+
false)
62+
INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReaderWrapperPass)
63+
INITIALIZE_PASS_END(
64+
InsertCodePrefetch, DEBUG_TYPE,
65+
"Reads prefetch", true,
66+
false)
67+
68+
bool InsertCodePrefetch::runOnMachineFunction(MachineFunction &MF) {
69+
assert(MF.getTarget().getBBSectionsType() == BasicBlockSection::List &&
70+
"BB Sections list not enabled!");
71+
if (hasInstrProfHashMismatch(MF))
72+
return false;
73+
SmallVector<BBPosition> PrefetchTargets =
74+
getAnalysis<BasicBlockSectionsProfileReaderWrapperPass>()
75+
.getPrefetchTargetsForFunction(MF.getName());
76+
DenseMap<UniqueBBID, SmallVector<unsigned>> PrefetchTargetsByBBID;
77+
for (const auto &Target: PrefetchTargets)
78+
PrefetchTargetsByBBID[Target.BBID].push_back(Target.CallsiteIndex);
79+
for (auto &MBB: MF) {
80+
auto R = PrefetchTargetsByBBID.find(*MBB.getBBID());
81+
if (R == PrefetchTargetsByBBID.end()) continue;
82+
MBB.setPrefetchTargetIndexes(R->second);
83+
}
84+
85+
return false;
86+
}
87+
88+
void InsertCodePrefetch::getAnalysisUsage(AnalysisUsage &AU) const {
89+
AU.setPreservesAll();
90+
AU.addRequired<BasicBlockSectionsProfileReaderWrapperPass>();
91+
MachineFunctionPass::getAnalysisUsage(AU);
92+
}
93+
94+
MachineFunctionPass *llvm::createInsertCodePrefetchPass() {
95+
return new InsertCodePrefetch();
96+
}

‎llvm/lib/CodeGen/MachineBasicBlock.cpp‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -90,19 +90,6 @@ MCSymbol *MachineBasicBlock::getSymbol() const {
9090
return CachedMCSymbol;
9191
}
9292

93-
MCSymbol *MachineBasicBlock::getCallInstSymbol(unsigned CallInstNumber) const {
94-
if (CallInstSymbols.size() <= CallInstNumber) {
95-
const MachineFunction *MF = getParent();
96-
MCContext &Ctx = MF->getContext();
97-
CallInstSymbols.resize(CallInstNumber + 1);
98-
CallInstSymbols[CallInstNumber] = Ctx.createBlockSymbol(
99-
"BB" + Twine(MF->getFunctionNumber()) + "_" + Twine(getNumber()) + "_" +
100-
Twine(CallInstNumber),
101-
/*AlwaysEmit=*/true);
102-
}
103-
return CallInstSymbols[CallInstNumber];
104-
}
105-
10693
MCSymbol *MachineBasicBlock::getEHContSymbol() const {
10794
if (!CachedEHContMCSymbol) {
10895
const MachineFunction *MF = getParent();

‎llvm/lib/CodeGen/TargetPassConfig.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1293,6 +1293,7 @@ void TargetPassConfig::addMachinePasses() {
12931293
addPass(llvm::createBasicBlockSectionsProfileReaderWrapperPass(
12941294
TM->getBBSectionsFuncListBuf()));
12951295
addPass(llvm::createBasicBlockPathCloningPass());
1296+
addPass(llvm::createInsertCodePrefetchPass());
12961297
}
12971298
addPass(llvm::createBasicBlockSectionsPass());
12981299
}

0 commit comments

Comments
 (0)