Skip to content

Commit 1dc08d5

Browse files
optimisanoptimisan
committed
[AMDGPU][NewPM] Port TailDuplicate pass to NPM
1 parent 75ec65e commit 1dc08d5

14 files changed

+113
-26
lines changed

‎llvm/include/llvm/CodeGen/Passes.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -261,11 +261,11 @@ namespace llvm {
261261

262262
/// TailDuplicate - Duplicate blocks with unconditional branches
263263
/// into tails of their predecessors.
264-
extern char &TailDuplicateID;
264+
extern char &TailDuplicateLegacyID;
265265

266266
/// Duplicate blocks with unconditional branches into tails of their
267267
/// predecessors. Variant that works before register allocation.
268-
extern char &EarlyTailDuplicateID;
268+
extern char &EarlyTailDuplicateLegacyID;
269269

270270
/// MachineTraceMetrics - This pass computes critical path and CPU resource
271271
/// usage in an ensemble of traces.

‎llvm/include/llvm/CodeGen/TailDuplication.h‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
//===- llvm/CodeGen/TailDuplication.h ---------------------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef LLVM_CODEGEN_TAILDUPLICATIONPASS_H
10+
#define LLVM_CODEGEN_TAILDUPLICATIONPASS_H
11+
12+
#include "llvm/CodeGen/MBFIWrapper.h"
13+
#include "llvm/CodeGen/MachineFunction.h"
14+
#include "llvm/CodeGen/MachinePassManager.h"
15+
16+
namespace llvm {
17+
18+
template <typename DerivedT, bool PreRegAlloc>
19+
class TailDuplicatePassBase : public PassInfoMixin<DerivedT> {
20+
private:
21+
std::unique_ptr<MBFIWrapper> MBFIW;
22+
23+
public:
24+
PreservedAnalyses run(MachineFunction &MF,
25+
MachineFunctionAnalysisManager &MFAM);
26+
};
27+
28+
class EarlyTailDuplicatePass
29+
: public TailDuplicatePassBase<EarlyTailDuplicatePass, true> {
30+
public:
31+
MachineFunctionProperties getClearedProperties() const {
32+
return MachineFunctionProperties().set(
33+
MachineFunctionProperties::Property::NoPHIs);
34+
}
35+
};
36+
37+
class TailDuplicatePass
38+
: public TailDuplicatePassBase<TailDuplicatePass, false> {};
39+
40+
} // namespace llvm
41+
42+
extern template class llvm::TailDuplicatePassBase<llvm::EarlyTailDuplicatePass,
43+
true>;
44+
extern template class llvm::TailDuplicatePassBase<llvm::TailDuplicatePass,
45+
false>;
46+
47+
#endif // LLVM_CODEGEN_TAILDUPLICATIONPASS_H

‎llvm/include/llvm/InitializePasses.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ void initializeEarlyCSEMemSSALegacyPassPass(PassRegistry &);
101101
void initializeEarlyIfConverterLegacyPass(PassRegistry &);
102102
void initializeEarlyIfPredicatorPass(PassRegistry &);
103103
void initializeEarlyMachineLICMPass(PassRegistry &);
104-
void initializeEarlyTailDuplicatePass(PassRegistry &);
104+
void initializeEarlyTailDuplicateLegacyPass(PassRegistry &);
105105
void initializeEdgeBundlesPass(PassRegistry &);
106106
void initializeEHContGuardCatchretPass(PassRegistry &);
107107
void initializeExpandLargeFpConvertLegacyPassPass(PassRegistry &);
@@ -300,7 +300,7 @@ void initializeStraightLineStrengthReduceLegacyPassPass(PassRegistry &);
300300
void initializeStripDebugMachineModulePass(PassRegistry &);
301301
void initializeStructurizeCFGLegacyPassPass(PassRegistry &);
302302
void initializeTailCallElimPass(PassRegistry &);
303-
void initializeTailDuplicatePass(PassRegistry &);
303+
void initializeTailDuplicateLegacyPass(PassRegistry &);
304304
void initializeTargetLibraryInfoWrapperPassPass(PassRegistry &);
305305
void initializeTargetPassConfigPass(PassRegistry &);
306306
void initializeTargetTransformInfoWrapperPassPass(PassRegistry &);

‎llvm/include/llvm/Passes/CodeGenPassBuilder.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
#include "llvm/CodeGen/SjLjEHPrepare.h"
6060
#include "llvm/CodeGen/StackColoring.h"
6161
#include "llvm/CodeGen/StackProtector.h"
62+
#include "llvm/CodeGen/TailDuplication.h"
6263
#include "llvm/CodeGen/TargetPassConfig.h"
6364
#include "llvm/CodeGen/TwoAddressInstructionPass.h"
6465
#include "llvm/CodeGen/UnreachableBlockElim.h"

‎llvm/include/llvm/Passes/MachinePassRegistry.def‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,7 @@ MACHINE_FUNCTION_ANALYSIS("virtregmap", VirtRegMapAnalysis())
132132
MACHINE_FUNCTION_PASS("dead-mi-elimination", DeadMachineInstructionElimPass())
133133
MACHINE_FUNCTION_PASS("early-ifcvt", EarlyIfConverterPass())
134134
MACHINE_FUNCTION_PASS("early-machinelicm", EarlyMachineLICMPass())
135+
MACHINE_FUNCTION_PASS("early-tailduplication", EarlyTailDuplicatePass())
135136
MACHINE_FUNCTION_PASS("finalize-isel", FinalizeISelPass())
136137
MACHINE_FUNCTION_PASS("localstackalloc", LocalStackSlotAllocationPass())
137138
MACHINE_FUNCTION_PASS("machine-cse", MachineCSEPass())
@@ -155,6 +156,7 @@ MACHINE_FUNCTION_PASS("print<virtregmap>", VirtRegMapPrinterPass(dbgs()))
155156
MACHINE_FUNCTION_PASS("require-all-machine-function-properties",
156157
RequireAllMachineFunctionPropertiesPass())
157158
MACHINE_FUNCTION_PASS("stack-coloring", StackColoringPass())
159+
MACHINE_FUNCTION_PASS("tailduplication", TailDuplicatePass())
158160
MACHINE_FUNCTION_PASS("trigger-verifier-error", TriggerVerifierErrorPass())
159161
MACHINE_FUNCTION_PASS("two-address-instruction", TwoAddressInstructionPass())
160162
MACHINE_FUNCTION_PASS("verify", MachineVerifierPass())
@@ -208,7 +210,6 @@ DUMMY_MACHINE_FUNCTION_PASS("cfi-fixup", CFIFixupPass)
208210
DUMMY_MACHINE_FUNCTION_PASS("cfi-instr-inserter", CFIInstrInserterPass)
209211
DUMMY_MACHINE_FUNCTION_PASS("detect-dead-lanes", DetectDeadLanesPass)
210212
DUMMY_MACHINE_FUNCTION_PASS("dot-machine-cfg", MachineCFGPrinter)
211-
DUMMY_MACHINE_FUNCTION_PASS("early-tailduplication", EarlyTailDuplicatePass)
212213
DUMMY_MACHINE_FUNCTION_PASS("fentry-insert", FEntryInserterPass)
213214
DUMMY_MACHINE_FUNCTION_PASS("fixup-statepoint-caller-saved", FixupStatepointCallerSavedPass)
214215
DUMMY_MACHINE_FUNCTION_PASS("fs-profile-loader", MIRProfileLoaderNewPass)
@@ -261,7 +262,6 @@ DUMMY_MACHINE_FUNCTION_PASS("simple-register-coalescing", RegisterCoalescerPass)
261262
DUMMY_MACHINE_FUNCTION_PASS("stack-frame-layout", StackFrameLayoutAnalysisPass)
262263
DUMMY_MACHINE_FUNCTION_PASS("stack-slot-coloring", StackSlotColoringPass)
263264
DUMMY_MACHINE_FUNCTION_PASS("stackmap-liveness", StackMapLivenessPass)
264-
DUMMY_MACHINE_FUNCTION_PASS("tailduplication", TailDuplicatePass)
265265
DUMMY_MACHINE_FUNCTION_PASS("unpack-mi-bundles", UnpackMachineBundlesPass)
266266
DUMMY_MACHINE_FUNCTION_PASS("virtregrewriter", VirtRegRewriterPass)
267267
DUMMY_MACHINE_FUNCTION_PASS("xray-instrumentation", XRayInstrumentationPass)

‎llvm/lib/CodeGen/CodeGen.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
3838
initializeEarlyIfConverterLegacyPass(Registry);
3939
initializeEarlyIfPredicatorPass(Registry);
4040
initializeEarlyMachineLICMPass(Registry);
41-
initializeEarlyTailDuplicatePass(Registry);
41+
initializeEarlyTailDuplicateLegacyPass(Registry);
4242
initializeExpandLargeDivRemLegacyPassPass(Registry);
4343
initializeExpandLargeFpConvertLegacyPassPass(Registry);
4444
initializeExpandMemCmpLegacyPassPass(Registry);
@@ -131,7 +131,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
131131
initializeStackProtectorPass(Registry);
132132
initializeStackSlotColoringPass(Registry);
133133
initializeStripDebugMachineModulePass(Registry);
134-
initializeTailDuplicatePass(Registry);
134+
initializeTailDuplicateLegacyPass(Registry);
135135
initializeTargetPassConfigPass(Registry);
136136
initializeTwoAddressInstructionLegacyPassPass(Registry);
137137
initializeTypePromotionLegacyPass(Registry);

‎llvm/lib/CodeGen/TailDuplication.cpp‎

Lines changed: 46 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,16 @@
1212
//
1313
//===----------------------------------------------------------------------===//
1414

15+
#include "llvm/CodeGen/TailDuplication.h"
1516
#include "llvm/Analysis/ProfileSummaryInfo.h"
1617
#include "llvm/CodeGen/LazyMachineBlockFrequencyInfo.h"
1718
#include "llvm/CodeGen/MBFIWrapper.h"
1819
#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
1920
#include "llvm/CodeGen/MachineFunction.h"
2021
#include "llvm/CodeGen/MachineFunctionPass.h"
22+
#include "llvm/CodeGen/MachinePassManager.h"
2123
#include "llvm/CodeGen/TailDuplicator.h"
24+
#include "llvm/IR/Analysis.h"
2225
#include "llvm/InitializePasses.h"
2326
#include "llvm/Pass.h"
2427
#include "llvm/PassRegistry.h"
@@ -47,19 +50,19 @@ class TailDuplicateBase : public MachineFunctionPass {
4750
}
4851
};
4952

50-
class TailDuplicate : public TailDuplicateBase {
53+
class TailDuplicateLegacy : public TailDuplicateBase {
5154
public:
5255
static char ID;
53-
TailDuplicate() : TailDuplicateBase(ID, false) {
54-
initializeTailDuplicatePass(*PassRegistry::getPassRegistry());
56+
TailDuplicateLegacy() : TailDuplicateBase(ID, false) {
57+
initializeTailDuplicateLegacyPass(*PassRegistry::getPassRegistry());
5558
}
5659
};
5760

58-
class EarlyTailDuplicate : public TailDuplicateBase {
61+
class EarlyTailDuplicateLegacy : public TailDuplicateBase {
5962
public:
6063
static char ID;
61-
EarlyTailDuplicate() : TailDuplicateBase(ID, true) {
62-
initializeEarlyTailDuplicatePass(*PassRegistry::getPassRegistry());
64+
EarlyTailDuplicateLegacy() : TailDuplicateBase(ID, true) {
65+
initializeEarlyTailDuplicateLegacyPass(*PassRegistry::getPassRegistry());
6366
}
6467

6568
MachineFunctionProperties getClearedProperties() const override {
@@ -70,14 +73,15 @@ class EarlyTailDuplicate : public TailDuplicateBase {
7073

7174
} // end anonymous namespace
7275

73-
char TailDuplicate::ID;
74-
char EarlyTailDuplicate::ID;
76+
char TailDuplicateLegacy::ID;
77+
char EarlyTailDuplicateLegacy::ID;
7578

76-
char &llvm::TailDuplicateID = TailDuplicate::ID;
77-
char &llvm::EarlyTailDuplicateID = EarlyTailDuplicate::ID;
79+
char &llvm::TailDuplicateLegacyID = TailDuplicateLegacy::ID;
80+
char &llvm::EarlyTailDuplicateLegacyID = EarlyTailDuplicateLegacy::ID;
7881

79-
INITIALIZE_PASS(TailDuplicate, DEBUG_TYPE, "Tail Duplication", false, false)
80-
INITIALIZE_PASS(EarlyTailDuplicate, "early-tailduplication",
82+
INITIALIZE_PASS(TailDuplicateLegacy, DEBUG_TYPE, "Tail Duplication", false,
83+
false)
84+
INITIALIZE_PASS(EarlyTailDuplicateLegacy, "early-tailduplication",
8185
"Early Tail Duplication", false, false)
8286

8387
bool TailDuplicateBase::runOnMachineFunction(MachineFunction &MF) {
@@ -100,3 +104,33 @@ bool TailDuplicateBase::runOnMachineFunction(MachineFunction &MF) {
100104

101105
return MadeChange;
102106
}
107+
108+
template <typename DerivedT, bool PreRegAlloc>
109+
PreservedAnalyses TailDuplicatePassBase<DerivedT, PreRegAlloc>::run(
110+
MachineFunction &MF, MachineFunctionAnalysisManager &MFAM) {
111+
MFPropsModifier<DerivedT> _(static_cast<DerivedT &>(*this), MF);
112+
113+
auto *MBPI = &MFAM.getResult<MachineBranchProbabilityAnalysis>(MF);
114+
auto *PSI = MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(MF)
115+
.getCachedResult<ProfileSummaryAnalysis>(
116+
*MF.getFunction().getParent());
117+
auto *MBFI = (PSI && PSI->hasProfileSummary()
118+
? &MFAM.getResult<MachineBlockFrequencyAnalysis>(MF)
119+
: nullptr);
120+
if (MBFI)
121+
MBFIW = std::make_unique<MBFIWrapper>(*MBFI);
122+
123+
TailDuplicator Duplicator;
124+
Duplicator.initMF(MF, PreRegAlloc, MBPI, MBFI ? MBFIW.get() : nullptr, PSI,
125+
/*LayoutMode=*/false);
126+
bool MadeChange = false;
127+
while (Duplicator.tailDuplicateBlocks())
128+
MadeChange = true;
129+
130+
if (!MadeChange)
131+
return PreservedAnalyses::all();
132+
return getMachineFunctionPassPreservedAnalyses();
133+
}
134+
135+
template class llvm::TailDuplicatePassBase<TailDuplicatePass, false>;
136+
template class llvm::TailDuplicatePassBase<EarlyTailDuplicatePass, true>;

‎llvm/lib/CodeGen/TargetPassConfig.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -290,10 +290,10 @@ static IdentifyingPassPtr overridePass(AnalysisID StandardID,
290290
if (StandardID == &BranchFolderPassID)
291291
return applyDisable(TargetID, DisableBranchFold);
292292

293-
if (StandardID == &TailDuplicateID)
293+
if (StandardID == &TailDuplicateLegacyID)
294294
return applyDisable(TargetID, DisableTailDuplicate);
295295

296-
if (StandardID == &EarlyTailDuplicateID)
296+
if (StandardID == &EarlyTailDuplicateLegacyID)
297297
return applyDisable(TargetID, DisableEarlyTailDup);
298298

299299
if (StandardID == &MachineBlockPlacementID)
@@ -1279,7 +1279,7 @@ void TargetPassConfig::addMachinePasses() {
12791279
/// Add passes that optimize machine instructions in SSA form.
12801280
void TargetPassConfig::addMachineSSAOptimization() {
12811281
// Pre-ra tail duplication.
1282-
addPass(&EarlyTailDuplicateID);
1282+
addPass(&EarlyTailDuplicateLegacyID);
12831283

12841284
// Optimize PHIs before DCE: removing dead PHI cycles may make more
12851285
// instructions dead.
@@ -1507,7 +1507,7 @@ void TargetPassConfig::addMachineLateOptimization() {
15071507
// performance for targets that require Structured Control Flow.
15081508
// In addition it can also make CFG irreducible. Thus we disable it.
15091509
if (!TM->requiresStructuredCFG())
1510-
addPass(&TailDuplicateID);
1510+
addPass(&TailDuplicateLegacyID);
15111511

15121512
// Copy propagation.
15131513
addPass(&MachineCopyPropagationID);

‎llvm/lib/Passes/PassBuilder.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@
123123
#include "llvm/CodeGen/SlotIndexes.h"
124124
#include "llvm/CodeGen/StackColoring.h"
125125
#include "llvm/CodeGen/StackProtector.h"
126+
#include "llvm/CodeGen/TailDuplication.h"
126127
#include "llvm/CodeGen/TargetPassConfig.h"
127128
#include "llvm/CodeGen/TwoAddressInstructionPass.h"
128129
#include "llvm/CodeGen/TypePromotion.h"

‎llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,7 @@ void NVPTXPassConfig::addIRPasses() {
307307
disablePass(&PrologEpilogCodeInserterID);
308308
disablePass(&MachineLateInstrsCleanupID);
309309
disablePass(&MachineCopyPropagationID);
310-
disablePass(&TailDuplicateID);
310+
disablePass(&TailDuplicateLegacyID);
311311
disablePass(&StackMapLivenessID);
312312
disablePass(&PostRAMachineSinkingID);
313313
disablePass(&PostRASchedulerID);
@@ -436,7 +436,7 @@ void NVPTXPassConfig::addOptimizedRegAlloc() {
436436

437437
void NVPTXPassConfig::addMachineSSAOptimization() {
438438
// Pre-ra tail duplication.
439-
if (addPass(&EarlyTailDuplicateID))
439+
if (addPass(&EarlyTailDuplicateLegacyID))
440440
printAndVerify("After Pre-RegAlloc TailDuplicate");
441441

442442
// Optimize PHIs before DCE: removing dead PHI cycles may make more

0 commit comments

Comments
 (0)