Skip to content

Commit 3cba379

Browse files
fhahnfhahn
authored
[VPlan] Populate and use VPIRMetadata from VPInstructions (NFC) (#167253)
Update VPlan to populate VPIRMetadata during VPInstruction construction and use it when creating widened recipes, instead of constructing VPIRMetadata from the underlying IR instruction each time. This centralizes VPIRMetadata in VPInstructions and ensures metadata is consistently available throughout VPlan transformations. PR: llvm/llvm-project#167253
1 parent ed617bd commit 3cba379

File tree

10 files changed

+127
-123
lines changed

10 files changed

+127
-123
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h‎

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,11 @@ class VPBuilder {
6363
}
6464

6565
VPInstruction *createInstruction(unsigned Opcode,
66-
ArrayRef<VPValue *> Operands, DebugLoc DL,
66+
ArrayRef<VPValue *> Operands,
67+
const VPIRMetadata &MD, DebugLoc DL,
6768
const Twine &Name = "") {
68-
return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL, Name));
69+
return tryInsertInstruction(
70+
new VPInstruction(Opcode, Operands, {}, MD, DL, Name));
6971
}
7072

7173
public:
@@ -150,17 +152,17 @@ class VPBuilder {
150152
/// its underlying Instruction.
151153
VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
152154
Instruction *Inst = nullptr,
155+
const VPIRMetadata &MD = {},
156+
DebugLoc DL = DebugLoc::getUnknown(),
153157
const Twine &Name = "") {
154-
DebugLoc DL = DebugLoc::getUnknown();
155-
if (Inst)
156-
DL = Inst->getDebugLoc();
157-
VPInstruction *NewVPInst = createInstruction(Opcode, Operands, DL, Name);
158+
VPInstruction *NewVPInst = tryInsertInstruction(
159+
new VPInstruction(Opcode, Operands, {}, MD, DL, Name));
158160
NewVPInst->setUnderlyingValue(Inst);
159161
return NewVPInst;
160162
}
161163
VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
162164
DebugLoc DL, const Twine &Name = "") {
163-
return createInstruction(Opcode, Operands, DL, Name);
165+
return createInstruction(Opcode, Operands, {}, DL, Name);
164166
}
165167
VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
166168
const VPIRFlags &Flags,
@@ -174,8 +176,8 @@ class VPBuilder {
174176
Type *ResultTy, const VPIRFlags &Flags = {},
175177
DebugLoc DL = DebugLoc::getUnknown(),
176178
const Twine &Name = "") {
177-
return tryInsertInstruction(
178-
new VPInstructionWithType(Opcode, Operands, ResultTy, Flags, DL, Name));
179+
return tryInsertInstruction(new VPInstructionWithType(
180+
Opcode, Operands, ResultTy, Flags, {}, DL, Name));
179181
}
180182

181183
VPInstruction *createOverflowingOp(
@@ -189,13 +191,14 @@ class VPBuilder {
189191
VPInstruction *createNot(VPValue *Operand,
190192
DebugLoc DL = DebugLoc::getUnknown(),
191193
const Twine &Name = "") {
192-
return createInstruction(VPInstruction::Not, {Operand}, DL, Name);
194+
return createInstruction(VPInstruction::Not, {Operand}, {}, DL, Name);
193195
}
194196

195197
VPInstruction *createAnd(VPValue *LHS, VPValue *RHS,
196198
DebugLoc DL = DebugLoc::getUnknown(),
197199
const Twine &Name = "") {
198-
return createInstruction(Instruction::BinaryOps::And, {LHS, RHS}, DL, Name);
200+
return createInstruction(Instruction::BinaryOps::And, {LHS, RHS}, {}, DL,
201+
Name);
199202
}
200203

201204
VPInstruction *createOr(VPValue *LHS, VPValue *RHS,
@@ -210,20 +213,18 @@ class VPBuilder {
210213
VPInstruction *createLogicalAnd(VPValue *LHS, VPValue *RHS,
211214
DebugLoc DL = DebugLoc::getUnknown(),
212215
const Twine &Name = "") {
213-
return tryInsertInstruction(
214-
new VPInstruction(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name));
216+
return createNaryOp(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name);
215217
}
216218

217219
VPInstruction *
218220
createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,
219221
DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "",
220222
std::optional<FastMathFlags> FMFs = std::nullopt) {
221-
auto *Select =
222-
FMFs ? new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
223-
*FMFs, {}, DL, Name)
224-
: new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
225-
DL, Name);
226-
return tryInsertInstruction(Select);
223+
if (!FMFs)
224+
return createNaryOp(Instruction::Select, {Cond, TrueVal, FalseVal}, DL,
225+
Name);
226+
return tryInsertInstruction(new VPInstruction(
227+
Instruction::Select, {Cond, TrueVal, FalseVal}, *FMFs, {}, DL, Name));
227228
}
228229

229230
/// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A
@@ -306,7 +307,7 @@ class VPBuilder {
306307
const VPIRFlags &Flags = {},
307308
const VPIRMetadata &Metadata = {}) {
308309
return tryInsertInstruction(
309-
new VPInstructionWithType(Opcode, Op, ResultTy, DL, Flags, Metadata));
310+
new VPInstructionWithType(Opcode, Op, ResultTy, Flags, Metadata, DL));
310311
}
311312

312313
VPValue *createScalarZExtOrTrunc(VPValue *Op, Type *ResultTy, Type *SrcTy,

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -7616,14 +7616,13 @@ VPWidenMemoryRecipe *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
76167616
}
76177617
if (VPI->getOpcode() == Instruction::Load) {
76187618
auto *Load = cast<LoadInst>(I);
7619-
return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
7620-
VPIRMetadata(*Load, LVer), I->getDebugLoc());
7619+
return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse, *VPI,
7620+
VPI->getDebugLoc());
76217621
}
76227622

76237623
StoreInst *Store = cast<StoreInst>(I);
76247624
return new VPWidenStoreRecipe(*Store, Ptr, VPI->getOperand(0), Mask,
7625-
Consecutive, Reverse,
7626-
VPIRMetadata(*Store, LVer), VPI->getDebugLoc());
7625+
Consecutive, Reverse, *VPI, VPI->getDebugLoc());
76277626
}
76287627

76297628
/// Creates a VPWidenIntOrFpInductionRecipe for \p PhiR. If needed, it will
@@ -7751,7 +7750,7 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI,
77517750
},
77527751
Range);
77537752
if (ShouldUseVectorIntrinsic)
7754-
return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(),
7753+
return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), *VPI,
77557754
VPI->getDebugLoc());
77567755

77577756
Function *Variant = nullptr;
@@ -7843,7 +7842,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
78437842
auto *SafeRHS =
78447843
Builder.createSelect(Mask, Ops[1], One, VPI->getDebugLoc());
78457844
Ops[1] = SafeRHS;
7846-
return new VPWidenRecipe(*I, Ops);
7845+
return new VPWidenRecipe(*I, Ops, *VPI, VPI->getDebugLoc());
78477846
}
78487847
[[fallthrough]];
78497848
}
@@ -7889,15 +7888,15 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
78897888
// For other binops, the legacy cost model only checks the second operand.
78907889
NewOps[1] = GetConstantViaSCEV(NewOps[1]);
78917890
}
7892-
return new VPWidenRecipe(*I, NewOps);
7891+
return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc());
78937892
}
78947893
case Instruction::ExtractValue: {
78957894
SmallVector<VPValue *> NewOps(VPI->operands());
78967895
auto *EVI = cast<ExtractValueInst>(I);
78977896
assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
78987897
unsigned Idx = EVI->getIndices()[0];
78997898
NewOps.push_back(Plan.getConstantInt(32, Idx));
7900-
return new VPWidenRecipe(*I, NewOps);
7899+
return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc());
79017900
}
79027901
};
79037902
}
@@ -7981,8 +7980,8 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
79817980
assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
79827981
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
79837982
"Should not predicate a uniform recipe");
7984-
auto *Recipe = new VPReplicateRecipe(I, VPI->operands(), IsUniform,
7985-
BlockInMask, VPIRMetadata(*I, LVer));
7983+
auto *Recipe =
7984+
new VPReplicateRecipe(I, VPI->operands(), IsUniform, BlockInMask, *VPI);
79867985
return Recipe;
79877986
}
79887987

@@ -8235,13 +8234,14 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
82358234
return new VPWidenGEPRecipe(cast<GetElementPtrInst>(Instr), R->operands());
82368235

82378236
if (VPI->getOpcode() == Instruction::Select)
8238-
return new VPWidenSelectRecipe(*cast<SelectInst>(Instr), R->operands());
8237+
return new VPWidenSelectRecipe(*cast<SelectInst>(Instr), R->operands(),
8238+
*VPI);
82398239

82408240
if (Instruction::isCast(VPI->getOpcode())) {
82418241
auto *CastR = cast<VPInstructionWithType>(R);
82428242
auto *CI = cast<CastInst>(Instr);
82438243
return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(0),
8244-
CastR->getResultType(), *CI);
8244+
CastR->getResultType(), *CI, *VPI);
82458245
}
82468246

82478247
return tryToWiden(VPI);
@@ -8269,7 +8269,8 @@ VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction,
82698269
SmallVector<VPValue *, 2> Ops;
82708270
Ops.push_back(Plan.getOrAddLiveIn(Zero));
82718271
Ops.push_back(BinOp);
8272-
BinOp = new VPWidenRecipe(*ReductionI, Ops);
8272+
BinOp = new VPWidenRecipe(*ReductionI, Ops, VPIRMetadata(),
8273+
ReductionI->getDebugLoc());
82738274
Builder.insert(BinOp->getDefiningRecipe());
82748275
ReductionOpcode = Instruction::Add;
82758276
}
@@ -8302,7 +8303,7 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
83028303
// candidates built later for specific VF ranges.
83038304
auto VPlan0 = VPlanTransforms::buildVPlan0(
83048305
OrigLoop, *LI, Legal->getWidestInductionType(),
8305-
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE);
8306+
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE, &LVer);
83068307

83078308
auto MaxVFTimes2 = MaxVF * 2;
83088309
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
@@ -8408,7 +8409,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
84088409
// VPInstructions in the loop.
84098410
// ---------------------------------------------------------------------------
84108411
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
8411-
Builder, BlockMaskCache, LVer);
8412+
Builder, BlockMaskCache);
84128413
// TODO: Handle partial reductions with EVL tail folding.
84138414
if (!CM.foldTailWithEVL())
84148415
RecipeBuilder.collectScaledReductions(Range);
@@ -8453,9 +8454,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
84538454
Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
84548455
// Only create recipe for the final invariant store of the reduction.
84558456
if (Legal->isInvariantStoreOfReduction(SI)) {
8456-
auto *Recipe =
8457-
new VPReplicateRecipe(SI, R.operands(), true /* IsUniform */,
8458-
nullptr /*Mask*/, VPIRMetadata(*SI, LVer));
8457+
auto *Recipe = new VPReplicateRecipe(
8458+
SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/,
8459+
*cast<VPInstruction>(SingleDef));
84598460
Recipe->insertBefore(*MiddleVPBB, MBIP);
84608461
}
84618462
R.eraseFromParent();
@@ -8606,7 +8607,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
86068607
// addScalarResumePhis.
86078608
DenseMap<VPBasicBlock *, VPValue *> BlockMaskCache;
86088609
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
8609-
Builder, BlockMaskCache, nullptr /*LVer*/);
8610+
Builder, BlockMaskCache);
86108611
for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
86118612
if (isa<VPCanonicalIVPHIRecipe>(&R))
86128613
continue;

‎llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -84,10 +84,6 @@ class VPRecipeBuilder {
8484
/// A mapping of partial reduction exit instructions to their scaling factor.
8585
DenseMap<const Instruction *, unsigned> ScaledReductionMap;
8686

87-
/// Loop versioning instance for getting noalias metadata guaranteed by
88-
/// runtime checks.
89-
LoopVersioning *LVer;
90-
9187
/// Check if \p I can be widened at the start of \p Range and possibly
9288
/// decrease the range such that the returned value holds for the entire \p
9389
/// Range. The function should not be called for memory instructions or calls.
@@ -144,11 +140,9 @@ class VPRecipeBuilder {
144140
LoopVectorizationLegality *Legal,
145141
LoopVectorizationCostModel &CM,
146142
PredicatedScalarEvolution &PSE, VPBuilder &Builder,
147-
DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache,
148-
LoopVersioning *LVer)
143+
DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache)
149144
: Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal),
150-
CM(CM), PSE(PSE), Builder(Builder), BlockMaskCache(BlockMaskCache),
151-
LVer(LVer) {}
145+
CM(CM), PSE(PSE), Builder(Builder), BlockMaskCache(BlockMaskCache) {}
152146

153147
std::optional<unsigned> getScalingForReduction(const Instruction *ExitInst) {
154148
auto It = ScaledReductionMap.find(ExitInst);

0 commit comments

Comments
 (0)