Skip to content

Commit 6bf9489

Browse files
Mel-ChenMel-Chen
authored
[VPlan] Store memory alignment in VPWidenMemoryRecipe. nfc (#165255)
Add an member Alignment to VPWidenMemoryRecipe to store memory alignment directly in the recipe. Update constructors, clone(), and relevant methods to use this stored alignment instead of querying the IR instruction. This allows VPWidenLoadRecipe/VPWidenStoreRecipe to be constructed without relying on the original IR instruction in the future.
1 parent 2984a8d commit 6bf9489

File tree

5 files changed

+40
-35
lines changed

5 files changed

+40
-35
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7543,12 +7543,13 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
75437543
}
75447544
if (LoadInst *Load = dyn_cast<LoadInst>(I))
75457545
return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
7546-
VPIRMetadata(*Load, LVer), I->getDebugLoc());
7546+
Load->getAlign(), VPIRMetadata(*Load, LVer),
7547+
I->getDebugLoc());
75477548

75487549
StoreInst *Store = cast<StoreInst>(I);
75497550
return new VPWidenStoreRecipe(*Store, Ptr, Operands[0], Mask, Consecutive,
7550-
Reverse, VPIRMetadata(*Store, LVer),
7551-
I->getDebugLoc());
7551+
Reverse, Store->getAlign(),
7552+
VPIRMetadata(*Store, LVer), I->getDebugLoc());
75527553
}
75537554

75547555
/// Creates a VPWidenIntOrFpInductionRecpipe for \p Phi. If needed, it will also

‎llvm/lib/Transforms/Vectorize/VPlan.h‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3179,6 +3179,9 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
31793179
protected:
31803180
Instruction &Ingredient;
31813181

3182+
/// Alignment information for this memory access.
3183+
Align Alignment;
3184+
31823185
/// Whether the accessed addresses are consecutive.
31833186
bool Consecutive;
31843187

@@ -3198,10 +3201,10 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
31983201

31993202
VPWidenMemoryRecipe(const char unsigned SC, Instruction &I,
32003203
std::initializer_list<VPValue *> Operands,
3201-
bool Consecutive, bool Reverse,
3204+
bool Consecutive, bool Reverse, Align Alignment,
32023205
const VPIRMetadata &Metadata, DebugLoc DL)
32033206
: VPRecipeBase(SC, Operands, DL), VPIRMetadata(Metadata), Ingredient(I),
3204-
Consecutive(Consecutive), Reverse(Reverse) {
3207+
Alignment(Alignment), Consecutive(Consecutive), Reverse(Reverse) {
32053208
assert((Consecutive || !Reverse) && "Reverse implies consecutive");
32063209
}
32073210

@@ -3242,6 +3245,9 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
32423245
return isMasked() ? getOperand(getNumOperands() - 1) : nullptr;
32433246
}
32443247

3248+
/// Returns the alignment of the memory access.
3249+
Align getAlign() const { return Alignment; }
3250+
32453251
/// Generate the wide load/store.
32463252
void execute(VPTransformState &State) override {
32473253
llvm_unreachable("VPWidenMemoryRecipe should not be instantiated.");
@@ -3259,18 +3265,18 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
32593265
struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
32603266
public VPValue {
32613267
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
3262-
bool Consecutive, bool Reverse,
3268+
bool Consecutive, bool Reverse, Align Alignment,
32633269
const VPIRMetadata &Metadata, DebugLoc DL)
32643270
: VPWidenMemoryRecipe(VPDef::VPWidenLoadSC, Load, {Addr}, Consecutive,
3265-
Reverse, Metadata, DL),
3271+
Reverse, Alignment, Metadata, DL),
32663272
VPValue(this, &Load) {
32673273
setMask(Mask);
32683274
}
32693275

32703276
VPWidenLoadRecipe *clone() override {
32713277
return new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
3272-
getMask(), Consecutive, Reverse, *this,
3273-
getDebugLoc());
3278+
getMask(), Consecutive, Reverse, getAlign(),
3279+
*this, getDebugLoc());
32743280
}
32753281

32763282
VP_CLASSOF_IMPL(VPDef::VPWidenLoadSC);
@@ -3301,8 +3307,8 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
33013307
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue *Addr, VPValue &EVL,
33023308
VPValue *Mask)
33033309
: VPWidenMemoryRecipe(VPDef::VPWidenLoadEVLSC, L.getIngredient(),
3304-
{Addr, &EVL}, L.isConsecutive(), L.isReverse(), L,
3305-
L.getDebugLoc()),
3310+
{Addr, &EVL}, L.isConsecutive(), L.isReverse(),
3311+
L.getAlign(), L, L.getDebugLoc()),
33063312
VPValue(this, &getIngredient()) {
33073313
setMask(Mask);
33083314
}
@@ -3340,16 +3346,16 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
33403346
struct LLVM_ABI_FOR_TEST VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
33413347
VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal,
33423348
VPValue *Mask, bool Consecutive, bool Reverse,
3343-
const VPIRMetadata &Metadata, DebugLoc DL)
3349+
Align Alignment, const VPIRMetadata &Metadata, DebugLoc DL)
33443350
: VPWidenMemoryRecipe(VPDef::VPWidenStoreSC, Store, {Addr, StoredVal},
3345-
Consecutive, Reverse, Metadata, DL) {
3351+
Consecutive, Reverse, Alignment, Metadata, DL) {
33463352
setMask(Mask);
33473353
}
33483354

33493355
VPWidenStoreRecipe *clone() override {
33503356
return new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
33513357
getStoredValue(), getMask(), Consecutive,
3352-
Reverse, *this, getDebugLoc());
3358+
Reverse, getAlign(), *this, getDebugLoc());
33533359
}
33543360

33553361
VP_CLASSOF_IMPL(VPDef::VPWidenStoreSC);
@@ -3384,7 +3390,7 @@ struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
33843390
VPValue *Mask)
33853391
: VPWidenMemoryRecipe(VPDef::VPWidenStoreEVLSC, S.getIngredient(),
33863392
{Addr, S.getStoredValue(), &EVL}, S.isConsecutive(),
3387-
S.isReverse(), S, S.getDebugLoc()) {
3393+
S.isReverse(), S.getAlign(), S, S.getDebugLoc()) {
33883394
setMask(Mask);
33893395
}
33903396

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3525,7 +3525,6 @@ void VPPredInstPHIRecipe::print(raw_ostream &O, const Twine &Indent,
35253525
InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
35263526
VPCostContext &Ctx) const {
35273527
Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF);
3528-
const Align Alignment = getLoadStoreAlignment(&Ingredient);
35293528
unsigned AS = cast<PointerType>(Ctx.Types.inferScalarType(getAddr()))
35303529
->getAddressSpace();
35313530
unsigned Opcode = isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(this)
@@ -3575,7 +3574,6 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
35753574
void VPWidenLoadRecipe::execute(VPTransformState &State) {
35763575
Type *ScalarDataTy = getLoadStoreType(&Ingredient);
35773576
auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
3578-
const Align Alignment = getLoadStoreAlignment(&Ingredient);
35793577
bool CreateGather = !isConsecutive();
35803578

35813579
auto &Builder = State.Builder;
@@ -3630,7 +3628,6 @@ static Instruction *createReverseEVL(IRBuilderBase &Builder, Value *Operand,
36303628
void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
36313629
Type *ScalarDataTy = getLoadStoreType(&Ingredient);
36323630
auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
3633-
const Align Alignment = getLoadStoreAlignment(&Ingredient);
36343631
bool CreateGather = !isConsecutive();
36353632

36363633
auto &Builder = State.Builder;
@@ -3674,7 +3671,6 @@ InstructionCost VPWidenLoadEVLRecipe::computeCost(ElementCount VF,
36743671
// TODO: Using getMemoryOpCost() instead of getMaskedMemoryOpCost when we
36753672
// don't need to compare to the legacy cost model.
36763673
Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF);
3677-
const Align Alignment = getLoadStoreAlignment(&Ingredient);
36783674
unsigned AS = cast<PointerType>(Ctx.Types.inferScalarType(getAddr()))
36793675
->getAddressSpace();
36803676
InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost(
@@ -3700,7 +3696,6 @@ void VPWidenLoadEVLRecipe::print(raw_ostream &O, const Twine &Indent,
37003696
void VPWidenStoreRecipe::execute(VPTransformState &State) {
37013697
VPValue *StoredVPValue = getStoredValue();
37023698
bool CreateScatter = !isConsecutive();
3703-
const Align Alignment = getLoadStoreAlignment(&Ingredient);
37043699

37053700
auto &Builder = State.Builder;
37063701

@@ -3743,7 +3738,6 @@ void VPWidenStoreRecipe::print(raw_ostream &O, const Twine &Indent,
37433738
void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
37443739
VPValue *StoredValue = getStoredValue();
37453740
bool CreateScatter = !isConsecutive();
3746-
const Align Alignment = getLoadStoreAlignment(&Ingredient);
37473741

37483742
auto &Builder = State.Builder;
37493743

@@ -3786,7 +3780,6 @@ InstructionCost VPWidenStoreEVLRecipe::computeCost(ElementCount VF,
37863780
// TODO: Using getMemoryOpCost() instead of getMaskedMemoryOpCost when we
37873781
// don't need to compare to the legacy cost model.
37883782
Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF);
3789-
const Align Alignment = getLoadStoreAlignment(&Ingredient);
37903783
unsigned AS = cast<PointerType>(Ctx.Types.inferScalarType(getAddr()))
37913784
->getAddressSpace();
37923785
InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost(

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -91,13 +91,14 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
9191
if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
9292
NewRecipe = new VPWidenLoadRecipe(
9393
*Load, Ingredient.getOperand(0), nullptr /*Mask*/,
94-
false /*Consecutive*/, false /*Reverse*/, VPIRMetadata(*Load),
95-
Ingredient.getDebugLoc());
94+
false /*Consecutive*/, false /*Reverse*/, Load->getAlign(),
95+
VPIRMetadata(*Load), Ingredient.getDebugLoc());
9696
} else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
9797
NewRecipe = new VPWidenStoreRecipe(
9898
*Store, Ingredient.getOperand(1), Ingredient.getOperand(0),
9999
nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/,
100-
VPIRMetadata(*Store), Ingredient.getDebugLoc());
100+
Store->getAlign(), VPIRMetadata(*Store),
101+
Ingredient.getDebugLoc());
101102
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
102103
NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands());
103104
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
@@ -4234,10 +4235,11 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
42344235
if (auto *LoadGroup = dyn_cast<VPInterleaveRecipe>(R)) {
42354236
// Narrow interleave group to wide load, as transformed VPlan will only
42364237
// process one original iteration.
4238+
auto *LI =
4239+
cast<LoadInst>(LoadGroup->getInterleaveGroup()->getInsertPos());
42374240
auto *L = new VPWidenLoadRecipe(
4238-
*cast<LoadInst>(LoadGroup->getInterleaveGroup()->getInsertPos()),
4239-
LoadGroup->getAddr(), LoadGroup->getMask(), /*Consecutive=*/true,
4240-
/*Reverse=*/false, {}, LoadGroup->getDebugLoc());
4241+
*LI, LoadGroup->getAddr(), LoadGroup->getMask(), /*Consecutive=*/true,
4242+
/*Reverse=*/false, LI->getAlign(), {}, LoadGroup->getDebugLoc());
42414243
L->insertBefore(LoadGroup);
42424244
NarrowedOps.insert(L);
42434245
return L;
@@ -4280,10 +4282,11 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
42804282
Res = NarrowOp(Member0);
42814283
}
42824284

4285+
auto *SI =
4286+
cast<StoreInst>(StoreGroup->getInterleaveGroup()->getInsertPos());
42834287
auto *S = new VPWidenStoreRecipe(
4284-
*cast<StoreInst>(StoreGroup->getInterleaveGroup()->getInsertPos()),
4285-
StoreGroup->getAddr(), Res, nullptr, /*Consecutive=*/true,
4286-
/*Reverse=*/false, {}, StoreGroup->getDebugLoc());
4288+
*SI, StoreGroup->getAddr(), Res, nullptr, /*Consecutive=*/true,
4289+
/*Reverse=*/false, SI->getAlign(), {}, StoreGroup->getDebugLoc());
42874290
S->insertBefore(StoreGroup);
42884291
StoreGroup->eraseFromParent();
42894292
}

‎llvm/unittests/Transforms/Vectorize/VPlanTest.cpp‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1132,7 +1132,8 @@ TEST_F(VPRecipeTest, CastVPWidenMemoryRecipeToVPUserAndVPDef) {
11321132
new LoadInst(Int32, PoisonValue::get(Int32Ptr), "", false, Align(1));
11331133
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
11341134
VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
1135-
VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, {}, {});
1135+
VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, Load->getAlign(), {},
1136+
{});
11361137
EXPECT_TRUE(isa<VPUser>(&Recipe));
11371138
VPRecipeBase *BaseR = &Recipe;
11381139
EXPECT_TRUE(isa<VPUser>(BaseR));
@@ -1249,7 +1250,8 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
12491250
new LoadInst(Int32, PoisonValue::get(Int32Ptr), "", false, Align(1));
12501251
VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
12511252
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
1252-
VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, {}, {});
1253+
VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, Load->getAlign(),
1254+
{}, {});
12531255
EXPECT_FALSE(Recipe.mayHaveSideEffects());
12541256
EXPECT_TRUE(Recipe.mayReadFromMemory());
12551257
EXPECT_FALSE(Recipe.mayWriteToMemory());
@@ -1263,8 +1265,8 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
12631265
VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
12641266
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
12651267
VPValue *StoredV = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 3));
1266-
VPWidenStoreRecipe Recipe(*Store, Addr, StoredV, Mask, false, false, {},
1267-
{});
1268+
VPWidenStoreRecipe Recipe(*Store, Addr, StoredV, Mask, false, false,
1269+
Store->getAlign(), {}, {});
12681270
EXPECT_TRUE(Recipe.mayHaveSideEffects());
12691271
EXPECT_FALSE(Recipe.mayReadFromMemory());
12701272
EXPECT_TRUE(Recipe.mayWriteToMemory());

0 commit comments

Comments
 (0)