Skip to content

Commit 7551383

Browse files
arcbbbarcbbb
committed
[TTI] Replace getStridedMemoryOpCost with getIntrinsicInstrCost
1 parent 31417ba commit 7551383

File tree

7 files changed

+106
-103
lines changed

7 files changed

+106
-103
lines changed

‎llvm/include/llvm/Analysis/TargetTransformInfo.h‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,9 @@ class IntrinsicCostAttributes {
135135
InstructionCost ScalarizationCost = InstructionCost::getInvalid();
136136
TargetLibraryInfo const *LibInfo = nullptr;
137137

138+
MaybeAlign Alignment;
139+
bool VariableMask = false;
140+
138141
public:
139142
LLVM_ABI IntrinsicCostAttributes(
140143
Intrinsic::ID Id, const CallBase &CI,
@@ -146,6 +149,10 @@ class IntrinsicCostAttributes {
146149
FastMathFlags Flags = FastMathFlags(), const IntrinsicInst *I = nullptr,
147150
InstructionCost ScalarCost = InstructionCost::getInvalid());
148151

152+
LLVM_ABI IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
153+
ArrayRef<Type *> Tys, Align Alignment,
154+
bool VariableMask = false);
155+
149156
LLVM_ABI IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
150157
ArrayRef<const Value *> Args);
151158

@@ -160,6 +167,8 @@ class IntrinsicCostAttributes {
160167
const IntrinsicInst *getInst() const { return II; }
161168
Type *getReturnType() const { return RetTy; }
162169
FastMathFlags getFlags() const { return FMF; }
170+
MaybeAlign getAlign() const { return Alignment; }
171+
bool getVariableMask() const { return VariableMask; }
163172
InstructionCost getScalarizationCost() const { return ScalarizationCost; }
164173
const SmallVectorImpl<const Value *> &getArgs() const { return Arguments; }
165174
const SmallVectorImpl<Type *> &getArgTypes() const { return ParamTys; }
@@ -1586,20 +1595,6 @@ class TargetTransformInfo {
15861595
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
15871596
const Instruction *I = nullptr) const;
15881597

1589-
/// \return The cost of strided memory operations.
1590-
/// \p Opcode - is a type of memory access Load or Store
1591-
/// \p DataTy - a vector type of the data to be loaded or stored
1592-
/// \p Ptr - pointer [or vector of pointers] - address[es] in memory
1593-
/// \p VariableMask - true when the memory access is predicated with a mask
1594-
/// that is not a compile-time constant
1595-
/// \p Alignment - alignment of single element
1596-
/// \p I - the optional original context instruction, if one exists, e.g. the
1597-
/// load/store to transform or the call to the gather/scatter intrinsic
1598-
LLVM_ABI InstructionCost getStridedMemoryOpCost(
1599-
unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
1600-
Align Alignment, TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
1601-
const Instruction *I = nullptr) const;
1602-
16031598
/// \return The cost of the interleaved memory operation.
16041599
/// \p Opcode is the memory operation code
16051600
/// \p VecTy is the vector type of the interleaved access.

‎llvm/include/llvm/Analysis/TargetTransformInfoImpl.h‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -862,14 +862,6 @@ class TargetTransformInfoImplBase {
862862
return 1;
863863
}
864864

865-
virtual InstructionCost
866-
getStridedMemoryOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
867-
bool VariableMask, Align Alignment,
868-
TTI::TargetCostKind CostKind,
869-
const Instruction *I = nullptr) const {
870-
return InstructionCost::getInvalid();
871-
}
872-
873865
virtual InstructionCost getInterleavedMemoryOpCost(
874866
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
875867
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,

‎llvm/include/llvm/CodeGen/BasicTTIImpl.h‎

Lines changed: 18 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1574,18 +1574,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
15741574
/*IsGatherScatter*/ true, CostKind);
15751575
}
15761576

1577-
InstructionCost getStridedMemoryOpCost(unsigned Opcode, Type *DataTy,
1578-
const Value *Ptr, bool VariableMask,
1579-
Align Alignment,
1580-
TTI::TargetCostKind CostKind,
1581-
const Instruction *I) const override {
1582-
// For a target without strided memory operations (or for an illegal
1583-
// operation type on one which does), assume we lower to a gather/scatter
1584-
// operation. (Which may in turn be scalarized.)
1585-
return thisT()->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
1586-
Alignment, CostKind, I);
1587-
}
1588-
15891577
InstructionCost getInterleavedMemoryOpCost(
15901578
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
15911579
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
@@ -1958,27 +1946,26 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
19581946
}
19591947
case Intrinsic::experimental_vp_strided_store: {
19601948
const Value *Data = Args[0];
1961-
const Value *Ptr = Args[1];
19621949
const Value *Mask = Args[3];
19631950
const Value *EVL = Args[4];
19641951
bool VarMask = !isa<Constant>(Mask) || !isa<Constant>(EVL);
19651952
Type *EltTy = cast<VectorType>(Data->getType())->getElementType();
19661953
Align Alignment =
19671954
I->getParamAlign(1).value_or(thisT()->DL.getABITypeAlign(EltTy));
1968-
return thisT()->getStridedMemoryOpCost(Instruction::Store,
1969-
Data->getType(), Ptr, VarMask,
1970-
Alignment, CostKind, I);
1955+
return thisT()->getCommonMaskedMemoryOpCost(
1956+
Instruction::Store, Data->getType(), Alignment, VarMask,
1957+
/*IsGatherScatter*/ true, CostKind);
19711958
}
19721959
case Intrinsic::experimental_vp_strided_load: {
1973-
const Value *Ptr = Args[0];
19741960
const Value *Mask = Args[2];
19751961
const Value *EVL = Args[3];
19761962
bool VarMask = !isa<Constant>(Mask) || !isa<Constant>(EVL);
19771963
Type *EltTy = cast<VectorType>(RetTy)->getElementType();
19781964
Align Alignment =
19791965
I->getParamAlign(0).value_or(thisT()->DL.getABITypeAlign(EltTy));
1980-
return thisT()->getStridedMemoryOpCost(Instruction::Load, RetTy, Ptr,
1981-
VarMask, Alignment, CostKind, I);
1966+
return thisT()->getCommonMaskedMemoryOpCost(
1967+
Instruction::Load, RetTy, Alignment, VarMask,
1968+
/*IsGatherScatter*/ true, CostKind);
19821969
}
19831970
case Intrinsic::stepvector: {
19841971
if (isa<ScalableVectorType>(RetTy))
@@ -2418,17 +2405,21 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
24182405
}
24192406
case Intrinsic::experimental_vp_strided_store: {
24202407
auto *Ty = cast<VectorType>(ICA.getArgTypes()[0]);
2421-
Align Alignment = thisT()->DL.getABITypeAlign(Ty->getElementType());
2422-
return thisT()->getStridedMemoryOpCost(
2423-
Instruction::Store, Ty, /*Ptr=*/nullptr, /*VariableMask=*/true,
2424-
Alignment, CostKind, ICA.getInst());
2408+
Align Alignment = ICA.getAlign().value_or(
2409+
thisT()->DL.getABITypeAlign(Ty->getElementType()));
2410+
return thisT()->getCommonMaskedMemoryOpCost(
2411+
Instruction::Store, Ty, Alignment,
2412+
/*VariableMask=*/true,
2413+
/*IsGatherScatter*/ true, CostKind);
24252414
}
24262415
case Intrinsic::experimental_vp_strided_load: {
24272416
auto *Ty = cast<VectorType>(ICA.getReturnType());
2428-
Align Alignment = thisT()->DL.getABITypeAlign(Ty->getElementType());
2429-
return thisT()->getStridedMemoryOpCost(
2430-
Instruction::Load, Ty, /*Ptr=*/nullptr, /*VariableMask=*/true,
2431-
Alignment, CostKind, ICA.getInst());
2417+
Align Alignment = ICA.getAlign().value_or(
2418+
thisT()->DL.getABITypeAlign(Ty->getElementType()));
2419+
return thisT()->getCommonMaskedMemoryOpCost(
2420+
Instruction::Load, Ty, Alignment,
2421+
/*VariableMask=*/true,
2422+
/*IsGatherScatter*/ true, CostKind);
24322423
}
24332424
case Intrinsic::vector_reduce_add:
24342425
case Intrinsic::vector_reduce_mul:

‎llvm/lib/Analysis/TargetTransformInfo.cpp‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,14 @@ IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
9696
ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end());
9797
}
9898

99+
IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
100+
ArrayRef<Type *> Tys,
101+
Align Alignment,
102+
bool VariableMask)
103+
: RetTy(RTy), IID(Id), Alignment(Alignment), VariableMask(VariableMask) {
104+
ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end());
105+
}
106+
99107
IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *Ty,
100108
ArrayRef<const Value *> Args)
101109
: RetTy(Ty), IID(Id) {
@@ -1210,15 +1218,6 @@ InstructionCost TargetTransformInfo::getExpandCompressMemoryOpCost(
12101218
return Cost;
12111219
}
12121220

1213-
InstructionCost TargetTransformInfo::getStridedMemoryOpCost(
1214-
unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
1215-
Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
1216-
InstructionCost Cost = TTIImpl->getStridedMemoryOpCost(
1217-
Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I);
1218-
assert(Cost >= 0 && "TTI should not produce negative costs!");
1219-
return Cost;
1220-
}
1221-
12221221
InstructionCost TargetTransformInfo::getInterleavedMemoryOpCost(
12231222
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
12241223
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,

‎llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp‎

Lines changed: 36 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1172,29 +1172,6 @@ InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
11721172
LT.first * getRISCVInstructionCost(Opcodes, LT.second, CostKind);
11731173
}
11741174

1175-
InstructionCost RISCVTTIImpl::getStridedMemoryOpCost(
1176-
unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
1177-
Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
1178-
if (((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
1179-
!isLegalStridedLoadStore(DataTy, Alignment)) ||
1180-
(Opcode != Instruction::Load && Opcode != Instruction::Store))
1181-
return BaseT::getStridedMemoryOpCost(Opcode, DataTy, Ptr, VariableMask,
1182-
Alignment, CostKind, I);
1183-
1184-
if (CostKind == TTI::TCK_CodeSize)
1185-
return TTI::TCC_Basic;
1186-
1187-
// Cost is proportional to the number of memory operations implied. For
1188-
// scalable vectors, we use an estimate on that number since we don't
1189-
// know exactly what VL will be.
1190-
auto &VTy = *cast<VectorType>(DataTy);
1191-
InstructionCost MemOpCost =
1192-
getMemoryOpCost(Opcode, VTy.getElementType(), Alignment, 0, CostKind,
1193-
{TTI::OK_AnyValue, TTI::OP_None}, I);
1194-
unsigned NumLoads = getEstimatedVLFor(&VTy);
1195-
return NumLoads * MemOpCost;
1196-
}
1197-
11981175
InstructionCost
11991176
RISCVTTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
12001177
// FIXME: This is a property of the default vector convention, not
@@ -1561,6 +1538,42 @@ RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
15611538
cast<VectorType>(ICA.getArgTypes()[0]), {}, CostKind,
15621539
0, cast<VectorType>(ICA.getReturnType()));
15631540
}
1541+
case Intrinsic::experimental_vp_strided_load:
1542+
case Intrinsic::experimental_vp_strided_store: {
1543+
if (CostKind == TTI::TCK_CodeSize)
1544+
return TTI::TCC_Basic;
1545+
1546+
auto *DataTy = (ICA.getID() == Intrinsic::experimental_vp_strided_load)
1547+
? cast<VectorType>(ICA.getReturnType())
1548+
: cast<VectorType>(ICA.getArgTypes()[0]);
1549+
Type *EltTy = DataTy->getElementType();
1550+
1551+
Align ABITyAlign = DL.getABITypeAlign(EltTy);
1552+
1553+
const IntrinsicInst *I = ICA.getInst();
1554+
Align Alignment;
1555+
if (ICA.isTypeBasedOnly())
1556+
Alignment = ICA.getAlign().value_or(ABITyAlign);
1557+
else {
1558+
unsigned Index = (ICA.getID() == Intrinsic::experimental_vp_strided_load) ? 0 : 1;
1559+
Alignment = I->getParamAlign(Index).value_or(ABITyAlign);
1560+
}
1561+
1562+
if (!isLegalStridedLoadStore(DataTy, Alignment))
1563+
return BaseT::getIntrinsicInstrCost(ICA, CostKind);
1564+
1565+
unsigned Opcode = ICA.getID() == Intrinsic::experimental_vp_strided_load
1566+
? Instruction::Load
1567+
: Instruction::Store;
1568+
// Cost is proportional to the number of memory operations implied. For
1569+
// scalable vectors, we use an estimate on that number since we don't
1570+
// know exactly what VL will be.
1571+
InstructionCost MemOpCost =
1572+
getMemoryOpCost(Opcode, EltTy, Alignment, 0, CostKind,
1573+
{TTI::OK_AnyValue, TTI::OP_None}, I);
1574+
unsigned NumLoads = getEstimatedVLFor(DataTy);
1575+
return NumLoads * MemOpCost;
1576+
}
15641577
case Intrinsic::fptoui_sat:
15651578
case Intrinsic::fptosi_sat: {
15661579
InstructionCost Cost = 0;

‎llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -202,12 +202,6 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
202202
Align Alignment, TTI::TargetCostKind CostKind,
203203
const Instruction *I = nullptr) const override;
204204

205-
InstructionCost getStridedMemoryOpCost(unsigned Opcode, Type *DataTy,
206-
const Value *Ptr, bool VariableMask,
207-
Align Alignment,
208-
TTI::TargetCostKind CostKind,
209-
const Instruction *I) const override;
210-
211205
InstructionCost
212206
getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const override;
213207

‎llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp‎

Lines changed: 35 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7224,10 +7224,13 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
72247224
VectorGEPCost;
72257225
break;
72267226
case LoadsState::StridedVectorize:
7227-
VecLdCost += TTI.getStridedMemoryOpCost(Instruction::Load, SubVecTy,
7228-
LI0->getPointerOperand(),
7229-
/*VariableMask=*/false,
7230-
CommonAlignment, CostKind) +
7227+
VecLdCost += TTI.getIntrinsicInstrCost(
7228+
{Intrinsic::experimental_vp_strided_load,
7229+
SubVecTy,
7230+
{},
7231+
CommonAlignment,
7232+
/*VariableMask=*/false},
7233+
CostKind) +
72317234
VectorGEPCost;
72327235
break;
72337236
case LoadsState::CompressVectorize:
@@ -13191,9 +13194,13 @@ void BoUpSLP::transformNodes() {
1319113194
BaseLI->getPointerAddressSpace(), CostKind,
1319213195
TTI::OperandValueInfo()) +
1319313196
::getShuffleCost(*TTI, TTI::SK_Reverse, VecTy, Mask, CostKind);
13194-
InstructionCost StridedCost = TTI->getStridedMemoryOpCost(
13195-
Instruction::Load, VecTy, BaseLI->getPointerOperand(),
13196-
/*VariableMask=*/false, CommonAlignment, CostKind, BaseLI);
13197+
InstructionCost StridedCost =
13198+
TTI->getIntrinsicInstrCost({Intrinsic::experimental_vp_strided_load,
13199+
VecTy,
13200+
{},
13201+
CommonAlignment,
13202+
/*VariableMask=*/false},
13203+
CostKind);
1319713204
if (StridedCost < OriginalVecCost || ForceStridedLoads) {
1319813205
// Strided load is more profitable than consecutive load + reverse -
1319913206
// transform the node to strided load.
@@ -13226,9 +13233,13 @@ void BoUpSLP::transformNodes() {
1322613233
BaseSI->getPointerAddressSpace(), CostKind,
1322713234
TTI::OperandValueInfo()) +
1322813235
::getShuffleCost(*TTI, TTI::SK_Reverse, VecTy, Mask, CostKind);
13229-
InstructionCost StridedCost = TTI->getStridedMemoryOpCost(
13230-
Instruction::Store, VecTy, BaseSI->getPointerOperand(),
13231-
/*VariableMask=*/false, CommonAlignment, CostKind, BaseSI);
13236+
InstructionCost StridedCost = TTI->getIntrinsicInstrCost(
13237+
{Intrinsic::experimental_vp_strided_store,
13238+
Type::getVoidTy(VecTy->getContext()),
13239+
{VecTy},
13240+
CommonAlignment,
13241+
/*VariableMask=*/false},
13242+
CostKind);
1323213243
if (StridedCost < OriginalVecCost)
1323313244
// Strided store is more profitable than reverse + consecutive store -
1323413245
// transform the node to strided store.
@@ -14991,9 +15002,13 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
1499115002
case TreeEntry::StridedVectorize: {
1499215003
Align CommonAlignment =
1499315004
computeCommonAlignment<LoadInst>(UniqueValues.getArrayRef());
14994-
VecLdCost = TTI->getStridedMemoryOpCost(
14995-
Instruction::Load, VecTy, LI0->getPointerOperand(),
14996-
/*VariableMask=*/false, CommonAlignment, CostKind);
15005+
VecLdCost =
15006+
TTI->getIntrinsicInstrCost({Intrinsic::experimental_vp_strided_load,
15007+
VecTy,
15008+
{},
15009+
CommonAlignment,
15010+
/*VariableMask=*/false},
15011+
CostKind);
1499715012
break;
1499815013
}
1499915014
case TreeEntry::CompressVectorize: {
@@ -15084,9 +15099,13 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
1508415099
if (E->State == TreeEntry::StridedVectorize) {
1508515100
Align CommonAlignment =
1508615101
computeCommonAlignment<StoreInst>(UniqueValues.getArrayRef());
15087-
VecStCost = TTI->getStridedMemoryOpCost(
15088-
Instruction::Store, VecTy, BaseSI->getPointerOperand(),
15089-
/*VariableMask=*/false, CommonAlignment, CostKind);
15102+
VecStCost = TTI->getIntrinsicInstrCost(
15103+
{Intrinsic::experimental_vp_strided_store,
15104+
Type::getVoidTy(VecTy->getContext()),
15105+
{VecTy},
15106+
CommonAlignment,
15107+
/*VariableMask=*/false},
15108+
CostKind);
1509015109
} else {
1509115110
assert(E->State == TreeEntry::Vectorize &&
1509215111
"Expected either strided or consecutive stores.");

0 commit comments

Comments
 (0)