Skip to content

Commit bcfe3dd

Browse files
arcbbbarcbbb
committed
Add getMemIntrinsicInstrCost
1 parent c4254cd commit bcfe3dd

File tree

7 files changed

+195
-161
lines changed

7 files changed

+195
-161
lines changed

‎llvm/include/llvm/Analysis/TargetTransformInfo.h‎

Lines changed: 31 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -125,26 +125,51 @@ struct HardwareLoopInfo {
125125

126126
/// Information for memory intrinsic cost model.
127127
class MemIntrinsicCostAttributes {
128+
/// Optional context instruction, if one exists, e.g. the
129+
/// load/store to transform to the intrinsic.
130+
const Instruction *I = nullptr;
131+
132+
/// Address in memory.
133+
const Value *Ptr = nullptr;
134+
128135
/// Vector type of the data to be loaded or stored.
129136
Type *DataTy = nullptr;
130137

131138
/// ID of the memory intrinsic.
132139
Intrinsic::ID IID;
133140

141+
/// True when the memory access is predicated with a mask
142+
/// that is not a compile-time constant.
143+
bool VariableMask = true;
144+
134145
/// Address space of the pointer.
135146
unsigned AddressSpace = 0;
136147

137148
/// Alignment of single element.
138149
Align Alignment;
139150

140151
public:
152+
LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
153+
const Value *Ptr, bool VariableMask,
154+
Align Alignment,
155+
const Instruction *I = nullptr)
156+
: I(I), Ptr(Ptr), DataTy(DataTy), IID(Id), VariableMask(VariableMask),
157+
Alignment(Alignment) {}
141158
LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
142159
Align Alignment, unsigned AddressSpace)
143160
: DataTy(DataTy), IID(Id), AddressSpace(AddressSpace),
144161
Alignment(Alignment) {}
162+
LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
163+
bool VariableMask, Align Alignment,
164+
const Instruction *I = nullptr)
165+
: I(I), DataTy(DataTy), IID(Id), VariableMask(VariableMask),
166+
Alignment(Alignment) {}
145167

146168
Intrinsic::ID getID() const { return IID; }
169+
const Instruction *getInst() const { return I; }
170+
const Value *getPointer() const { return Ptr; }
147171
Type *getDataType() const { return DataTy; }
172+
bool getVariableMask() const { return VariableMask; }
148173
unsigned getAddressSpace() const { return AddressSpace; }
149174
Align getAlignment() const { return Alignment; }
150175
};
@@ -1584,52 +1609,6 @@ class TargetTransformInfo {
15841609
OperandValueInfo OpdInfo = {OK_AnyValue, OP_None},
15851610
const Instruction *I = nullptr) const;
15861611

1587-
/// \return The cost of masked Load and Store instructions.
1588-
LLVM_ABI InstructionCost getMaskedMemoryOpCost(
1589-
const MemIntrinsicCostAttributes &MICA,
1590-
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const;
1591-
1592-
/// \return The cost of Gather or Scatter operation
1593-
/// \p Opcode - is a type of memory access Load or Store
1594-
/// \p DataTy - a vector type of the data to be loaded or stored
1595-
/// \p Ptr - pointer [or vector of pointers] - address[es] in memory
1596-
/// \p VariableMask - true when the memory access is predicated with a mask
1597-
/// that is not a compile-time constant
1598-
/// \p Alignment - alignment of single element
1599-
/// \p I - the optional original context instruction, if one exists, e.g. the
1600-
/// load/store to transform or the call to the gather/scatter intrinsic
1601-
LLVM_ABI InstructionCost getGatherScatterOpCost(
1602-
unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
1603-
Align Alignment, TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
1604-
const Instruction *I = nullptr) const;
1605-
1606-
/// \return The cost of Expand Load or Compress Store operation
1607-
/// \p Opcode - is a type of memory access Load or Store
1608-
/// \p Src - a vector type of the data to be loaded or stored
1609-
/// \p VariableMask - true when the memory access is predicated with a mask
1610-
/// that is not a compile-time constant
1611-
/// \p Alignment - alignment of single element
1612-
/// \p I - the optional original context instruction, if one exists, e.g. the
1613-
/// load/store to transform or the call to the gather/scatter intrinsic
1614-
LLVM_ABI InstructionCost getExpandCompressMemoryOpCost(
1615-
unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
1616-
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
1617-
const Instruction *I = nullptr) const;
1618-
1619-
/// \return The cost of strided memory operations.
1620-
/// \p Opcode - is a type of memory access Load or Store
1621-
/// \p DataTy - a vector type of the data to be loaded or stored
1622-
/// \p Ptr - pointer [or vector of pointers] - address[es] in memory
1623-
/// \p VariableMask - true when the memory access is predicated with a mask
1624-
/// that is not a compile-time constant
1625-
/// \p Alignment - alignment of single element
1626-
/// \p I - the optional original context instruction, if one exists, e.g. the
1627-
/// load/store to transform or the call to the gather/scatter intrinsic
1628-
LLVM_ABI InstructionCost getStridedMemoryOpCost(
1629-
unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
1630-
Align Alignment, TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
1631-
const Instruction *I = nullptr) const;
1632-
16331612
/// \return The cost of the interleaved memory operation.
16341613
/// \p Opcode is the memory operation code
16351614
/// \p VecTy is the vector type of the interleaved access.
@@ -1708,6 +1687,12 @@ class TargetTransformInfo {
17081687
LLVM_ABI InstructionCost getIntrinsicInstrCost(
17091688
const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind) const;
17101689

1690+
/// \returns The cost of memory intrinsic instructions.
1691+
/// Used when IntrinsicInst is not materialized.
1692+
LLVM_ABI InstructionCost
1693+
getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
1694+
TTI::TargetCostKind CostKind) const;
1695+
17111696
/// \returns The cost of Call instructions.
17121697
LLVM_ABI InstructionCost getCallInstrCost(
17131698
Function *F, Type *RetTy, ArrayRef<Type *> Tys,

‎llvm/include/llvm/Analysis/TargetTransformInfoImpl.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -927,6 +927,11 @@ class TargetTransformInfoImplBase {
927927
return 1;
928928
}
929929

930+
virtual InstructionCost
931+
getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
932+
TTI::TargetCostKind CostKind) const {
933+
return 1;
934+
}
930935
virtual InstructionCost getCallInstrCost(Function *F, Type *RetTy,
931936
ArrayRef<Type *> Tys,
932937
TTI::TargetCostKind CostKind) const {

‎llvm/include/llvm/CodeGen/BasicTTIImpl.h‎

Lines changed: 84 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1624,7 +1624,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
16241624
if (UseMaskForCond || UseMaskForGaps) {
16251625
unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_load
16261626
: Intrinsic::masked_store;
1627-
Cost = thisT()->getMaskedMemoryOpCost(
1627+
Cost = thisT()->getMemIntrinsicInstrCost(
16281628
{IID, VecTy, Alignment, AddressSpace}, CostKind);
16291629
} else
16301630
Cost = thisT()->getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace,
@@ -1825,9 +1825,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
18251825
if (auto *VPI = dyn_cast_or_null<VPIntrinsic>(ICA.getInst()))
18261826
Alignment = VPI->getPointerAlignment().valueOrOne();
18271827
bool VarMask = isa<Constant>(ICA.getArgs()[2]);
1828-
return thisT()->getGatherScatterOpCost(
1829-
Instruction::Store, ICA.getArgTypes()[0], ICA.getArgs()[1], VarMask,
1830-
Alignment, CostKind, nullptr);
1828+
return thisT()->getMemIntrinsicInstrCost(
1829+
{Intrinsic::vp_scatter, ICA.getArgTypes()[0], ICA.getArgs()[1],
1830+
VarMask, Alignment, nullptr},
1831+
CostKind);
18311832
}
18321833
if (ICA.getID() == Intrinsic::vp_gather) {
18331834
if (ICA.isTypeBasedOnly()) {
@@ -1841,9 +1842,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
18411842
if (auto *VPI = dyn_cast_or_null<VPIntrinsic>(ICA.getInst()))
18421843
Alignment = VPI->getPointerAlignment().valueOrOne();
18431844
bool VarMask = isa<Constant>(ICA.getArgs()[1]);
1844-
return thisT()->getGatherScatterOpCost(
1845-
Instruction::Load, ICA.getReturnType(), ICA.getArgs()[0], VarMask,
1846-
Alignment, CostKind, nullptr);
1845+
return thisT()->getMemIntrinsicInstrCost(
1846+
{Intrinsic::vp_gather, ICA.getReturnType(), ICA.getArgs()[0],
1847+
VarMask, Alignment, nullptr},
1848+
CostKind);
18471849
}
18481850

18491851
if (ICA.getID() == Intrinsic::vp_select ||
@@ -1948,31 +1950,35 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
19481950
const Value *Mask = Args[2];
19491951
bool VarMask = !isa<Constant>(Mask);
19501952
Align Alignment = I->getParamAlign(1).valueOrOne();
1951-
return thisT()->getGatherScatterOpCost(Instruction::Store,
1952-
ICA.getArgTypes()[0], Args[1],
1953-
VarMask, Alignment, CostKind, I);
1953+
return thisT()->getMemIntrinsicInstrCost({Intrinsic::masked_scatter,
1954+
ICA.getArgTypes()[0], Args[1],
1955+
VarMask, Alignment, I},
1956+
CostKind);
19541957
}
19551958
case Intrinsic::masked_gather: {
19561959
const Value *Mask = Args[1];
19571960
bool VarMask = !isa<Constant>(Mask);
19581961
Align Alignment = I->getParamAlign(0).valueOrOne();
1959-
return thisT()->getGatherScatterOpCost(Instruction::Load, RetTy, Args[0],
1960-
VarMask, Alignment, CostKind, I);
1962+
return thisT()->getMemIntrinsicInstrCost(
1963+
{Intrinsic::masked_gather, RetTy, Args[0], VarMask, Alignment, I},
1964+
CostKind);
19611965
}
19621966
case Intrinsic::masked_compressstore: {
19631967
const Value *Data = Args[0];
19641968
const Value *Mask = Args[2];
19651969
Align Alignment = I->getParamAlign(1).valueOrOne();
1966-
return thisT()->getExpandCompressMemoryOpCost(
1967-
Instruction::Store, Data->getType(), !isa<Constant>(Mask), Alignment,
1968-
CostKind, I);
1970+
return thisT()->getMemIntrinsicInstrCost(
1971+
{Intrinsic::masked_compressstore, Data->getType(),
1972+
!isa<Constant>(Mask), Alignment, I},
1973+
CostKind);
19691974
}
19701975
case Intrinsic::masked_expandload: {
19711976
const Value *Mask = Args[1];
19721977
Align Alignment = I->getParamAlign(0).valueOrOne();
1973-
return thisT()->getExpandCompressMemoryOpCost(Instruction::Load, RetTy,
1974-
!isa<Constant>(Mask),
1975-
Alignment, CostKind, I);
1978+
return thisT()->getMemIntrinsicInstrCost({Intrinsic::masked_expandload,
1979+
RetTy, !isa<Constant>(Mask),
1980+
Alignment, I},
1981+
CostKind);
19761982
}
19771983
case Intrinsic::experimental_vp_strided_store: {
19781984
const Value *Data = Args[0];
@@ -1983,9 +1989,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
19831989
Type *EltTy = cast<VectorType>(Data->getType())->getElementType();
19841990
Align Alignment =
19851991
I->getParamAlign(1).value_or(thisT()->DL.getABITypeAlign(EltTy));
1986-
return thisT()->getStridedMemoryOpCost(Instruction::Store,
1987-
Data->getType(), Ptr, VarMask,
1988-
Alignment, CostKind, I);
1992+
return thisT()->getMemIntrinsicInstrCost(
1993+
{IID, Data->getType(), Ptr, VarMask, Alignment, I}, CostKind);
19891994
}
19901995
case Intrinsic::experimental_vp_strided_load: {
19911996
const Value *Ptr = Args[0];
@@ -1995,8 +2000,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
19952000
Type *EltTy = cast<VectorType>(RetTy)->getElementType();
19962001
Align Alignment =
19972002
I->getParamAlign(0).value_or(thisT()->DL.getABITypeAlign(EltTy));
1998-
return thisT()->getStridedMemoryOpCost(Instruction::Load, RetTy, Ptr,
1999-
VarMask, Alignment, CostKind, I);
2003+
return thisT()->getMemIntrinsicInstrCost(
2004+
{IID, RetTy, Ptr, VarMask, Alignment, I}, CostKind);
20002005
}
20012006
case Intrinsic::stepvector: {
20022007
if (isa<ScalableVectorType>(RetTy))
@@ -2409,26 +2414,28 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
24092414
case Intrinsic::masked_store: {
24102415
Type *Ty = Tys[0];
24112416
Align TyAlign = thisT()->DL.getABITypeAlign(Ty);
2412-
return thisT()->getMaskedMemoryOpCost({IID, Ty, TyAlign, 0}, CostKind);
2417+
return thisT()->getMemIntrinsicInstrCost({IID, Ty, TyAlign, 0}, CostKind);
24132418
}
24142419
case Intrinsic::masked_load: {
24152420
Type *Ty = RetTy;
24162421
Align TyAlign = thisT()->DL.getABITypeAlign(Ty);
2417-
return thisT()->getMaskedMemoryOpCost({IID, Ty, TyAlign, 0}, CostKind);
2422+
return thisT()->getMemIntrinsicInstrCost({IID, Ty, TyAlign, 0}, CostKind);
24182423
}
24192424
case Intrinsic::experimental_vp_strided_store: {
24202425
auto *Ty = cast<VectorType>(ICA.getArgTypes()[0]);
24212426
Align Alignment = thisT()->DL.getABITypeAlign(Ty->getElementType());
2422-
return thisT()->getStridedMemoryOpCost(
2423-
Instruction::Store, Ty, /*Ptr=*/nullptr, /*VariableMask=*/true,
2424-
Alignment, CostKind, ICA.getInst());
2427+
return thisT()->getMemIntrinsicInstrCost({IID, Ty, /*Ptr=*/nullptr,
2428+
/*VariableMask=*/true,
2429+
Alignment, ICA.getInst()},
2430+
CostKind);
24252431
}
24262432
case Intrinsic::experimental_vp_strided_load: {
24272433
auto *Ty = cast<VectorType>(ICA.getReturnType());
24282434
Align Alignment = thisT()->DL.getABITypeAlign(Ty->getElementType());
2429-
return thisT()->getStridedMemoryOpCost(
2430-
Instruction::Load, Ty, /*Ptr=*/nullptr, /*VariableMask=*/true,
2431-
Alignment, CostKind, ICA.getInst());
2435+
return thisT()->getMemIntrinsicInstrCost({IID, Ty, /*Ptr=*/nullptr,
2436+
/*VariableMask=*/true,
2437+
Alignment, ICA.getInst()},
2438+
CostKind);
24322439
}
24332440
case Intrinsic::vector_reduce_add:
24342441
case Intrinsic::vector_reduce_mul:
@@ -3016,6 +3023,52 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
30163023
return SingleCallCost;
30173024
}
30183025

3026+
/// Get memory intrinsic cost based on arguments.
3027+
InstructionCost
3028+
getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
3029+
TTI::TargetCostKind CostKind) const override {
3030+
unsigned Id = MICA.getID();
3031+
Type *DataTy = MICA.getDataType();
3032+
const Value *Ptr = MICA.getPointer();
3033+
const Instruction *I = MICA.getInst();
3034+
bool VariableMask = MICA.getVariableMask();
3035+
Align Alignment = MICA.getAlignment();
3036+
3037+
switch (Id) {
3038+
case Intrinsic::experimental_vp_strided_load:
3039+
case Intrinsic::experimental_vp_strided_store: {
3040+
unsigned Opcode = Id == Intrinsic::experimental_vp_strided_load
3041+
? Instruction::Load
3042+
: Instruction::Store;
3043+
return thisT()->getStridedMemoryOpCost(Opcode, DataTy, Ptr, VariableMask,
3044+
Alignment, CostKind, I);
3045+
}
3046+
case Intrinsic::masked_scatter:
3047+
case Intrinsic::masked_gather:
3048+
case Intrinsic::vp_scatter:
3049+
case Intrinsic::vp_gather: {
3050+
unsigned Opcode =
3051+
(Id == Intrinsic::masked_gather || Id == Intrinsic::vp_gather)
3052+
? Instruction::Load
3053+
: Instruction::Store;
3054+
return thisT()->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
3055+
Alignment, CostKind, I);
3056+
}
3057+
case Intrinsic::masked_load:
3058+
case Intrinsic::masked_store:
3059+
return thisT()->getMaskedMemoryOpCost(MICA, CostKind);
3060+
case Intrinsic::masked_compressstore:
3061+
case Intrinsic::masked_expandload: {
3062+
unsigned Opcode = Id == Intrinsic::masked_expandload ? Instruction::Load
3063+
: Instruction::Store;
3064+
return thisT()->getExpandCompressMemoryOpCost(
3065+
Opcode, DataTy, VariableMask, Alignment, CostKind, I);
3066+
}
3067+
default:
3068+
llvm_unreachable("unexpected intrinsic");
3069+
}
3070+
}
3071+
30193072
/// Compute a cost of the given call instruction.
30203073
///
30213074
/// Compute the cost of calling function F with return type RetTy and

‎llvm/lib/Analysis/TargetTransformInfo.cpp‎

Lines changed: 8 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1186,42 +1186,6 @@ InstructionCost TargetTransformInfo::getMemoryOpCost(
11861186
return Cost;
11871187
}
11881188

1189-
InstructionCost TargetTransformInfo::getMaskedMemoryOpCost(
1190-
const MemIntrinsicCostAttributes &MICA,
1191-
TTI::TargetCostKind CostKind) const {
1192-
InstructionCost Cost = TTIImpl->getMaskedMemoryOpCost(MICA, CostKind);
1193-
assert(Cost >= 0 && "TTI should not produce negative costs!");
1194-
return Cost;
1195-
}
1196-
1197-
InstructionCost TargetTransformInfo::getGatherScatterOpCost(
1198-
unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
1199-
Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
1200-
InstructionCost Cost = TTIImpl->getGatherScatterOpCost(
1201-
Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I);
1202-
assert((!Cost.isValid() || Cost >= 0) &&
1203-
"TTI should not produce negative costs!");
1204-
return Cost;
1205-
}
1206-
1207-
InstructionCost TargetTransformInfo::getExpandCompressMemoryOpCost(
1208-
unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
1209-
TTI::TargetCostKind CostKind, const Instruction *I) const {
1210-
InstructionCost Cost = TTIImpl->getExpandCompressMemoryOpCost(
1211-
Opcode, DataTy, VariableMask, Alignment, CostKind, I);
1212-
assert(Cost >= 0 && "TTI should not produce negative costs!");
1213-
return Cost;
1214-
}
1215-
1216-
InstructionCost TargetTransformInfo::getStridedMemoryOpCost(
1217-
unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
1218-
Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
1219-
InstructionCost Cost = TTIImpl->getStridedMemoryOpCost(
1220-
Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I);
1221-
assert(Cost >= 0 && "TTI should not produce negative costs!");
1222-
return Cost;
1223-
}
1224-
12251189
InstructionCost TargetTransformInfo::getInterleavedMemoryOpCost(
12261190
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
12271191
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
@@ -1241,6 +1205,14 @@ TargetTransformInfo::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
12411205
return Cost;
12421206
}
12431207

1208+
InstructionCost TargetTransformInfo::getMemIntrinsicInstrCost(
1209+
const MemIntrinsicCostAttributes &MICA,
1210+
TTI::TargetCostKind CostKind) const {
1211+
InstructionCost Cost = TTIImpl->getMemIntrinsicInstrCost(MICA, CostKind);
1212+
assert(Cost >= 0 && "TTI should not produce negative costs!");
1213+
return Cost;
1214+
}
1215+
12441216
InstructionCost
12451217
TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy,
12461218
ArrayRef<Type *> Tys,

0 commit comments

Comments
 (0)