Skip to content

Commit 961940e

Browse files
authored
[TTI] Use MemIntrinsicCostAttributes for getMaskedMemoryOpCost (#168029)
- Split from #165532. This is a step toward a unified interface for masked/gather-scatter/strided/expand-compress cost modeling. - Replace the ad-hoc parameter list with a single attributes object. API change: ``` - InstructionCost getMaskedMemoryOpCost(Opcode, Src, Alignment, - AddressSpace, CostKind); + InstructionCost getMaskedMemoryOpCost(MemIntrinsicCostAttributes, + CostKind); ``` Notes: - NFCI intended: callers populate MemIntrinsicCostAttributes with the same information as before. - Follow-up: migrate gather/scatter, strided, and expand/compress cost queries to the same attributes-based entry point.
1 parent bfb9539 commit 961940e

17 files changed

+107
-62
lines changed

llvm/include/llvm/Analysis/TargetTransformInfo.h

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,32 @@ struct HardwareLoopInfo {
123123
LLVM_ABI bool canAnalyze(LoopInfo &LI);
124124
};
125125

126+
/// Information for memory intrinsic cost model.
127+
class MemIntrinsicCostAttributes {
128+
/// Vector type of the data to be loaded or stored.
129+
Type *DataTy = nullptr;
130+
131+
/// ID of the memory intrinsic.
132+
Intrinsic::ID IID;
133+
134+
/// Address space of the pointer.
135+
unsigned AddressSpace = 0;
136+
137+
/// Alignment of single element.
138+
Align Alignment;
139+
140+
public:
141+
LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
142+
Align Alignment, unsigned AddressSpace)
143+
: DataTy(DataTy), IID(Id), AddressSpace(AddressSpace),
144+
Alignment(Alignment) {}
145+
146+
Intrinsic::ID getID() const { return IID; }
147+
Type *getDataType() const { return DataTy; }
148+
unsigned getAddressSpace() const { return AddressSpace; }
149+
Align getAlignment() const { return Alignment; }
150+
};
151+
126152
class IntrinsicCostAttributes {
127153
const IntrinsicInst *II = nullptr;
128154
Type *RetTy = nullptr;
@@ -1556,7 +1582,7 @@ class TargetTransformInfo {
15561582

15571583
/// \return The cost of masked Load and Store instructions.
15581584
LLVM_ABI InstructionCost getMaskedMemoryOpCost(
1559-
unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
1585+
const MemIntrinsicCostAttributes &MICA,
15601586
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const;
15611587

15621588
/// \return The cost of Gather or Scatter operation

llvm/include/llvm/Analysis/TargetTransformInfoImpl.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -842,8 +842,7 @@ class TargetTransformInfoImplBase {
842842
}
843843

844844
virtual InstructionCost
845-
getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
846-
unsigned AddressSpace,
845+
getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
847846
TTI::TargetCostKind CostKind) const {
848847
return 1;
849848
}

llvm/include/llvm/CodeGen/BasicTTIImpl.h

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1558,9 +1558,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
15581558
}
15591559

15601560
InstructionCost
1561-
getMaskedMemoryOpCost(unsigned Opcode, Type *DataTy, Align Alignment,
1562-
unsigned AddressSpace,
1561+
getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
15631562
TTI::TargetCostKind CostKind) const override {
1563+
Type *DataTy = MICA.getDataType();
1564+
Align Alignment = MICA.getAlignment();
1565+
unsigned Opcode = MICA.getID() == Intrinsic::masked_load
1566+
? Instruction::Load
1567+
: Instruction::Store;
15641568
// TODO: Pass on AddressSpace when we have test coverage.
15651569
return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, true, false,
15661570
CostKind);
@@ -1617,10 +1621,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
16171621

16181622
// Firstly, the cost of load/store operation.
16191623
InstructionCost Cost;
1620-
if (UseMaskForCond || UseMaskForGaps)
1621-
Cost = thisT()->getMaskedMemoryOpCost(Opcode, VecTy, Alignment,
1622-
AddressSpace, CostKind);
1623-
else
1624+
if (UseMaskForCond || UseMaskForGaps) {
1625+
unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_load
1626+
: Intrinsic::masked_store;
1627+
Cost = thisT()->getMaskedMemoryOpCost(
1628+
{IID, VecTy, Alignment, AddressSpace}, CostKind);
1629+
} else
16241630
Cost = thisT()->getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace,
16251631
CostKind);
16261632

@@ -2403,14 +2409,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
24032409
case Intrinsic::masked_store: {
24042410
Type *Ty = Tys[0];
24052411
Align TyAlign = thisT()->DL.getABITypeAlign(Ty);
2406-
return thisT()->getMaskedMemoryOpCost(Instruction::Store, Ty, TyAlign, 0,
2407-
CostKind);
2412+
return thisT()->getMaskedMemoryOpCost({IID, Ty, TyAlign, 0}, CostKind);
24082413
}
24092414
case Intrinsic::masked_load: {
24102415
Type *Ty = RetTy;
24112416
Align TyAlign = thisT()->DL.getABITypeAlign(Ty);
2412-
return thisT()->getMaskedMemoryOpCost(Instruction::Load, Ty, TyAlign, 0,
2413-
CostKind);
2417+
return thisT()->getMaskedMemoryOpCost({IID, Ty, TyAlign, 0}, CostKind);
24142418
}
24152419
case Intrinsic::experimental_vp_strided_store: {
24162420
auto *Ty = cast<VectorType>(ICA.getArgTypes()[0]);

llvm/lib/Analysis/TargetTransformInfo.cpp

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1183,10 +1183,9 @@ InstructionCost TargetTransformInfo::getMemoryOpCost(
11831183
}
11841184

11851185
InstructionCost TargetTransformInfo::getMaskedMemoryOpCost(
1186-
unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
1186+
const MemIntrinsicCostAttributes &MICA,
11871187
TTI::TargetCostKind CostKind) const {
1188-
InstructionCost Cost = TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment,
1189-
AddressSpace, CostKind);
1188+
InstructionCost Cost = TTIImpl->getMaskedMemoryOpCost(MICA, CostKind);
11901189
assert(Cost >= 0 && "TTI should not produce negative costs!");
11911190
return Cost;
11921191
}

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4730,12 +4730,12 @@ bool AArch64TTIImpl::prefersVectorizedAddressing() const {
47304730
}
47314731

47324732
InstructionCost
4733-
AArch64TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
4734-
Align Alignment, unsigned AddressSpace,
4733+
AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
47354734
TTI::TargetCostKind CostKind) const {
4735+
Type *Src = MICA.getDataType();
4736+
47364737
if (useNeonVector(Src))
4737-
return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
4738-
CostKind);
4738+
return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
47394739
auto LT = getTypeLegalizationCost(Src);
47404740
if (!LT.first.isValid())
47414741
return InstructionCost::getInvalid();

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -188,8 +188,7 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
188188
unsigned Opcode2) const;
189189

190190
InstructionCost
191-
getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
192-
unsigned AddressSpace,
191+
getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
193192
TTI::TargetCostKind CostKind) const override;
194193

195194
InstructionCost

llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1631,20 +1631,22 @@ InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
16311631
}
16321632

16331633
InstructionCost
1634-
ARMTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
1635-
unsigned AddressSpace,
1634+
ARMTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
16361635
TTI::TargetCostKind CostKind) const {
1636+
unsigned IID = MICA.getID();
1637+
Type *Src = MICA.getDataType();
1638+
Align Alignment = MICA.getAlignment();
1639+
unsigned AddressSpace = MICA.getAddressSpace();
16371640
if (ST->hasMVEIntegerOps()) {
1638-
if (Opcode == Instruction::Load &&
1641+
if (IID == Intrinsic::masked_load &&
16391642
isLegalMaskedLoad(Src, Alignment, AddressSpace))
16401643
return ST->getMVEVectorCostFactor(CostKind);
1641-
if (Opcode == Instruction::Store &&
1644+
if (IID == Intrinsic::masked_store &&
16421645
isLegalMaskedStore(Src, Alignment, AddressSpace))
16431646
return ST->getMVEVectorCostFactor(CostKind);
16441647
}
16451648
if (!isa<FixedVectorType>(Src))
1646-
return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
1647-
CostKind);
1649+
return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
16481650
// Scalar cost, which is currently very high due to the efficiency of the
16491651
// generated code.
16501652
return cast<FixedVectorType>(Src)->getNumElements() * 8;

llvm/lib/Target/ARM/ARMTargetTransformInfo.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -275,8 +275,7 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
275275
const Instruction *I = nullptr) const override;
276276

277277
InstructionCost
278-
getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
279-
unsigned AddressSpace,
278+
getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
280279
TTI::TargetCostKind CostKind) const override;
281280

282281
InstructionCost getInterleavedMemoryOpCost(

llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -224,11 +224,9 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
224224
}
225225

226226
InstructionCost
227-
HexagonTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
228-
Align Alignment, unsigned AddressSpace,
227+
HexagonTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
229228
TTI::TargetCostKind CostKind) const {
230-
return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
231-
CostKind);
229+
return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
232230
}
233231

234232
InstructionCost

llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,8 +120,7 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
120120
TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
121121
const Instruction *I = nullptr) const override;
122122
InstructionCost
123-
getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
124-
unsigned AddressSpace,
123+
getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
125124
TTI::TargetCostKind CostKind) const override;
126125
InstructionCost
127126
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,

0 commit comments

Comments
 (0)