Skip to content

[TTI][Vectorize] Migrate masked/gather-scatter/strided/expand-compress costing (NFCI) #165532

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Nov 28, 2025
Merged

[TTI][Vectorize] Migrate masked/gather-scatter/strided/expand-compress costing (NFCI) #165532

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bcfe3dd
Add getMemIntrinsicInstrCost
arcbbb Nov 24, 2025
228ead7
Use constructor explicitly
arcbbb Nov 27, 2025
a5a8db8
Merge remote-tracking branch 'pub/main' into mem-intrinsic-cost-v3
arcbbb Nov 28, 2025
960333b
Fix conflict after merge
arcbbb Nov 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 31 additions & 46 deletions llvm/include/llvm/Analysis/TargetTransformInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,26 +125,51 @@ struct HardwareLoopInfo {

/// Information for memory intrinsic cost model.
class MemIntrinsicCostAttributes {
/// Optional context instruction, if one exists, e.g. the
/// load/store to transform to the intrinsic.
const Instruction *I = nullptr;

/// Address in memory.
const Value *Ptr = nullptr;

/// Vector type of the data to be loaded or stored.
Type *DataTy = nullptr;

/// ID of the memory intrinsic.
Intrinsic::ID IID;

/// True when the memory access is predicated with a mask
/// that is not a compile-time constant.
bool VariableMask = true;

/// Address space of the pointer.
unsigned AddressSpace = 0;

/// Alignment of single element.
Align Alignment;

public:
LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
const Value *Ptr, bool VariableMask,
Align Alignment,
const Instruction *I = nullptr)
: I(I), Ptr(Ptr), DataTy(DataTy), IID(Id), VariableMask(VariableMask),
Alignment(Alignment) {}
LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
Align Alignment, unsigned AddressSpace)
: DataTy(DataTy), IID(Id), AddressSpace(AddressSpace),
Alignment(Alignment) {}
LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
bool VariableMask, Align Alignment,
const Instruction *I = nullptr)
: I(I), DataTy(DataTy), IID(Id), VariableMask(VariableMask),
Alignment(Alignment) {}

Intrinsic::ID getID() const { return IID; }
const Instruction *getInst() const { return I; }
const Value *getPointer() const { return Ptr; }
Type *getDataType() const { return DataTy; }
bool getVariableMask() const { return VariableMask; }
unsigned getAddressSpace() const { return AddressSpace; }
Align getAlignment() const { return Alignment; }
};
Expand Down Expand Up @@ -1584,52 +1609,6 @@ class TargetTransformInfo {
OperandValueInfo OpdInfo = {OK_AnyValue, OP_None},
const Instruction *I = nullptr) const;

/// \return The cost of masked Load and Store instructions.
LLVM_ABI InstructionCost getMaskedMemoryOpCost(
const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const;

/// \return The cost of Gather or Scatter operation
/// \p Opcode - is a type of memory access Load or Store
/// \p DataTy - a vector type of the data to be loaded or stored
/// \p Ptr - pointer [or vector of pointers] - address[es] in memory
/// \p VariableMask - true when the memory access is predicated with a mask
/// that is not a compile-time constant
/// \p Alignment - alignment of single element
/// \p I - the optional original context instruction, if one exists, e.g. the
/// load/store to transform or the call to the gather/scatter intrinsic
LLVM_ABI InstructionCost getGatherScatterOpCost(
unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
Align Alignment, TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
const Instruction *I = nullptr) const;

/// \return The cost of Expand Load or Compress Store operation
/// \p Opcode - is a type of memory access Load or Store
/// \p Src - a vector type of the data to be loaded or stored
/// \p VariableMask - true when the memory access is predicated with a mask
/// that is not a compile-time constant
/// \p Alignment - alignment of single element
/// \p I - the optional original context instruction, if one exists, e.g. the
/// load/store to transform or the call to the gather/scatter intrinsic
LLVM_ABI InstructionCost getExpandCompressMemoryOpCost(
unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
const Instruction *I = nullptr) const;

/// \return The cost of strided memory operations.
/// \p Opcode - is a type of memory access Load or Store
/// \p DataTy - a vector type of the data to be loaded or stored
/// \p Ptr - pointer [or vector of pointers] - address[es] in memory
/// \p VariableMask - true when the memory access is predicated with a mask
/// that is not a compile-time constant
/// \p Alignment - alignment of single element
/// \p I - the optional original context instruction, if one exists, e.g. the
/// load/store to transform or the call to the gather/scatter intrinsic
LLVM_ABI InstructionCost getStridedMemoryOpCost(
unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
Align Alignment, TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
const Instruction *I = nullptr) const;

/// \return The cost of the interleaved memory operation.
/// \p Opcode is the memory operation code
/// \p VecTy is the vector type of the interleaved access.
Expand Down Expand Up @@ -1708,6 +1687,12 @@ class TargetTransformInfo {
LLVM_ABI InstructionCost getIntrinsicInstrCost(
const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind) const;

/// \returns The cost of memory intrinsic instructions.
/// Used when IntrinsicInst is not materialized.
LLVM_ABI InstructionCost
getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const;

/// \returns The cost of Call instructions.
LLVM_ABI InstructionCost getCallInstrCost(
Function *F, Type *RetTy, ArrayRef<Type *> Tys,
Expand Down
5 changes: 5 additions & 0 deletions llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -927,6 +927,11 @@ class TargetTransformInfoImplBase {
return 1;
}

virtual InstructionCost
getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
return 1;
}
virtual InstructionCost getCallInstrCost(Function *F, Type *RetTy,
ArrayRef<Type *> Tys,
TTI::TargetCostKind CostKind) const {
Expand Down
130 changes: 98 additions & 32 deletions llvm/include/llvm/CodeGen/BasicTTIImpl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1624,8 +1624,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
if (UseMaskForCond || UseMaskForGaps) {
unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_load
: Intrinsic::masked_store;
Cost = thisT()->getMaskedMemoryOpCost(
{IID, VecTy, Alignment, AddressSpace}, CostKind);
Cost = thisT()->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(IID, VecTy, Alignment, AddressSpace),
CostKind);
} else
Cost = thisT()->getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace,
CostKind);
Expand Down Expand Up @@ -1825,9 +1826,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
if (auto *VPI = dyn_cast_or_null<VPIntrinsic>(ICA.getInst()))
Alignment = VPI->getPointerAlignment().valueOrOne();
bool VarMask = isa<Constant>(ICA.getArgs()[2]);
return thisT()->getGatherScatterOpCost(
Instruction::Store, ICA.getArgTypes()[0], ICA.getArgs()[1], VarMask,
Alignment, CostKind, nullptr);
return thisT()->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::vp_scatter,
ICA.getArgTypes()[0], ICA.getArgs()[1],
VarMask, Alignment, nullptr),
CostKind);
}
if (ICA.getID() == Intrinsic::vp_gather) {
if (ICA.isTypeBasedOnly()) {
Expand All @@ -1841,9 +1844,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
if (auto *VPI = dyn_cast_or_null<VPIntrinsic>(ICA.getInst()))
Alignment = VPI->getPointerAlignment().valueOrOne();
bool VarMask = isa<Constant>(ICA.getArgs()[1]);
return thisT()->getGatherScatterOpCost(
Instruction::Load, ICA.getReturnType(), ICA.getArgs()[0], VarMask,
Alignment, CostKind, nullptr);
return thisT()->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::vp_gather,
ICA.getReturnType(), ICA.getArgs()[0],
VarMask, Alignment, nullptr),
CostKind);
}

if (ICA.getID() == Intrinsic::vp_select ||
Expand Down Expand Up @@ -1948,31 +1953,38 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
const Value *Mask = Args[2];
bool VarMask = !isa<Constant>(Mask);
Align Alignment = I->getParamAlign(1).valueOrOne();
return thisT()->getGatherScatterOpCost(Instruction::Store,
ICA.getArgTypes()[0], Args[1],
VarMask, Alignment, CostKind, I);
return thisT()->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::masked_scatter,
ICA.getArgTypes()[0], Args[1], VarMask,
Alignment, I),
CostKind);
}
case Intrinsic::masked_gather: {
const Value *Mask = Args[1];
bool VarMask = !isa<Constant>(Mask);
Align Alignment = I->getParamAlign(0).valueOrOne();
return thisT()->getGatherScatterOpCost(Instruction::Load, RetTy, Args[0],
VarMask, Alignment, CostKind, I);
return thisT()->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::masked_gather, RetTy, Args[0],
VarMask, Alignment, I),
CostKind);
}
case Intrinsic::masked_compressstore: {
const Value *Data = Args[0];
const Value *Mask = Args[2];
Align Alignment = I->getParamAlign(1).valueOrOne();
return thisT()->getExpandCompressMemoryOpCost(
Instruction::Store, Data->getType(), !isa<Constant>(Mask), Alignment,
CostKind, I);
return thisT()->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::masked_compressstore,
Data->getType(), !isa<Constant>(Mask),
Alignment, I),
CostKind);
}
case Intrinsic::masked_expandload: {
const Value *Mask = Args[1];
Align Alignment = I->getParamAlign(0).valueOrOne();
return thisT()->getExpandCompressMemoryOpCost(Instruction::Load, RetTy,
!isa<Constant>(Mask),
Alignment, CostKind, I);
return thisT()->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::masked_expandload, RetTy,
!isa<Constant>(Mask), Alignment, I),
CostKind);
}
case Intrinsic::experimental_vp_strided_store: {
const Value *Data = Args[0];
Expand All @@ -1983,9 +1995,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
Type *EltTy = cast<VectorType>(Data->getType())->getElementType();
Align Alignment =
I->getParamAlign(1).value_or(thisT()->DL.getABITypeAlign(EltTy));
return thisT()->getStridedMemoryOpCost(Instruction::Store,
Data->getType(), Ptr, VarMask,
Alignment, CostKind, I);
return thisT()->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(IID, Data->getType(), Ptr, VarMask,
Alignment, I),
CostKind);
}
case Intrinsic::experimental_vp_strided_load: {
const Value *Ptr = Args[0];
Expand All @@ -1995,8 +2008,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
Type *EltTy = cast<VectorType>(RetTy)->getElementType();
Align Alignment =
I->getParamAlign(0).value_or(thisT()->DL.getABITypeAlign(EltTy));
return thisT()->getStridedMemoryOpCost(Instruction::Load, RetTy, Ptr,
VarMask, Alignment, CostKind, I);
return thisT()->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(IID, RetTy, Ptr, VarMask, Alignment, I),
CostKind);
}
case Intrinsic::stepvector: {
if (isa<ScalableVectorType>(RetTy))
Expand Down Expand Up @@ -2409,26 +2423,32 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
case Intrinsic::masked_store: {
Type *Ty = Tys[0];
Align TyAlign = thisT()->DL.getABITypeAlign(Ty);
return thisT()->getMaskedMemoryOpCost({IID, Ty, TyAlign, 0}, CostKind);
return thisT()->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(IID, Ty, TyAlign, 0), CostKind);
}
case Intrinsic::masked_load: {
Type *Ty = RetTy;
Align TyAlign = thisT()->DL.getABITypeAlign(Ty);
return thisT()->getMaskedMemoryOpCost({IID, Ty, TyAlign, 0}, CostKind);
return thisT()->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(IID, Ty, TyAlign, 0), CostKind);
}
case Intrinsic::experimental_vp_strided_store: {
auto *Ty = cast<VectorType>(ICA.getArgTypes()[0]);
Align Alignment = thisT()->DL.getABITypeAlign(Ty->getElementType());
return thisT()->getStridedMemoryOpCost(
Instruction::Store, Ty, /*Ptr=*/nullptr, /*VariableMask=*/true,
Alignment, CostKind, ICA.getInst());
return thisT()->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(IID, Ty, /*Ptr=*/nullptr,
/*VariableMask=*/true, Alignment,
ICA.getInst()),
CostKind);
}
case Intrinsic::experimental_vp_strided_load: {
auto *Ty = cast<VectorType>(ICA.getReturnType());
Align Alignment = thisT()->DL.getABITypeAlign(Ty->getElementType());
return thisT()->getStridedMemoryOpCost(
Instruction::Load, Ty, /*Ptr=*/nullptr, /*VariableMask=*/true,
Alignment, CostKind, ICA.getInst());
return thisT()->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(IID, Ty, /*Ptr=*/nullptr,
/*VariableMask=*/true, Alignment,
ICA.getInst()),
CostKind);
}
case Intrinsic::vector_reduce_add:
case Intrinsic::vector_reduce_mul:
Expand Down Expand Up @@ -3016,6 +3036,52 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return SingleCallCost;
}

/// Get memory intrinsic cost based on arguments.
InstructionCost
getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const override {
unsigned Id = MICA.getID();
Type *DataTy = MICA.getDataType();
const Value *Ptr = MICA.getPointer();
const Instruction *I = MICA.getInst();
bool VariableMask = MICA.getVariableMask();
Align Alignment = MICA.getAlignment();

switch (Id) {
case Intrinsic::experimental_vp_strided_load:
case Intrinsic::experimental_vp_strided_store: {
unsigned Opcode = Id == Intrinsic::experimental_vp_strided_load
? Instruction::Load
: Instruction::Store;
return thisT()->getStridedMemoryOpCost(Opcode, DataTy, Ptr, VariableMask,
Copy link
Contributor

@fhahn fhahn Nov 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should those also be migrated, possibly not in the initial PR but as follow-ups?

Copy link
Contributor Author

@arcbbb arcbbb Nov 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, getStridedMemoryOpCost was straightforward (RISCV-only). The others are implemented across several backends, so I’ll stage those as follow-ups with the relevant target reviewers

Copy link
Contributor

@fhahn fhahn Nov 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds great, thanks!

Alignment, CostKind, I);
}
case Intrinsic::masked_scatter:
case Intrinsic::masked_gather:
case Intrinsic::vp_scatter:
case Intrinsic::vp_gather: {
unsigned Opcode =
(Id == Intrinsic::masked_gather || Id == Intrinsic::vp_gather)
? Instruction::Load
: Instruction::Store;
return thisT()->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
Alignment, CostKind, I);
}
case Intrinsic::masked_load:
case Intrinsic::masked_store:
return thisT()->getMaskedMemoryOpCost(MICA, CostKind);
case Intrinsic::masked_compressstore:
case Intrinsic::masked_expandload: {
unsigned Opcode = Id == Intrinsic::masked_expandload ? Instruction::Load
: Instruction::Store;
return thisT()->getExpandCompressMemoryOpCost(
Opcode, DataTy, VariableMask, Alignment, CostKind, I);
}
default:
llvm_unreachable("unexpected intrinsic");
}
}

/// Compute a cost of the given call instruction.
///
/// Compute the cost of calling function F with return type RetTy and
Expand Down
44 changes: 8 additions & 36 deletions llvm/lib/Analysis/TargetTransformInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1186,42 +1186,6 @@ InstructionCost TargetTransformInfo::getMemoryOpCost(
return Cost;
}

InstructionCost TargetTransformInfo::getMaskedMemoryOpCost(
const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
InstructionCost Cost = TTIImpl->getMaskedMemoryOpCost(MICA, CostKind);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}

InstructionCost TargetTransformInfo::getGatherScatterOpCost(
unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
InstructionCost Cost = TTIImpl->getGatherScatterOpCost(
Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I);
assert((!Cost.isValid() || Cost >= 0) &&
"TTI should not produce negative costs!");
return Cost;
}

InstructionCost TargetTransformInfo::getExpandCompressMemoryOpCost(
unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
TTI::TargetCostKind CostKind, const Instruction *I) const {
InstructionCost Cost = TTIImpl->getExpandCompressMemoryOpCost(
Opcode, DataTy, VariableMask, Alignment, CostKind, I);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}

InstructionCost TargetTransformInfo::getStridedMemoryOpCost(
unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
InstructionCost Cost = TTIImpl->getStridedMemoryOpCost(
Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}

InstructionCost TargetTransformInfo::getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
Expand All @@ -1241,6 +1205,14 @@ TargetTransformInfo::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
return Cost;
}

InstructionCost TargetTransformInfo::getMemIntrinsicInstrCost(
const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
InstructionCost Cost = TTIImpl->getMemIntrinsicInstrCost(MICA, CostKind);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}

InstructionCost
TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy,
ArrayRef<Type *> Tys,
Expand Down
Loading