diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index 53c91bfe16804..9186419715cc4 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -1681,8 +1681,8 @@ class TargetTransformInfo { /// was used in order to get the Ptr step value. \p Ptr holds the SCEV of the /// access pointer. LLVM_ABI InstructionCost - getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE = nullptr, - const SCEV *Ptr = nullptr) const; + getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr, + TTI::TargetCostKind CostKind) const; /// \returns The cost, if any, of keeping values of the given types alive /// over a callsite. diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index e879712121b59..200cbafbaa6e2 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -939,7 +939,8 @@ class TargetTransformInfoImplBase { virtual InstructionCost getAddressComputationCost(Type *PtrTy, ScalarEvolution *, - const SCEV *) const { + const SCEV *, + TTI::TargetCostKind) const { return 0; } diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index 1216433fec594..aa9d1f0a1ccea 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -3026,8 +3026,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { return LT.first.getValue(); } - InstructionCost getAddressComputationCost(Type *PtrTy, ScalarEvolution *, - const SCEV *) const override { + InstructionCost + getAddressComputationCost(Type *PtrTy, ScalarEvolution *, const SCEV *, + TTI::TargetCostKind) const override { return 0; } diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp index 4f04209cf4cfc..3141060a710ce 100644 --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -1230,10 +1230,11 @@ unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const { return TTIImpl->getNumberOfParts(Tp); } -InstructionCost -TargetTransformInfo::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, - const SCEV *Ptr) const { - InstructionCost Cost = TTIImpl->getAddressComputationCost(PtrTy, SE, Ptr); +InstructionCost TargetTransformInfo::getAddressComputationCost( + Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr, + TTI::TargetCostKind CostKind) const { + InstructionCost Cost = + TTIImpl->getAddressComputationCost(PtrTy, SE, Ptr, CostKind); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index a756ca212f6af..3fba7e853eafb 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -4337,7 +4337,8 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost( InstructionCost AArch64TTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, - const SCEV *Ptr) const { + const SCEV *Ptr, + TTI::TargetCostKind CostKind) const { // Address computations in vectorized code with non-consecutive addresses will // likely result in more instructions compared to scalar code where the // computation can more often be merged into the index mode. The resulting diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h index 647b242d74fb3..9c96fdd427814 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -238,8 +238,9 @@ class AArch64TTIImpl final : public BasicTTIImplBase { ArrayRef Args = {}, const Instruction *CxtI = nullptr) const override; - InstructionCost getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, - const SCEV *Ptr) const override; + InstructionCost + getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr, + TTI::TargetCostKind CostKind) const override; InstructionCost getCmpSelInstrCost( unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred, diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp index 4dc2eb88c18bb..6b2854171c819 100644 --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp @@ -1084,9 +1084,10 @@ InstructionCost ARMTTIImpl::getCmpSelInstrCost( CostKind, Op1Info, Op2Info, I); } -InstructionCost ARMTTIImpl::getAddressComputationCost(Type *PtrTy, - ScalarEvolution *SE, - const SCEV *Ptr) const { +InstructionCost +ARMTTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, + const SCEV *Ptr, + TTI::TargetCostKind CostKind) const { // Address computations in vectorized code with non-consecutive addresses will // likely result in more instructions compared to scalar code where the // computation can more often be merged into the index mode. The resulting @@ -1103,7 +1104,7 @@ InstructionCost ARMTTIImpl::getAddressComputationCost(Type *PtrTy, // addressing mode. return 1; } - return BaseT::getAddressComputationCost(PtrTy, SE, Ptr); + return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind); } bool ARMTTIImpl::isProfitableLSRChainElement(Instruction *I) const { diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h index 522c235a90a8f..cdd8bcb9f7416 100644 --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h @@ -257,8 +257,9 @@ class ARMTTIImpl final : public BasicTTIImplBase { unsigned Index, const Value *Op0, const Value *Op1) const override; - InstructionCost getAddressComputationCost(Type *Val, ScalarEvolution *SE, - const SCEV *Ptr) const override; + InstructionCost + getAddressComputationCost(Type *Val, ScalarEvolution *SE, const SCEV *Ptr, + TTI::TargetCostKind CostKind) const override; InstructionCost getArithmeticInstrCost( unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp index 5c212816fbdb3..171e2949366ad 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp +++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp @@ -156,9 +156,10 @@ HexagonTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, return BaseT::getIntrinsicInstrCost(ICA, CostKind); } -InstructionCost HexagonTTIImpl::getAddressComputationCost(Type *PtrTy, - ScalarEvolution *SE, - const SCEV *S) const { +InstructionCost +HexagonTTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, + const SCEV *S, + TTI::TargetCostKind CostKind) const { return 0; } diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h index 0a5766d1dadf5..dbf16c99c314c 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h +++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h @@ -111,8 +111,9 @@ class HexagonTTIImpl final : public BasicTTIImplBase { InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind) const override; - InstructionCost getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, - const SCEV *S) const override; + InstructionCost + getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *S, + TTI::TargetCostKind CostKind) const override; InstructionCost getMemoryOpCost( unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index 9ef21faea2b60..cae6bb99d9637 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -5488,9 +5488,10 @@ InstructionCost X86TTIImpl::getPointersChainCost( return BaseT::getPointersChainCost(Ptrs, Base, Info, AccessTy, CostKind); } -InstructionCost X86TTIImpl::getAddressComputationCost(Type *PtrTy, - ScalarEvolution *SE, - const SCEV *Ptr) const { +InstructionCost +X86TTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, + const SCEV *Ptr, + TTI::TargetCostKind CostKind) const { // Address computations in vectorized code with non-consecutive addresses will // likely result in more instructions compared to scalar code where the // computation can more often be merged into the index mode. The resulting @@ -5513,7 +5514,7 @@ InstructionCost X86TTIImpl::getAddressComputationCost(Type *PtrTy, return 1; } - return BaseT::getAddressComputationCost(PtrTy, SE, Ptr); + return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind); } InstructionCost diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h index bc06c4746c3c4..5718c0c9535f3 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.h +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h @@ -194,8 +194,9 @@ class X86TTIImpl final : public BasicTTIImplBase { getPointersChainCost(ArrayRef Ptrs, const Value *Base, const TTI::PointersChainInfo &Info, Type *AccessTy, TTI::TargetCostKind CostKind) const override; - InstructionCost getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, - const SCEV *Ptr) const override; + InstructionCost + getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr, + TTI::TargetCostKind CostKind) const override; std::optional instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const override; diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp index 844219a64d2f4..8b15445ae92d3 100644 --- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp +++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp @@ -2309,7 +2309,9 @@ chainToBasePointerCost(SmallVectorImpl &Chain, } else if (GetElementPtrInst *GEP = dyn_cast(Instr)) { // Cost of the address calculation - Cost += TTI.getAddressComputationCost(GEP->getType()); + Cost += TTI.getAddressComputationCost( + GEP->getType(), nullptr, nullptr, + TargetTransformInfo::TCK_SizeAndLatency); // And cost of the GEP itself // TODO: Use TTI->getGEPCost here (it exists, but appears to be not diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 2cee36003a39e..cb37ec3e94809 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -5214,8 +5214,8 @@ LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I, const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, Legal, PSE, TheLoop); // Get the cost of the scalar memory instruction and address computation. - InstructionCost Cost = - VF.getFixedValue() * TTI.getAddressComputationCost(PtrTy, SE, PtrSCEV); + InstructionCost Cost = VF.getFixedValue() * TTI.getAddressComputationCost( + PtrTy, SE, PtrSCEV, CostKind); // Don't pass *I here, since it is scalar but will actually be part of a // vectorized loop where the user of it is a vectorized instruction. @@ -5291,7 +5291,7 @@ LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I, const Align Alignment = getLoadStoreAlignment(I); unsigned AS = getLoadStoreAddressSpace(I); if (isa(I)) { - return TTI.getAddressComputationCost(PtrTy) + + return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) + TTI.getMemoryOpCost(Instruction::Load, ValTy, Alignment, AS, CostKind) + TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VectorTy, @@ -5304,7 +5304,7 @@ LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I, // VF.getKnownMinValue() - 1 from a scalable vector. This does not represent // the actual generated code, which involves extracting the last element of // a scalable vector where the lane to extract is unknown at compile time. - return TTI.getAddressComputationCost(PtrTy) + + return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) + TTI.getMemoryOpCost(Instruction::Store, ValTy, Alignment, AS, CostKind) + (IsLoopInvariantStoreValue @@ -5322,7 +5322,7 @@ LoopVectorizationCostModel::getGatherScatterCost(Instruction *I, const Value *Ptr = getLoadStorePointerOperand(I); Type *PtrTy = toVectorTy(Ptr->getType(), VF); - return TTI.getAddressComputationCost(PtrTy) + + return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) + TTI.getGatherScatterOpCost(I->getOpcode(), VectorTy, Ptr, Legal->isMaskRequired(I), Alignment, CostKind, I); @@ -5562,7 +5562,7 @@ LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I, unsigned AS = getLoadStoreAddressSpace(I); TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0)); - return TTI.getAddressComputationCost(PtrTy) + + return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) + TTI.getMemoryOpCost(I->getOpcode(), ValTy, Alignment, AS, CostKind, OpInfo, I); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 23c10d2b25263..7bbd0dc325a9b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -3130,7 +3130,8 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF, Type *PtrTy = toVectorTy(Ptr->getType(), VF); assert(!Reverse && "Inconsecutive memory access should not have the order."); - return Ctx.TTI.getAddressComputationCost(PtrTy) + + return Ctx.TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, + Ctx.CostKind) + Ctx.TTI.getGatherScatterOpCost(Opcode, Ty, Ptr, IsMasked, Alignment, Ctx.CostKind, &Ingredient); } diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp index c45005db6016b..4a681cbdab8ca 100644 --- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp +++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp @@ -1796,8 +1796,8 @@ bool VectorCombine::scalarizeLoadExtract(Instruction &I) { ScalarizedCost += TTI.getMemoryOpCost(Instruction::Load, VecTy->getElementType(), Align(1), LI->getPointerAddressSpace(), CostKind); - ScalarizedCost += - TTI.getAddressComputationCost(LI->getPointerOperandType()); + ScalarizedCost += TTI.getAddressComputationCost(LI->getPointerOperandType(), + nullptr, nullptr, CostKind); } LLVM_DEBUG(dbgs() << "Found all extractions of a vector load: " << I