Skip to content

Commit 5f9b9ec

Browse files
JonPsson1JonPsson1
committed
Pass VL to getScalarizationOverhead
1 parent a51f452 commit 5f9b9ec

File tree

14 files changed

+120
-59
lines changed

14 files changed

+120
-59
lines changed

‎llvm/include/llvm/Analysis/TargetTransformInfo.h‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -909,11 +909,11 @@ class TargetTransformInfo {
909909

910910
/// Estimate the overhead of scalarizing an instruction. Insert and Extract
911911
/// are set if the demanded result elements need to be inserted and/or
912-
/// extracted from vectors.
913-
InstructionCost getScalarizationOverhead(VectorType *Ty,
914-
const APInt &DemandedElts,
915-
bool Insert, bool Extract,
916-
TTI::TargetCostKind CostKind) const;
912+
/// extracted from vectors. The involved values may be passed in VL if
913+
/// Insert is true.
914+
InstructionCost getScalarizationOverhead(
915+
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
916+
TTI::TargetCostKind CostKind, ArrayRef<Value *> VL = std::nullopt) const;
917917

918918
/// Estimate the overhead of scalarizing an instructions unique
919919
/// non-constant operands. The (potentially vector) types to use for each of
@@ -2001,10 +2001,10 @@ class TargetTransformInfo::Concept {
20012001
unsigned ScalarOpdIdx) = 0;
20022002
virtual bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID,
20032003
int ScalarOpdIdx) = 0;
2004-
virtual InstructionCost getScalarizationOverhead(VectorType *Ty,
2005-
const APInt &DemandedElts,
2006-
bool Insert, bool Extract,
2007-
TargetCostKind CostKind) = 0;
2004+
virtual InstructionCost
2005+
getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
2006+
bool Insert, bool Extract, TargetCostKind CostKind,
2007+
ArrayRef<Value *> VL = std::nullopt) = 0;
20082008
virtual InstructionCost
20092009
getOperandsScalarizationOverhead(ArrayRef<const Value *> Args,
20102010
ArrayRef<Type *> Tys,
@@ -2582,12 +2582,12 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
25822582
return Impl.isVectorIntrinsicWithOverloadTypeAtArg(ID, ScalarOpdIdx);
25832583
}
25842584

2585-
InstructionCost getScalarizationOverhead(VectorType *Ty,
2586-
const APInt &DemandedElts,
2587-
bool Insert, bool Extract,
2588-
TargetCostKind CostKind) override {
2585+
InstructionCost
2586+
getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
2587+
bool Insert, bool Extract, TargetCostKind CostKind,
2588+
ArrayRef<Value *> VL = std::nullopt) override {
25892589
return Impl.getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
2590-
CostKind);
2590+
CostKind, VL);
25912591
}
25922592
InstructionCost
25932593
getOperandsScalarizationOverhead(ArrayRef<const Value *> Args,

‎llvm/include/llvm/Analysis/TargetTransformInfoImpl.h‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -401,10 +401,9 @@ class TargetTransformInfoImplBase {
401401
return ScalarOpdIdx == -1;
402402
}
403403

404-
InstructionCost getScalarizationOverhead(VectorType *Ty,
405-
const APInt &DemandedElts,
406-
bool Insert, bool Extract,
407-
TTI::TargetCostKind CostKind) const {
404+
InstructionCost getScalarizationOverhead(
405+
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
406+
TTI::TargetCostKind CostKind, ArrayRef<Value *> VL = std::nullopt) const {
408407
return 0;
409408
}
410409

‎llvm/include/llvm/CodeGen/BasicTTIImpl.h‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -777,10 +777,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
777777
/// Estimate the overhead of scalarizing an instruction. Insert and Extract
778778
/// are set if the demanded result elements need to be inserted and/or
779779
/// extracted from vectors.
780-
InstructionCost getScalarizationOverhead(VectorType *InTy,
781-
const APInt &DemandedElts,
782-
bool Insert, bool Extract,
783-
TTI::TargetCostKind CostKind) {
780+
InstructionCost getScalarizationOverhead(
781+
VectorType *InTy, const APInt &DemandedElts, bool Insert, bool Extract,
782+
TTI::TargetCostKind CostKind, ArrayRef<Value *> VL = std::nullopt) {
783+
assert((VL.empty() ||
784+
VL.size() == cast<FixedVectorType>(InTy)->getNumElements()) &&
785+
"Type does not match the values.");
784786
/// FIXME: a bitfield is not a reasonable abstraction for talking about
785787
/// which elements are needed from a scalable vector
786788
if (isa<ScalableVectorType>(InTy))
@@ -795,9 +797,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
795797
for (int i = 0, e = Ty->getNumElements(); i < e; ++i) {
796798
if (!DemandedElts[i])
797799
continue;
798-
if (Insert)
800+
if (Insert) {
801+
Value *InsertedVal = VL.size() ? VL[i] : nullptr;
799802
Cost += thisT()->getVectorInstrCost(Instruction::InsertElement, Ty,
800-
CostKind, i, nullptr, nullptr);
803+
CostKind, i, nullptr, InsertedVal);
804+
}
801805
if (Extract)
802806
Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, Ty,
803807
CostKind, i, nullptr, nullptr);

‎llvm/lib/Analysis/TargetTransformInfo.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -622,9 +622,9 @@ bool TargetTransformInfo::isVectorIntrinsicWithOverloadTypeAtArg(
622622

623623
InstructionCost TargetTransformInfo::getScalarizationOverhead(
624624
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
625-
TTI::TargetCostKind CostKind) const {
625+
TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) const {
626626
return TTIImpl->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
627-
CostKind);
627+
CostKind, VL);
628628
}
629629

630630
InstructionCost TargetTransformInfo::getOperandsScalarizationOverhead(

‎llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3363,7 +3363,7 @@ InstructionCost AArch64TTIImpl::getVectorInstrCost(const Instruction &I,
33633363

33643364
InstructionCost AArch64TTIImpl::getScalarizationOverhead(
33653365
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
3366-
TTI::TargetCostKind CostKind) {
3366+
TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) {
33673367
if (isa<ScalableVectorType>(Ty))
33683368
return InstructionCost::getInvalid();
33693369
if (Ty->getElementType()->isFloatingPointTy())

‎llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,8 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
423423
InstructionCost getScalarizationOverhead(VectorType *Ty,
424424
const APInt &DemandedElts,
425425
bool Insert, bool Extract,
426-
TTI::TargetCostKind CostKind);
426+
TTI::TargetCostKind CostKind,
427+
ArrayRef<Value *> VL = std::nullopt);
427428

428429
/// Return the cost of the scaling factor used in the addressing
429430
/// mode represented by AM for this target, for a load/store

‎llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -669,7 +669,7 @@ static unsigned isM1OrSmaller(MVT VT) {
669669

670670
InstructionCost RISCVTTIImpl::getScalarizationOverhead(
671671
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
672-
TTI::TargetCostKind CostKind) {
672+
TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) {
673673
if (isa<ScalableVectorType>(Ty))
674674
return InstructionCost::getInvalid();
675675

‎llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,8 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
149149
InstructionCost getScalarizationOverhead(VectorType *Ty,
150150
const APInt &DemandedElts,
151151
bool Insert, bool Extract,
152-
TTI::TargetCostKind CostKind);
152+
TTI::TargetCostKind CostKind,
153+
ArrayRef<Value *> VL = std::nullopt);
153154

154155
InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
155156
TTI::TargetCostKind CostKind);

‎llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -468,17 +468,28 @@ bool SystemZTTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) {
468468
return (VT.isScalarInteger() && TLI->isTypeLegal(VT));
469469
}
470470

471+
static bool isFreeEltLoad(Value *Op) {
472+
if (isa<LoadInst>(Op) && Op->hasOneUse()) {
473+
const Instruction *UserI = cast<Instruction>(*Op->user_begin());
474+
return !isa<StoreInst>(UserI); // Prefer MVC
475+
}
476+
return false;
477+
}
478+
471479
InstructionCost SystemZTTIImpl::getScalarizationOverhead(
472480
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
473-
TTI::TargetCostKind CostKind) {
481+
TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) {
474482
unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
475483
InstructionCost Cost = 0;
476484

477485
if (Insert && Ty->isIntOrIntVectorTy(64)) {
478-
// VLVGP will insert two GPRs with one instruction.
486+
// VLVGP will insert two GPRs with one instruction, while VLE will load
487+
// an element directly with no extra cost
488+
assert((VL.empty() || VL.size() == NumElts) &&
489+
"Type does not match the number of values.");
479490
InstructionCost CurrVectorCost = 0;
480491
for (unsigned Idx = 0; Idx < NumElts; ++Idx) {
481-
if (DemandedElts[Idx])
492+
if (DemandedElts[Idx] && !(VL.size() && isFreeEltLoad(VL[Idx])))
482493
++CurrVectorCost;
483494
if (Idx % 2 == 1) {
484495
Cost += std::min(InstructionCost(1), CurrVectorCost);
@@ -489,7 +500,7 @@ InstructionCost SystemZTTIImpl::getScalarizationOverhead(
489500
}
490501

491502
Cost += BaseT::getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
492-
CostKind);
503+
CostKind, VL);
493504
return Cost;
494505
}
495506

@@ -1142,7 +1153,7 @@ InstructionCost SystemZTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
11421153
Value *Op1) {
11431154
if (Opcode == Instruction::InsertElement) {
11441155
// Vector Element Load.
1145-
if (Op1 != nullptr && Op1->hasOneUse() && isa<LoadInst>(Op1))
1156+
if (Op1 != nullptr && isFreeEltLoad(Op1))
11461157
return 0;
11471158

11481159
// vlvgp will insert two grs into a vector register, so count half the

‎llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,8 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
8484
InstructionCost getScalarizationOverhead(VectorType *Ty,
8585
const APInt &DemandedElts,
8686
bool Insert, bool Extract,
87-
TTI::TargetCostKind CostKind);
87+
TTI::TargetCostKind CostKind,
88+
ArrayRef<Value *> VL = std::nullopt);
8889
bool supportsEfficientVectorElementLoadStore() { return true; }
8990
bool enableInterleavedAccessVectorization() { return true; }
9091

0 commit comments

Comments
 (0)