Skip to content

Commit ba3eb68

Browse files
JonPsson1JonPsson1
committed
Pass VL to getScalarizationOverhead
1 parent 4a85911 commit ba3eb68

File tree

14 files changed

+120
-59
lines changed

14 files changed

+120
-59
lines changed

‎llvm/include/llvm/Analysis/TargetTransformInfo.h‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -898,11 +898,11 @@ class TargetTransformInfo {
898898

899899
/// Estimate the overhead of scalarizing an instruction. Insert and Extract
900900
/// are set if the demanded result elements need to be inserted and/or
901-
/// extracted from vectors.
902-
InstructionCost getScalarizationOverhead(VectorType *Ty,
903-
const APInt &DemandedElts,
904-
bool Insert, bool Extract,
905-
TTI::TargetCostKind CostKind) const;
901+
/// extracted from vectors. The involved values may be passed in VL if
902+
/// Insert is true.
903+
InstructionCost getScalarizationOverhead(
904+
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
905+
TTI::TargetCostKind CostKind, ArrayRef<Value *> VL = std::nullopt) const;
906906

907907
/// Estimate the overhead of scalarizing an instructions unique
908908
/// non-constant operands. The (potentially vector) types to use for each of
@@ -1973,10 +1973,10 @@ class TargetTransformInfo::Concept {
19731973
virtual bool isTargetIntrinsicTriviallyScalarizable(Intrinsic::ID ID) = 0;
19741974
virtual bool isTargetIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
19751975
unsigned ScalarOpdIdx) = 0;
1976-
virtual InstructionCost getScalarizationOverhead(VectorType *Ty,
1977-
const APInt &DemandedElts,
1978-
bool Insert, bool Extract,
1979-
TargetCostKind CostKind) = 0;
1976+
virtual InstructionCost
1977+
getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
1978+
bool Insert, bool Extract, TargetCostKind CostKind,
1979+
ArrayRef<Value *> VL = std::nullopt) = 0;
19801980
virtual InstructionCost
19811981
getOperandsScalarizationOverhead(ArrayRef<const Value *> Args,
19821982
ArrayRef<Type *> Tys,
@@ -2536,12 +2536,12 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
25362536
return Impl.isTargetIntrinsicWithScalarOpAtArg(ID, ScalarOpdIdx);
25372537
}
25382538

2539-
InstructionCost getScalarizationOverhead(VectorType *Ty,
2540-
const APInt &DemandedElts,
2541-
bool Insert, bool Extract,
2542-
TargetCostKind CostKind) override {
2539+
InstructionCost
2540+
getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
2541+
bool Insert, bool Extract, TargetCostKind CostKind,
2542+
ArrayRef<Value *> VL = std::nullopt) override {
25432543
return Impl.getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
2544-
CostKind);
2544+
CostKind, VL);
25452545
}
25462546
InstructionCost
25472547
getOperandsScalarizationOverhead(ArrayRef<const Value *> Args,

‎llvm/include/llvm/Analysis/TargetTransformInfoImpl.h‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -393,10 +393,9 @@ class TargetTransformInfoImplBase {
393393
return false;
394394
}
395395

396-
InstructionCost getScalarizationOverhead(VectorType *Ty,
397-
const APInt &DemandedElts,
398-
bool Insert, bool Extract,
399-
TTI::TargetCostKind CostKind) const {
396+
InstructionCost getScalarizationOverhead(
397+
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
398+
TTI::TargetCostKind CostKind, ArrayRef<Value *> VL = std::nullopt) const {
400399
return 0;
401400
}
402401

‎llvm/include/llvm/CodeGen/BasicTTIImpl.h‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -760,10 +760,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
760760
/// Estimate the overhead of scalarizing an instruction. Insert and Extract
761761
/// are set if the demanded result elements need to be inserted and/or
762762
/// extracted from vectors.
763-
InstructionCost getScalarizationOverhead(VectorType *InTy,
764-
const APInt &DemandedElts,
765-
bool Insert, bool Extract,
766-
TTI::TargetCostKind CostKind) {
763+
InstructionCost getScalarizationOverhead(
764+
VectorType *InTy, const APInt &DemandedElts, bool Insert, bool Extract,
765+
TTI::TargetCostKind CostKind, ArrayRef<Value *> VL = std::nullopt) {
766+
assert((VL.empty() ||
767+
VL.size() == cast<FixedVectorType>(InTy)->getNumElements()) &&
768+
"Type does not match the values.");
767769
/// FIXME: a bitfield is not a reasonable abstraction for talking about
768770
/// which elements are needed from a scalable vector
769771
if (isa<ScalableVectorType>(InTy))
@@ -778,9 +780,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
778780
for (int i = 0, e = Ty->getNumElements(); i < e; ++i) {
779781
if (!DemandedElts[i])
780782
continue;
781-
if (Insert)
783+
if (Insert) {
784+
Value *InsertedVal = VL.size() ? VL[i] : nullptr;
782785
Cost += thisT()->getVectorInstrCost(Instruction::InsertElement, Ty,
783-
CostKind, i, nullptr, nullptr);
786+
CostKind, i, nullptr, InsertedVal);
787+
}
784788
if (Extract)
785789
Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, Ty,
786790
CostKind, i, nullptr, nullptr);

‎llvm/lib/Analysis/TargetTransformInfo.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -613,9 +613,9 @@ bool TargetTransformInfo::isTargetIntrinsicWithScalarOpAtArg(
613613

614614
InstructionCost TargetTransformInfo::getScalarizationOverhead(
615615
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
616-
TTI::TargetCostKind CostKind) const {
616+
TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) const {
617617
return TTIImpl->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
618-
CostKind);
618+
CostKind, VL);
619619
}
620620

621621
InstructionCost TargetTransformInfo::getOperandsScalarizationOverhead(

‎llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3248,7 +3248,7 @@ InstructionCost AArch64TTIImpl::getVectorInstrCost(const Instruction &I,
32483248

32493249
InstructionCost AArch64TTIImpl::getScalarizationOverhead(
32503250
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
3251-
TTI::TargetCostKind CostKind) {
3251+
TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) {
32523252
if (isa<ScalableVectorType>(Ty))
32533253
return InstructionCost::getInvalid();
32543254
if (Ty->getElementType()->isFloatingPointTy())

‎llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -406,7 +406,8 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
406406
InstructionCost getScalarizationOverhead(VectorType *Ty,
407407
const APInt &DemandedElts,
408408
bool Insert, bool Extract,
409-
TTI::TargetCostKind CostKind);
409+
TTI::TargetCostKind CostKind,
410+
ArrayRef<Value *> VL = std::nullopt);
410411

411412
/// Return the cost of the scaling factor used in the addressing
412413
/// mode represented by AM for this target, for a load/store

‎llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -669,7 +669,7 @@ static unsigned isM1OrSmaller(MVT VT) {
669669

670670
InstructionCost RISCVTTIImpl::getScalarizationOverhead(
671671
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
672-
TTI::TargetCostKind CostKind) {
672+
TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) {
673673
if (isa<ScalableVectorType>(Ty))
674674
return InstructionCost::getInvalid();
675675

‎llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,8 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
152152
InstructionCost getScalarizationOverhead(VectorType *Ty,
153153
const APInt &DemandedElts,
154154
bool Insert, bool Extract,
155-
TTI::TargetCostKind CostKind);
155+
TTI::TargetCostKind CostKind,
156+
ArrayRef<Value *> VL = std::nullopt);
156157

157158
InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
158159
TTI::TargetCostKind CostKind);

‎llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -469,17 +469,28 @@ bool SystemZTTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) {
469469
return (VT.isScalarInteger() && TLI->isTypeLegal(VT));
470470
}
471471

472+
static bool isFreeEltLoad(Value *Op) {
473+
if (isa<LoadInst>(Op) && Op->hasOneUse()) {
474+
const Instruction *UserI = cast<Instruction>(*Op->user_begin());
475+
return !isa<StoreInst>(UserI); // Prefer MVC
476+
}
477+
return false;
478+
}
479+
472480
InstructionCost SystemZTTIImpl::getScalarizationOverhead(
473481
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
474-
TTI::TargetCostKind CostKind) {
482+
TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) {
475483
unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
476484
InstructionCost Cost = 0;
477485

478486
if (Insert && Ty->isIntOrIntVectorTy(64)) {
479-
// VLVGP will insert two GPRs with one instruction.
487+
// VLVGP will insert two GPRs with one instruction, while VLE will load
488+
// an element directly with no extra cost
489+
assert((VL.empty() || VL.size() == NumElts) &&
490+
"Type does not match the number of values.");
480491
InstructionCost CurrVectorCost = 0;
481492
for (unsigned Idx = 0; Idx < NumElts; ++Idx) {
482-
if (DemandedElts[Idx])
493+
if (DemandedElts[Idx] && !(VL.size() && isFreeEltLoad(VL[Idx])))
483494
++CurrVectorCost;
484495
if (Idx % 2 == 1) {
485496
Cost += std::min(InstructionCost(1), CurrVectorCost);
@@ -490,7 +501,7 @@ InstructionCost SystemZTTIImpl::getScalarizationOverhead(
490501
}
491502

492503
Cost += BaseT::getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
493-
CostKind);
504+
CostKind, VL);
494505
return Cost;
495506
}
496507

@@ -1143,7 +1154,7 @@ InstructionCost SystemZTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
11431154
Value *Op1) {
11441155
if (Opcode == Instruction::InsertElement) {
11451156
// Vector Element Load.
1146-
if (Op1 != nullptr && Op1->hasOneUse() && isa<LoadInst>(Op1))
1157+
if (Op1 != nullptr && isFreeEltLoad(Op1))
11471158
return 0;
11481159

11491160
// vlvgp will insert two grs into a vector register, so count half the

‎llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,8 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
8484
InstructionCost getScalarizationOverhead(VectorType *Ty,
8585
const APInt &DemandedElts,
8686
bool Insert, bool Extract,
87-
TTI::TargetCostKind CostKind);
87+
TTI::TargetCostKind CostKind,
88+
ArrayRef<Value *> VL = std::nullopt);
8889
bool supportsEfficientVectorElementLoadStore() { return true; }
8990
bool enableInterleavedAccessVectorization() { return true; }
9091

0 commit comments

Comments
 (0)