Skip to content

Commit 37a5288

Browse files
sushgokhsushgokh
committed
[AArch64][CostModel] Improve cost estimate of scalarizing a vector division
In the backend, last resort of finding the vector division cost is to use its scalar cost. However, without knowledge about the division operands, the cost can be off in certain cases. For SLP, this patch tries to pass scalars for better scalar cost estimation in the backend.
1 parent d02c167 commit 37a5288

29 files changed

+250
-240
lines changed

‎llvm/include/llvm/Analysis/TargetTransformInfo.h‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1297,13 +1297,16 @@ class TargetTransformInfo {
12971297
/// provide even more information.
12981298
/// \p TLibInfo is used to search for platform specific vector library
12991299
/// functions for instructions that might be converted to calls (e.g. frem).
1300+
/// \p Scalars refers to individual scalars/instructions being used for
1301+
/// vectorization.
13001302
InstructionCost getArithmeticInstrCost(
13011303
unsigned Opcode, Type *Ty,
13021304
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
13031305
TTI::OperandValueInfo Opd1Info = {TTI::OK_AnyValue, TTI::OP_None},
13041306
TTI::OperandValueInfo Opd2Info = {TTI::OK_AnyValue, TTI::OP_None},
13051307
ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr,
1306-
const TargetLibraryInfo *TLibInfo = nullptr) const;
1308+
const TargetLibraryInfo *TLibInfo = nullptr,
1309+
ArrayRef<Value *> Scalars = {}) const;
13071310

13081311
/// Returns the cost estimation for alternating opcode pattern that can be
13091312
/// lowered to a single instruction on the target. In X86 this is for the
@@ -2099,7 +2102,8 @@ class TargetTransformInfo::Concept {
20992102
virtual InstructionCost getArithmeticInstrCost(
21002103
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
21012104
OperandValueInfo Opd1Info, OperandValueInfo Opd2Info,
2102-
ArrayRef<const Value *> Args, const Instruction *CxtI = nullptr) = 0;
2105+
ArrayRef<const Value *> Args, const Instruction *CxtI = nullptr,
2106+
ArrayRef<Value *> Scalars = {}) = 0;
21032107
virtual InstructionCost getAltInstrCost(
21042108
VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
21052109
const SmallBitVector &OpcodeMask,
@@ -2780,10 +2784,10 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
27802784
InstructionCost getArithmeticInstrCost(
27812785
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
27822786
OperandValueInfo Opd1Info, OperandValueInfo Opd2Info,
2783-
ArrayRef<const Value *> Args,
2784-
const Instruction *CxtI = nullptr) override {
2787+
ArrayRef<const Value *> Args, const Instruction *CxtI = nullptr,
2788+
ArrayRef<Value *> Scalars = {}) override {
27852789
return Impl.getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info, Opd2Info,
2786-
Args, CxtI);
2790+
Args, CxtI, Scalars);
27872791
}
27882792
InstructionCost getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
27892793
unsigned Opcode1,

‎llvm/include/llvm/Analysis/TargetTransformInfoImpl.h‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -581,11 +581,13 @@ class TargetTransformInfoImplBase {
581581

582582
unsigned getMaxInterleaveFactor(ElementCount VF) const { return 1; }
583583

584-
InstructionCost getArithmeticInstrCost(
585-
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
586-
TTI::OperandValueInfo Opd1Info, TTI::OperandValueInfo Opd2Info,
587-
ArrayRef<const Value *> Args,
588-
const Instruction *CxtI = nullptr) const {
584+
InstructionCost getArithmeticInstrCost(unsigned Opcode, Type *Ty,
585+
TTI::TargetCostKind CostKind,
586+
TTI::OperandValueInfo Opd1Info,
587+
TTI::OperandValueInfo Opd2Info,
588+
ArrayRef<const Value *> Args,
589+
const Instruction *CxtI = nullptr,
590+
ArrayRef<Value *> Scalars = {}) const {
589591
// Widenable conditions will eventually lower into constants, so some
590592
// operations with them will be trivially optimized away.
591593
auto IsWidenableCondition = [](const Value *V) {

‎llvm/include/llvm/CodeGen/BasicTTIImpl.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -924,7 +924,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
924924
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
925925
TTI::OperandValueInfo Opd1Info = {TTI::OK_AnyValue, TTI::OP_None},
926926
TTI::OperandValueInfo Opd2Info = {TTI::OK_AnyValue, TTI::OP_None},
927-
ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr) {
927+
ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr,
928+
ArrayRef<Value *> Scalars = {}) {
928929
// Check if any of the operands are vector operands.
929930
const TargetLoweringBase *TLI = getTLI();
930931
int ISD = TLI->InstructionOpcodeToISD(Opcode);

‎llvm/lib/Analysis/TargetTransformInfo.cpp‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -927,7 +927,7 @@ InstructionCost TargetTransformInfo::getArithmeticInstrCost(
927927
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
928928
OperandValueInfo Op1Info, OperandValueInfo Op2Info,
929929
ArrayRef<const Value *> Args, const Instruction *CxtI,
930-
const TargetLibraryInfo *TLibInfo) const {
930+
const TargetLibraryInfo *TLibInfo, ArrayRef<Value *> Scalars) const {
931931

932932
// Use call cost for frem intructions that have platform specific vector math
933933
// functions, as those will be replaced with calls later by SelectionDAG or
@@ -942,10 +942,8 @@ InstructionCost TargetTransformInfo::getArithmeticInstrCost(
942942
return getCallInstrCost(nullptr, VecTy, {VecTy, VecTy}, CostKind);
943943
}
944944

945-
InstructionCost Cost =
946-
TTIImpl->getArithmeticInstrCost(Opcode, Ty, CostKind,
947-
Op1Info, Op2Info,
948-
Args, CxtI);
945+
InstructionCost Cost = TTIImpl->getArithmeticInstrCost(
946+
Opcode, Ty, CostKind, Op1Info, Op2Info, Args, CxtI, Scalars);
949947
assert(Cost >= 0 && "TTI should not produce negative costs!");
950948
return Cost;
951949
}

‎llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp‎

Lines changed: 34 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3376,8 +3376,8 @@ InstructionCost AArch64TTIImpl::getScalarizationOverhead(
33763376
InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
33773377
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
33783378
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
3379-
ArrayRef<const Value *> Args,
3380-
const Instruction *CxtI) {
3379+
ArrayRef<const Value *> Args, const Instruction *CxtI,
3380+
ArrayRef<Value *> Scalars) {
33813381

33823382
// The code-generator is currently not able to handle scalable vectors
33833383
// of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
@@ -3442,8 +3442,8 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
34423442
if (!VT.isVector() && VT.getSizeInBits() > 64)
34433443
return getCallInstrCost(/*Function*/ nullptr, Ty, {Ty, Ty}, CostKind);
34443444

3445-
InstructionCost Cost = BaseT::getArithmeticInstrCost(
3446-
Opcode, Ty, CostKind, Op1Info, Op2Info);
3445+
InstructionCost Cost =
3446+
BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info);
34473447
if (Ty->isVectorTy()) {
34483448
if (TLI->isOperationLegalOrCustom(ISD, LT.second) && ST->hasSVE()) {
34493449
// SDIV/UDIV operations are lowered using SVE, then we can have less
@@ -3472,29 +3472,41 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
34723472
Cost *= 4;
34733473
return Cost;
34743474
} else {
3475-
// If one of the operands is a uniform constant then the cost for each
3476-
// element is Cost for insertion, extraction and division.
3477-
// Insertion cost = 2, Extraction Cost = 2, Division = cost for the
3478-
// operation with scalar type
3479-
if ((Op1Info.isConstant() && Op1Info.isUniform()) ||
3480-
(Op2Info.isConstant() && Op2Info.isUniform())) {
3481-
if (auto *VTy = dyn_cast<FixedVectorType>(Ty)) {
3475+
if (auto *VTy = dyn_cast<FixedVectorType>(Ty)) {
3476+
if ((Op1Info.isConstant() && Op1Info.isUniform()) ||
3477+
(Op2Info.isConstant() && Op2Info.isUniform())) {
34823478
InstructionCost DivCost = BaseT::getArithmeticInstrCost(
34833479
Opcode, Ty->getScalarType(), CostKind, Op1Info, Op2Info);
3484-
return (4 + DivCost) * VTy->getNumElements();
3480+
// If #vector_elements = n then we need
3481+
// n inserts + 2n extracts + n divisions.
3482+
InstructionCost InsertExtractCost =
3483+
ST->getVectorInsertExtractBaseCost();
3484+
Cost = (3 * InsertExtractCost + DivCost) * VTy->getNumElements();
3485+
} else if (!Scalars.empty()) {
3486+
// If #vector_elements = n then we need
3487+
// n inserts + 2n extracts + n divisions.
3488+
InstructionCost InsertExtractCost =
3489+
ST->getVectorInsertExtractBaseCost();
3490+
Cost = (3 * InsertExtractCost) * VTy->getNumElements();
3491+
for (auto *V : Scalars) {
3492+
auto *I = cast<Instruction>(V);
3493+
Cost +=
3494+
getArithmeticInstrCost(I->getOpcode(), I->getType(), CostKind,
3495+
TTI::getOperandInfo(I->getOperand(0)),
3496+
TTI::getOperandInfo(I->getOperand(1)));
3497+
}
3498+
} else {
3499+
// FIXME: The initial cost calculated should have considered extract
3500+
// cost twice. For now, we just add additional cost to avoid
3501+
// underestimating the total cost.
3502+
Cost += Cost;
34853503
}
3504+
} else {
3505+
// We can't predict the cost of div/extract/insert without knowing the
3506+
// vector width.
3507+
Cost.setInvalid();
34863508
}
3487-
// On AArch64, without SVE, vector divisions are expanded
3488-
// into scalar divisions of each pair of elements.
3489-
Cost += getArithmeticInstrCost(Instruction::ExtractElement, Ty,
3490-
CostKind, Op1Info, Op2Info);
3491-
Cost += getArithmeticInstrCost(Instruction::InsertElement, Ty, CostKind,
3492-
Op1Info, Op2Info);
34933509
}
3494-
3495-
// TODO: if one of the arguments is scalar, then it's not necessary to
3496-
// double the cost of handling the vector elements.
3497-
Cost += Cost;
34983510
}
34993511
return Cost;
35003512
}

‎llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,8 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
218218
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
219219
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
220220
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
221-
ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr);
221+
ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr,
222+
ArrayRef<Value *> Scalars = {});
222223

223224
InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *SE,
224225
const SCEV *Ptr);

‎llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -546,8 +546,8 @@ bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
546546
InstructionCost GCNTTIImpl::getArithmeticInstrCost(
547547
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
548548
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
549-
ArrayRef<const Value *> Args,
550-
const Instruction *CxtI) {
549+
ArrayRef<const Value *> Args, const Instruction *CxtI,
550+
ArrayRef<Value *> Scalars) {
551551

552552
// Legalize the type.
553553
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);

‎llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,8 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
157157
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
158158
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
159159
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
160-
ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr);
160+
ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr,
161+
ArrayRef<Value *> Scalars = {});
161162

162163
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
163164
const Instruction *I = nullptr);

‎llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "ARMSubtarget.h"
1111
#include "MCTargetDesc/ARMAddressingModes.h"
1212
#include "llvm/ADT/APInt.h"
13+
#include "llvm/ADT/ArrayRef.h"
1314
#include "llvm/ADT/SmallVector.h"
1415
#include "llvm/Analysis/LoopInfo.h"
1516
#include "llvm/CodeGen/CostTable.h"
@@ -1349,8 +1350,8 @@ InstructionCost ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
13491350
InstructionCost ARMTTIImpl::getArithmeticInstrCost(
13501351
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
13511352
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
1352-
ArrayRef<const Value *> Args,
1353-
const Instruction *CxtI) {
1353+
ArrayRef<const Value *> Args, const Instruction *CxtI,
1354+
ArrayRef<Value *> Scalars) {
13541355
int ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);
13551356
if (ST->isThumb() && CostKind == TTI::TCK_CodeSize && Ty->isIntegerTy(1)) {
13561357
// Make operations on i1 relatively expensive as this often involves

‎llvm/lib/Target/ARM/ARMTargetTransformInfo.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,8 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
258258
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
259259
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
260260
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
261-
ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr);
261+
ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr,
262+
ArrayRef<Value *> Scalars = {});
262263

263264
InstructionCost
264265
getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,

0 commit comments

Comments
 (0)