Skip to content

Commit 11f178b

Browse files
MacDueMacDue
committed
[LV] Teach the vectorizer to cost and vectorize llvm.sincos intrinsics
This teaches the loop vectorizer that `llvm.sincos` is trivially vectorizable. Additionally, this patch updates the cost model to cost intrinsics that return multiple values correctly. Previously, the cost model only thought intrinsics that return `VectorType` need scalarizing, which meant it cost intrinsics that return multiple vectors (that need scalarizing) way too cheap (giving it the cost of a single function call). The `llvm.sincos` intrinsic also has a custom cost when a vector function library is available, as certain VFs can be expanded (later in code-gen) to a vector function, reducing the cost to a single call (+ the possible loads from the vector function returns values via output pointers).
1 parent 3f8e280 commit 11f178b

File tree

7 files changed

+256
-28
lines changed

7 files changed

+256
-28
lines changed

‎llvm/include/llvm/Analysis/TargetTransformInfo.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ class IntrinsicCostAttributes {
126126
// If ScalarizationCost is UINT_MAX, the cost of scalarizing the
127127
// arguments and the return value will be computed based on types.
128128
InstructionCost ScalarizationCost = InstructionCost::getInvalid();
129+
TargetLibraryInfo const *LibInfo = nullptr;
129130

130131
public:
131132
IntrinsicCostAttributes(
@@ -145,7 +146,8 @@ class IntrinsicCostAttributes {
145146
Intrinsic::ID Id, Type *RTy, ArrayRef<const Value *> Args,
146147
ArrayRef<Type *> Tys, FastMathFlags Flags = FastMathFlags(),
147148
const IntrinsicInst *I = nullptr,
148-
InstructionCost ScalarCost = InstructionCost::getInvalid());
149+
InstructionCost ScalarCost = InstructionCost::getInvalid(),
150+
TargetLibraryInfo const *LibInfo = nullptr);
149151

150152
Intrinsic::ID getID() const { return IID; }
151153
const IntrinsicInst *getInst() const { return II; }
@@ -154,6 +156,7 @@ class IntrinsicCostAttributes {
154156
InstructionCost getScalarizationCost() const { return ScalarizationCost; }
155157
const SmallVectorImpl<const Value *> &getArgs() const { return Arguments; }
156158
const SmallVectorImpl<Type *> &getArgTypes() const { return ParamTys; }
159+
const TargetLibraryInfo *getLibInfo() const { return LibInfo; }
157160

158161
bool isTypeBasedOnly() const {
159162
return Arguments.empty();

‎llvm/include/llvm/CodeGen/BasicTTIImpl.h‎

Lines changed: 70 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include "llvm/ADT/SmallVector.h"
2323
#include "llvm/Analysis/LoopInfo.h"
2424
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
25+
#include "llvm/Analysis/TargetLibraryInfo.h"
2526
#include "llvm/Analysis/TargetTransformInfo.h"
2627
#include "llvm/Analysis/TargetTransformInfoImpl.h"
2728
#include "llvm/Analysis/ValueTracking.h"
@@ -1725,9 +1726,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
17251726

17261727
Type *RetTy = ICA.getReturnType();
17271728

1728-
ElementCount RetVF =
1729-
(RetTy->isVectorTy() ? cast<VectorType>(RetTy)->getElementCount()
1730-
: ElementCount::getFixed(1));
1729+
ElementCount RetVF = isVectorizedTy(RetTy) ? getVectorizedTypeVF(RetTy)
1730+
: ElementCount::getFixed(1);
1731+
17311732
const IntrinsicInst *I = ICA.getInst();
17321733
const SmallVectorImpl<const Value *> &Args = ICA.getArgs();
17331734
FastMathFlags FMF = ICA.getFlags();
@@ -1995,6 +1996,49 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
19951996
}
19961997
case Intrinsic::experimental_vector_match:
19971998
return thisT()->getTypeBasedIntrinsicInstrCost(ICA, CostKind);
1999+
case Intrinsic::sincos: {
2000+
// Vector variants of llvm.sincos can be mapped to a vector library call.
2001+
auto const *LibInfo = ICA.getLibInfo();
2002+
if (!LibInfo || !isVectorizedTy(RetTy))
2003+
break;
2004+
2005+
// Find associated libcall.
2006+
VectorType *VectorTy = cast<VectorType>(getContainedTypes(RetTy).front());
2007+
EVT VT = getTLI()->getValueType(DL, VectorTy);
2008+
RTLIB::Libcall LC = RTLIB::getFSINCOS(VT.getVectorElementType());
2009+
const char *LCName = getTLI()->getLibcallName(LC);
2010+
if (!LC || !LCName)
2011+
break;
2012+
2013+
// Search for a corresponding vector variant.
2014+
LLVMContext &Ctx = RetTy->getContext();
2015+
auto VF = getVectorizedTypeVF(RetTy);
2016+
VecDesc const *VD = nullptr;
2017+
for (bool Masked : {false, true}) {
2018+
if ((VD = LibInfo->getVectorMappingInfo(LCName, VF, Masked)))
2019+
break;
2020+
}
2021+
if (!VD)
2022+
break;
2023+
2024+
// Cost the call + mask.
2025+
auto Cost = thisT()->getCallInstrCost(nullptr, RetTy, ICA.getArgTypes(),
2026+
CostKind);
2027+
if (VD->isMasked())
2028+
Cost += thisT()->getShuffleCost(
2029+
TargetTransformInfo::SK_Broadcast,
2030+
VectorType::get(IntegerType::getInt1Ty(Ctx), VF), {}, CostKind, 0,
2031+
nullptr, {});
2032+
2033+
// Lowering to a sincos library call (with output pointers) may require us
2034+
// to emit reloads for the results.
2035+
Cost +=
2036+
thisT()->getMemoryOpCost(
2037+
Instruction::Load, VectorTy,
2038+
thisT()->getDataLayout().getABITypeAlign(VectorTy), 0, CostKind) *
2039+
2;
2040+
return Cost;
2041+
}
19982042
}
19992043

20002044
// Assume that we need to scalarize this intrinsic.)
@@ -2003,10 +2047,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
20032047
InstructionCost ScalarizationCost = InstructionCost::getInvalid();
20042048
if (RetVF.isVector() && !RetVF.isScalable()) {
20052049
ScalarizationCost = 0;
2006-
if (!RetTy->isVoidTy())
2007-
ScalarizationCost += getScalarizationOverhead(
2008-
cast<VectorType>(RetTy),
2009-
/*Insert*/ true, /*Extract*/ false, CostKind);
2050+
if (!RetTy->isVoidTy()) {
2051+
for (Type *VectorTy : getContainedTypes(RetTy)) {
2052+
ScalarizationCost += getScalarizationOverhead(
2053+
cast<VectorType>(VectorTy),
2054+
/*Insert*/ true, /*Extract*/ false, CostKind);
2055+
}
2056+
}
20102057
ScalarizationCost +=
20112058
getOperandsScalarizationOverhead(Args, ICA.getArgTypes(), CostKind);
20122059
}
@@ -2678,27 +2725,32 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
26782725
// Else, assume that we need to scalarize this intrinsic. For math builtins
26792726
// this will emit a costly libcall, adding call overhead and spills. Make it
26802727
// very expensive.
2681-
if (auto *RetVTy = dyn_cast<VectorType>(RetTy)) {
2728+
if (isVectorizedTy(RetTy)) {
2729+
ArrayRef<Type *> RetVTys = getContainedTypes(RetTy);
2730+
26822731
// Scalable vectors cannot be scalarized, so return Invalid.
2683-
if (isa<ScalableVectorType>(RetTy) || any_of(Tys, [](const Type *Ty) {
2684-
return isa<ScalableVectorType>(Ty);
2685-
}))
2732+
if (any_of(concat<Type *const>(RetVTys, Tys),
2733+
[](Type *Ty) { return isa<ScalableVectorType>(Ty); }))
26862734
return InstructionCost::getInvalid();
26872735

2688-
InstructionCost ScalarizationCost =
2689-
SkipScalarizationCost
2690-
? ScalarizationCostPassed
2691-
: getScalarizationOverhead(RetVTy, /*Insert*/ true,
2692-
/*Extract*/ false, CostKind);
2736+
InstructionCost ScalarizationCost = ScalarizationCostPassed;
2737+
if (!SkipScalarizationCost) {
2738+
ScalarizationCost = 0;
2739+
for (Type *RetVTy : RetVTys) {
2740+
ScalarizationCost += getScalarizationOverhead(
2741+
cast<VectorType>(RetVTy), /*Insert*/ true,
2742+
/*Extract*/ false, CostKind);
2743+
}
2744+
}
26932745

2694-
unsigned ScalarCalls = cast<FixedVectorType>(RetVTy)->getNumElements();
2746+
unsigned ScalarCalls = getVectorizedTypeVF(RetTy).getFixedValue();
26952747
SmallVector<Type *, 4> ScalarTys;
26962748
for (Type *Ty : Tys) {
26972749
if (Ty->isVectorTy())
26982750
Ty = Ty->getScalarType();
26992751
ScalarTys.push_back(Ty);
27002752
}
2701-
IntrinsicCostAttributes Attrs(IID, RetTy->getScalarType(), ScalarTys, FMF);
2753+
IntrinsicCostAttributes Attrs(IID, toScalarizedTy(RetTy), ScalarTys, FMF);
27022754
InstructionCost ScalarCost =
27032755
thisT()->getIntrinsicInstrCost(Attrs, CostKind);
27042756
for (Type *Ty : Tys) {

‎llvm/lib/Analysis/TargetTransformInfo.cpp‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -101,13 +101,12 @@ IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *Ty,
101101
ParamTys.push_back(Argument->getType());
102102
}
103103

104-
IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
105-
ArrayRef<const Value *> Args,
106-
ArrayRef<Type *> Tys,
107-
FastMathFlags Flags,
108-
const IntrinsicInst *I,
109-
InstructionCost ScalarCost)
110-
: II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) {
104+
IntrinsicCostAttributes::IntrinsicCostAttributes(
105+
Intrinsic::ID Id, Type *RTy, ArrayRef<const Value *> Args,
106+
ArrayRef<Type *> Tys, FastMathFlags Flags, const IntrinsicInst *I,
107+
InstructionCost ScalarCost, TargetLibraryInfo const *LibInfo)
108+
: II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost),
109+
LibInfo(LibInfo) {
111110
ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end());
112111
Arguments.insert(Arguments.begin(), Args.begin(), Args.end());
113112
}

‎llvm/lib/Analysis/VectorUtils.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) {
7272
case Intrinsic::atan2:
7373
case Intrinsic::sin:
7474
case Intrinsic::cos:
75+
case Intrinsic::sincos:
7576
case Intrinsic::tan:
7677
case Intrinsic::sinh:
7778
case Intrinsic::cosh:
@@ -179,6 +180,7 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(
179180
case Intrinsic::ucmp:
180181
case Intrinsic::scmp:
181182
return OpdIdx == -1 || OpdIdx == 0;
183+
case Intrinsic::sincos:
182184
case Intrinsic::is_fpclass:
183185
case Intrinsic::vp_is_fpclass:
184186
return OpdIdx == 0;

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2919,7 +2919,8 @@ LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
29192919
[&](Type *Ty) { return maybeVectorizeType(Ty, VF); });
29202920

29212921
IntrinsicCostAttributes CostAttrs(ID, RetTy, Arguments, ParamTys, FMF,
2922-
dyn_cast<IntrinsicInst>(CI));
2922+
dyn_cast<IntrinsicInst>(CI),
2923+
InstructionCost::getInvalid(), TLI);
29232924
return TTI.getIntrinsicInstrCost(CostAttrs, CostKind);
29242925
}
29252926

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1180,7 +1180,8 @@ InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
11801180
FastMathFlags FMF = hasFastMathFlags() ? getFastMathFlags() : FastMathFlags();
11811181
IntrinsicCostAttributes CostAttrs(
11821182
VectorIntrinsicID, RetTy, Arguments, ParamTys, FMF,
1183-
dyn_cast_or_null<IntrinsicInst>(getUnderlyingValue()));
1183+
dyn_cast_or_null<IntrinsicInst>(getUnderlyingValue()),
1184+
InstructionCost::getInvalid(), &Ctx.TLI);
11841185
return Ctx.TTI.getIntrinsicInstrCost(CostAttrs, Ctx.CostKind);
11851186
}
11861187

0 commit comments

Comments
 (0)