Skip to content

Commit 0dbf0e8

Browse files
MacDueMacDue
committed
[LV] Teach the vectorizer to cost and vectorize llvm.sincos intrinsics
This teaches the loop vectorizer that `llvm.sincos` is trivially vectorizable. Additionally, this patch updates the cost model to cost intrinsics that return multiple values correctly. Previously, the cost model only thought intrinsics that return `VectorType` need scalarizing, which meant it cost intrinsics that return multiple vectors (that need scalarizing) way too cheap (giving it the cost of a single function call). The `llvm.sincos` intrinsic also has a custom cost when a vector function library is available, as certain VFs can be expanded (later in code-gen) to a vector function, reducing the cost to a single call (+ the possible loads from the vector function returns values via output pointers).
1 parent c1de9b9 commit 0dbf0e8

File tree

7 files changed

+253
-27
lines changed

7 files changed

+253
-27
lines changed

‎llvm/include/llvm/Analysis/TargetTransformInfo.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ class IntrinsicCostAttributes {
126126
// If ScalarizationCost is UINT_MAX, the cost of scalarizing the
127127
// arguments and the return value will be computed based on types.
128128
InstructionCost ScalarizationCost = InstructionCost::getInvalid();
129+
TargetLibraryInfo const *LibInfo = nullptr;
129130

130131
public:
131132
IntrinsicCostAttributes(
@@ -145,7 +146,8 @@ class IntrinsicCostAttributes {
145146
Intrinsic::ID Id, Type *RTy, ArrayRef<const Value *> Args,
146147
ArrayRef<Type *> Tys, FastMathFlags Flags = FastMathFlags(),
147148
const IntrinsicInst *I = nullptr,
148-
InstructionCost ScalarCost = InstructionCost::getInvalid());
149+
InstructionCost ScalarCost = InstructionCost::getInvalid(),
150+
TargetLibraryInfo const *LibInfo = nullptr);
149151

150152
Intrinsic::ID getID() const { return IID; }
151153
const IntrinsicInst *getInst() const { return II; }
@@ -154,6 +156,7 @@ class IntrinsicCostAttributes {
154156
InstructionCost getScalarizationCost() const { return ScalarizationCost; }
155157
const SmallVectorImpl<const Value *> &getArgs() const { return Arguments; }
156158
const SmallVectorImpl<Type *> &getArgTypes() const { return ParamTys; }
159+
const TargetLibraryInfo *getLibInfo() const { return LibInfo; }
157160

158161
bool isTypeBasedOnly() const {
159162
return Arguments.empty();

‎llvm/include/llvm/CodeGen/BasicTTIImpl.h‎

Lines changed: 67 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include "llvm/ADT/SmallVector.h"
2323
#include "llvm/Analysis/LoopInfo.h"
2424
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
25+
#include "llvm/Analysis/TargetLibraryInfo.h"
2526
#include "llvm/Analysis/TargetTransformInfo.h"
2627
#include "llvm/Analysis/TargetTransformInfoImpl.h"
2728
#include "llvm/Analysis/ValueTracking.h"
@@ -1718,8 +1719,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
17181719
Type *RetTy = ICA.getReturnType();
17191720

17201721
ElementCount RetVF =
1721-
(RetTy->isVectorTy() ? cast<VectorType>(RetTy)->getElementCount()
1722-
: ElementCount::getFixed(1));
1722+
isVectorizedTy(RetTy) ? getVectorizedTypeVF(RetTy) : ElementCount::getFixed(1);
1723+
17231724
const IntrinsicInst *I = ICA.getInst();
17241725
const SmallVectorImpl<const Value *> &Args = ICA.getArgs();
17251726
FastMathFlags FMF = ICA.getFlags();
@@ -1972,6 +1973,47 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
19721973
}
19731974
case Intrinsic::experimental_vector_match:
19741975
return thisT()->getTypeBasedIntrinsicInstrCost(ICA, CostKind);
1976+
case Intrinsic::sincos: {
1977+
// Vector variants of llvm.sincos can be mapped to a vector library call.
1978+
auto const *LibInfo = ICA.getLibInfo();
1979+
if (!LibInfo || !isVectorizedTy(RetTy))
1980+
break;
1981+
1982+
// Find associated libcall.
1983+
VectorType *VectorTy = cast<VectorType>(getContainedTypes(RetTy).front());
1984+
EVT VT = getTLI()->getValueType(DL, VectorTy);
1985+
RTLIB::Libcall LC = RTLIB::getFSINCOS(VT.getVectorElementType());
1986+
const char *LCName = getTLI()->getLibcallName(LC);
1987+
if (!LC || !LCName)
1988+
break;
1989+
1990+
// Search for a corresponding vector variant.
1991+
LLVMContext &Ctx = RetTy->getContext();
1992+
auto VF = getVectorizedTypeVF(RetTy);
1993+
VecDesc const *VD = nullptr;
1994+
for (bool Masked : {false, true}) {
1995+
if ((VD = LibInfo->getVectorMappingInfo(LCName, VF, Masked)))
1996+
break;
1997+
}
1998+
if (!VD)
1999+
break;
2000+
2001+
// Cost the call + mask.
2002+
auto Cost = thisT()->getCallInstrCost(nullptr, RetTy, ICA.getArgTypes(), CostKind);
2003+
if (VD->isMasked())
2004+
Cost += thisT()->getShuffleCost(TargetTransformInfo::SK_Broadcast,
2005+
VectorType::get(IntegerType::getInt1Ty(Ctx), VF),
2006+
{}, CostKind, 0, nullptr, {});
2007+
2008+
// Lowering to a sincos library call (with output pointers) may require us
2009+
// to emit reloads for the results.
2010+
Cost +=
2011+
thisT()->getMemoryOpCost(Instruction::Load, VectorTy,
2012+
thisT()->getDataLayout().getABITypeAlign(VectorTy), 0,
2013+
CostKind) *
2014+
2;
2015+
return Cost;
2016+
}
19752017
}
19762018

19772019
// Assume that we need to scalarize this intrinsic.)
@@ -1980,10 +2022,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
19802022
InstructionCost ScalarizationCost = InstructionCost::getInvalid();
19812023
if (RetVF.isVector() && !RetVF.isScalable()) {
19822024
ScalarizationCost = 0;
1983-
if (!RetTy->isVoidTy())
1984-
ScalarizationCost += getScalarizationOverhead(
1985-
cast<VectorType>(RetTy),
1986-
/*Insert*/ true, /*Extract*/ false, CostKind);
2025+
if (!RetTy->isVoidTy()) {
2026+
for (Type *VectorTy : getContainedTypes(RetTy)) {
2027+
ScalarizationCost += getScalarizationOverhead(
2028+
cast<VectorType>(VectorTy),
2029+
/*Insert*/ true, /*Extract*/ false, CostKind);
2030+
}
2031+
}
19872032
ScalarizationCost +=
19882033
getOperandsScalarizationOverhead(Args, ICA.getArgTypes(), CostKind);
19892034
}
@@ -2609,27 +2654,32 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
26092654
// Else, assume that we need to scalarize this intrinsic. For math builtins
26102655
// this will emit a costly libcall, adding call overhead and spills. Make it
26112656
// very expensive.
2612-
if (auto *RetVTy = dyn_cast<VectorType>(RetTy)) {
2657+
if (isVectorizedTy(RetTy)) {
2658+
ArrayRef<Type *> RetVTys = getContainedTypes(RetTy);
2659+
26132660
// Scalable vectors cannot be scalarized, so return Invalid.
2614-
if (isa<ScalableVectorType>(RetTy) || any_of(Tys, [](const Type *Ty) {
2615-
return isa<ScalableVectorType>(Ty);
2616-
}))
2661+
if (any_of(concat<Type *const>(RetVTys, Tys),
2662+
[](Type *Ty) { return isa<ScalableVectorType>(Ty); }))
26172663
return InstructionCost::getInvalid();
26182664

2619-
InstructionCost ScalarizationCost =
2620-
SkipScalarizationCost
2621-
? ScalarizationCostPassed
2622-
: getScalarizationOverhead(RetVTy, /*Insert*/ true,
2623-
/*Extract*/ false, CostKind);
2665+
InstructionCost ScalarizationCost = ScalarizationCostPassed;
2666+
if (!SkipScalarizationCost) {
2667+
ScalarizationCost = 0;
2668+
for (Type *RetVTy : RetVTys) {
2669+
ScalarizationCost += getScalarizationOverhead(
2670+
cast<VectorType>(RetVTy), /*Insert*/ true,
2671+
/*Extract*/ false, CostKind);
2672+
}
2673+
}
26242674

2625-
unsigned ScalarCalls = cast<FixedVectorType>(RetVTy)->getNumElements();
2675+
unsigned ScalarCalls = getVectorizedTypeVF(RetTy).getFixedValue();
26262676
SmallVector<Type *, 4> ScalarTys;
26272677
for (Type *Ty : Tys) {
26282678
if (Ty->isVectorTy())
26292679
Ty = Ty->getScalarType();
26302680
ScalarTys.push_back(Ty);
26312681
}
2632-
IntrinsicCostAttributes Attrs(IID, RetTy->getScalarType(), ScalarTys, FMF);
2682+
IntrinsicCostAttributes Attrs(IID, toScalarizedTy(RetTy), ScalarTys, FMF);
26332683
InstructionCost ScalarCost =
26342684
thisT()->getIntrinsicInstrCost(Attrs, CostKind);
26352685
for (Type *Ty : Tys) {

‎llvm/lib/Analysis/TargetTransformInfo.cpp‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -101,13 +101,12 @@ IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *Ty,
101101
ParamTys.push_back(Argument->getType());
102102
}
103103

104-
IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
105-
ArrayRef<const Value *> Args,
106-
ArrayRef<Type *> Tys,
107-
FastMathFlags Flags,
108-
const IntrinsicInst *I,
109-
InstructionCost ScalarCost)
110-
: II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) {
104+
IntrinsicCostAttributes::IntrinsicCostAttributes(
105+
Intrinsic::ID Id, Type *RTy, ArrayRef<const Value *> Args,
106+
ArrayRef<Type *> Tys, FastMathFlags Flags, const IntrinsicInst *I,
107+
InstructionCost ScalarCost, TargetLibraryInfo const *LibInfo)
108+
: II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost),
109+
LibInfo(LibInfo) {
111110
ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end());
112111
Arguments.insert(Arguments.begin(), Args.begin(), Args.end());
113112
}

‎llvm/lib/Analysis/VectorUtils.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) {
7272
case Intrinsic::atan2:
7373
case Intrinsic::sin:
7474
case Intrinsic::cos:
75+
case Intrinsic::sincos:
7576
case Intrinsic::tan:
7677
case Intrinsic::sinh:
7778
case Intrinsic::cosh:
@@ -179,6 +180,7 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(
179180
case Intrinsic::ucmp:
180181
case Intrinsic::scmp:
181182
return OpdIdx == -1 || OpdIdx == 0;
183+
case Intrinsic::sincos:
182184
case Intrinsic::is_fpclass:
183185
case Intrinsic::vp_is_fpclass:
184186
return OpdIdx == 0;

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2971,7 +2971,8 @@ LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
29712971
[&](Type *Ty) { return maybeVectorizeType(Ty, VF); });
29722972

29732973
IntrinsicCostAttributes CostAttrs(ID, RetTy, Arguments, ParamTys, FMF,
2974-
dyn_cast<IntrinsicInst>(CI));
2974+
dyn_cast<IntrinsicInst>(CI),
2975+
InstructionCost::getInvalid(), TLI);
29752976
return TTI.getIntrinsicInstrCost(CostAttrs,
29762977
TargetTransformInfo::TCK_RecipThroughput);
29772978
}

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1112,7 +1112,8 @@ InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
11121112
FastMathFlags FMF = hasFastMathFlags() ? getFastMathFlags() : FastMathFlags();
11131113
IntrinsicCostAttributes CostAttrs(
11141114
VectorIntrinsicID, RetTy, Arguments, ParamTys, FMF,
1115-
dyn_cast_or_null<IntrinsicInst>(getUnderlyingValue()));
1115+
dyn_cast_or_null<IntrinsicInst>(getUnderlyingValue()),
1116+
InstructionCost::getInvalid(), &Ctx.TLI);
11161117
return Ctx.TTI.getIntrinsicInstrCost(CostAttrs, CostKind);
11171118
}
11181119

0 commit comments

Comments
 (0)