@@ -7634,6 +7634,60 @@ bool BoUpSLP::areAltOperandsProfitable(const InstructionsState &S,
76347634 NumAltInsts) < S.getMainOp()->getNumOperands() * VL.size());
76357635}
76367636
7637+ /// Builds the arguments types vector for the given call instruction with the
7638+ /// given \p ID for the specified vector factor.
7639+ static SmallVector<Type *>
7640+ buildIntrinsicArgTypes(const CallInst *CI, const Intrinsic::ID ID,
7641+ const unsigned VF, unsigned MinBW,
7642+ const TargetTransformInfo *TTI) {
7643+ SmallVector<Type *> ArgTys;
7644+ for (auto [Idx, Arg] : enumerate(CI->args())) {
7645+ if (ID != Intrinsic::not_intrinsic) {
7646+ if (isVectorIntrinsicWithScalarOpAtArg(ID, Idx, TTI)) {
7647+ ArgTys.push_back(Arg->getType());
7648+ continue;
7649+ }
7650+ if (MinBW > 0) {
7651+ ArgTys.push_back(
7652+ getWidenedType(IntegerType::get(CI->getContext(), MinBW), VF));
7653+ continue;
7654+ }
7655+ }
7656+ ArgTys.push_back(getWidenedType(Arg->getType(), VF));
7657+ }
7658+ return ArgTys;
7659+ }
7660+
7661+ /// Calculates the costs of vectorized intrinsic (if possible) and vectorized
7662+ /// function (if possible) calls.
7663+ static std::pair<InstructionCost, InstructionCost>
7664+ getVectorCallCosts(CallInst *CI, FixedVectorType *VecTy,
7665+ TargetTransformInfo *TTI, TargetLibraryInfo *TLI,
7666+ ArrayRef<Type *> ArgTys) {
7667+ Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
7668+
7669+ // Calculate the cost of the scalar and vector calls.
7670+ FastMathFlags FMF;
7671+ if (auto *FPCI = dyn_cast<FPMathOperator>(CI))
7672+ FMF = FPCI->getFastMathFlags();
7673+ IntrinsicCostAttributes CostAttrs(ID, VecTy, ArgTys, FMF);
7674+ auto IntrinsicCost =
7675+ TTI->getIntrinsicInstrCost(CostAttrs, TTI::TCK_RecipThroughput);
7676+
7677+ auto Shape = VFShape::get(CI->getFunctionType(),
7678+ ElementCount::getFixed(VecTy->getNumElements()),
7679+ false /*HasGlobalPred*/);
7680+ Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
7681+ auto LibCost = IntrinsicCost;
7682+ if (!CI->isNoBuiltin() && VecFunc) {
7683+ // Calculate the cost of the vector library call.
7684+ // If the corresponding vector call is cheaper, return its cost.
7685+ LibCost =
7686+ TTI->getCallInstrCost(nullptr, VecTy, ArgTys, TTI::TCK_RecipThroughput);
7687+ }
7688+ return {IntrinsicCost, LibCost};
7689+ }
7690+
76377691BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
76387692 const InstructionsState &S, ArrayRef<Value *> VL,
76397693 bool IsScatterVectorizeUserTE, OrdersType &CurrentOrder,
@@ -9017,34 +9071,6 @@ bool BoUpSLP::areAllUsersVectorized(
90179071 });
90189072}
90199073
9020- static std::pair<InstructionCost, InstructionCost>
9021- getVectorCallCosts(CallInst *CI, FixedVectorType *VecTy,
9022- TargetTransformInfo *TTI, TargetLibraryInfo *TLI,
9023- ArrayRef<Type *> ArgTys) {
9024- Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
9025-
9026- // Calculate the cost of the scalar and vector calls.
9027- FastMathFlags FMF;
9028- if (auto *FPCI = dyn_cast<FPMathOperator>(CI))
9029- FMF = FPCI->getFastMathFlags();
9030- IntrinsicCostAttributes CostAttrs(ID, VecTy, ArgTys, FMF);
9031- auto IntrinsicCost =
9032- TTI->getIntrinsicInstrCost(CostAttrs, TTI::TCK_RecipThroughput);
9033-
9034- auto Shape = VFShape::get(CI->getFunctionType(),
9035- ElementCount::getFixed(VecTy->getNumElements()),
9036- false /*HasGlobalPred*/);
9037- Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
9038- auto LibCost = IntrinsicCost;
9039- if (!CI->isNoBuiltin() && VecFunc) {
9040- // Calculate the cost of the vector library call.
9041- // If the corresponding vector call is cheaper, return its cost.
9042- LibCost =
9043- TTI->getCallInstrCost(nullptr, VecTy, ArgTys, TTI::TCK_RecipThroughput);
9044- }
9045- return {IntrinsicCost, LibCost};
9046- }
9047-
90489074void BoUpSLP::TreeEntry::buildAltOpShuffleMask(
90499075 const function_ref<bool(Instruction *)> IsAltOp, SmallVectorImpl<int> &Mask,
90509076 SmallVectorImpl<Value *> *OpScalars,
@@ -11045,30 +11071,6 @@ TTI::CastContextHint BoUpSLP::getCastContextHint(const TreeEntry &TE) const {
1104511071 return TTI::CastContextHint::None;
1104611072}
1104711073
11048- /// Builds the arguments types vector for the given call instruction with the
11049- /// given \p ID for the specified vector factor.
11050- static SmallVector<Type *>
11051- buildIntrinsicArgTypes(const CallInst *CI, const Intrinsic::ID ID,
11052- const unsigned VF, unsigned MinBW,
11053- const TargetTransformInfo *TTI) {
11054- SmallVector<Type *> ArgTys;
11055- for (auto [Idx, Arg] : enumerate(CI->args())) {
11056- if (ID != Intrinsic::not_intrinsic) {
11057- if (isVectorIntrinsicWithScalarOpAtArg(ID, Idx, TTI)) {
11058- ArgTys.push_back(Arg->getType());
11059- continue;
11060- }
11061- if (MinBW > 0) {
11062- ArgTys.push_back(
11063- getWidenedType(IntegerType::get(CI->getContext(), MinBW), VF));
11064- continue;
11065- }
11066- }
11067- ArgTys.push_back(getWidenedType(Arg->getType(), VF));
11068- }
11069- return ArgTys;
11070- }
11071-
1107211074InstructionCost
1107311075BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
1107411076 SmallPtrSetImpl<Value *> &CheckedExtracts) {
0 commit comments