@@ -3835,24 +3835,17 @@ void VPFirstOrderRecurrencePHIRecipe::print(raw_ostream &O, const Twine &Indent,
38353835#endif
38363836
38373837void VPReductionPHIRecipe::execute (VPTransformState &State) {
3838- auto &Builder = State.Builder ;
3839-
38403838 // If this phi is fed by a scaled reduction then it should output a
38413839 // vector with fewer elements than the VF.
38423840 ElementCount VF = State.VF .divideCoefficientBy (VFScaleFactor);
38433841
3844- // Reductions do not have to start at zero. They can start with
3845- // any loop invariant values.
3846- VPValue *StartVPV = getStartValue ();
3847- Value *StartV = StartVPV->getLiveInIRValue ();
3848-
38493842 // In order to support recurrences we need to be able to vectorize Phi nodes.
38503843 // Phi nodes have cycles, so we need to vectorize them in two stages. This is
38513844 // stage #1: We create a new vector PHI node with no incoming edges. We'll use
38523845 // this value when we vectorize all of the instructions that use the PHI.
3846+ auto *ScalarTy = State.TypeAnalysis .inferScalarType (this );
38533847 bool ScalarPHI = State.VF .isScalar () || IsInLoop;
3854- Type *VecTy =
3855- ScalarPHI ? StartV->getType () : VectorType::get (StartV->getType (), VF);
3848+ Type *VecTy = ScalarPHI ? ScalarTy : VectorType::get (ScalarTy, VF);
38563849
38573850 BasicBlock *HeaderBB = State.CFG .PrevBB ;
38583851 assert (State.CurrentParentLoop ->getHeader () == HeaderBB &&
@@ -3863,22 +3856,18 @@ void VPReductionPHIRecipe::execute(VPTransformState &State) {
38633856
38643857 BasicBlock *VectorPH =
38653858 State.CFG .VPBB2IRBB .at (getParent ()->getCFGPredecessor (0 ));
3859+ // Create start and identity vector values for the reduction in the preheader.
3860+ // TODO: Introduce recipes in VPlan preheader to create initial values.
3861+ IRBuilderBase::InsertPointGuard IPBuilder (State.Builder );
3862+ State.Builder .SetInsertPoint (VectorPH->getTerminator ());
38663863
3867- Value *Iden = nullptr ;
3864+ // Reductions do not have to start at zero. They can start with
3865+ // any loop invariant values.
3866+ VPValue *StartVPV = getStartValue ();
38683867 RecurKind RK = RdxDesc.getRecurrenceKind ();
3869- unsigned CurrentPart = getUnrollPart (*this );
3870-
38713868 if (RecurrenceDescriptor::isMinMaxRecurrenceKind (RK) ||
3872- RecurrenceDescriptor::isAnyOfRecurrenceKind (RK)) {
3873- // MinMax and AnyOf reductions have the start value as their identity.
3874- if (ScalarPHI) {
3875- Iden = StartV;
3876- } else {
3877- IRBuilderBase::InsertPointGuard IPBuilder (Builder);
3878- Builder.SetInsertPoint (VectorPH->getTerminator ());
3879- StartV = Iden = State.get (StartVPV);
3880- }
3881- } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind (RK)) {
3869+ RecurrenceDescriptor::isAnyOfRecurrenceKind (RK) ||
3870+ RecurrenceDescriptor::isFindLastIVRecurrenceKind (RK)) {
38823871 // [I|F]FindLastIV will use a sentinel value to initialize the reduction
38833872 // phi or the resume value from the main vector loop when vectorizing the
38843873 // epilogue loop. In the exit block, ComputeReductionResult will generate
@@ -3888,33 +3877,24 @@ void VPReductionPHIRecipe::execute(VPTransformState &State) {
38883877 // TODO: The sentinel value is not always necessary. When the start value is
38893878 // a constant, and smaller than the start value of the induction variable,
38903879 // the start value can be directly used to initialize the reduction phi.
3891- Iden = StartV;
3892- if (!ScalarPHI) {
3893- IRBuilderBase::InsertPointGuard IPBuilder (Builder);
3894- Builder.SetInsertPoint (VectorPH->getTerminator ());
3895- StartV = Iden = Builder.CreateVectorSplat (State.VF , Iden);
3896- }
3897- } else {
3898- Iden = llvm::getRecurrenceIdentity (RK, VecTy->getScalarType (),
3899- RdxDesc.getFastMathFlags ());
3900-
3901- if (!ScalarPHI) {
3902- if (CurrentPart == 0 ) {
3903- // Create start and identity vector values for the reduction in the
3904- // preheader.
3905- // TODO: Introduce recipes in VPlan preheader to create initial values.
3906- Iden = Builder.CreateVectorSplat (VF, Iden);
3907- IRBuilderBase::InsertPointGuard IPBuilder (Builder);
3908- Builder.SetInsertPoint (VectorPH->getTerminator ());
3909- Constant *Zero = Builder.getInt32 (0 );
3910- StartV = Builder.CreateInsertElement (Iden, StartV, Zero);
3911- } else {
3912- Iden = Builder.CreateVectorSplat (VF, Iden);
3913- }
3880+ Phi->addIncoming (State.get (StartVPV, ScalarPHI), VectorPH);
3881+ return ;
3882+ }
3883+
3884+ Value *Iden = getRecurrenceIdentity (RK, VecTy->getScalarType (),
3885+ RdxDesc.getFastMathFlags ());
3886+ unsigned CurrentPart = getUnrollPart (*this );
3887+ Value *StartV = StartVPV->getLiveInIRValue ();
3888+ if (!ScalarPHI) {
3889+ if (CurrentPart == 0 ) {
3890+ Iden = State.Builder .CreateVectorSplat (VF, Iden);
3891+ Constant *Zero = State.Builder .getInt32 (0 );
3892+ StartV = State.Builder .CreateInsertElement (Iden, StartV, Zero);
3893+ } else {
3894+ Iden = State.Builder .CreateVectorSplat (VF, Iden);
39143895 }
39153896 }
39163897
3917- Phi = cast<PHINode>(State.get (this , IsInLoop));
39183898 Value *StartVal = (CurrentPart == 0 ) ? StartV : Iden;
39193899 Phi->addIncoming (StartVal, VectorPH);
39203900}
0 commit comments