Consolidate all IR logic for getting the identity value of a reduction [nfc]

This change merges the three different places (at the IR layer) for
finding the identity value of a reduction into a single copy.  This
depends on several prior commits which fix ommissions and bugs in
the distinct copies, but this patch itself should be fully
non-functional.

As the new comments and naming try to make clear, the identity value
is a property of the @llvm.vector.reduce.* intrinsic, not of e.g.
the recurrence descriptor.  (We still provide an interface for
clients using recurrence descriptors, but the implementation simply
translates to the intrinsic which each corresponds to.)

As a note, the getIntrinsicIdentity API does not support fminnum/fmaxnum
or fminimum/fmaximum which is why we still need manual logic (but at
least only one copy of manual logic) for those cases.

Author: preames

llvm/include/llvm/Analysis/IVDescriptors.h

@@ -155,9 +155,6 @@ class RecurrenceDescriptor {
   /// Select(FCmp(X, Y), (Z = X op PHINode), PHINode) instruction pattern.
   static InstDesc isConditionalRdxPattern(RecurKind Kind, Instruction *I);
 
-  /// Returns identity corresponding to the RecurrenceKind.
-  static Value *getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF);
-
   /// Returns the opcode corresponding to the RecurrenceKind.
   static unsigned getOpcode(RecurKind Kind);
 

llvm/include/llvm/Transforms/Utils/LoopUtils.h

@@ -378,6 +378,14 @@ RecurKind getMinMaxReductionRecurKind(Intrinsic::ID RdxID);
 /// Returns the comparison predicate used when expanding a min/max reduction.
 CmpInst::Predicate getMinMaxReductionPredicate(RecurKind RK);
 
+/// Given information about an @llvm.vector.reduce.* intrinsic, return
+/// the identity value for the reduction.
+Value *getReductionIdentity(Intrinsic::ID RdxID, Type *Ty, FastMathFlags FMF);
+
+/// Given information about an recurrence kind, return the identity
+/// for the @llvm.vector.reduce.* used to generate it.
+Value *getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF);
+
 /// Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
 /// The Builder's fast-math-flags must be set to propagate the expected values.
 Value *createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,

llvm/lib/Analysis/IVDescriptors.cpp

@@ -1031,52 +1031,6 @@ bool RecurrenceDescriptor::isFixedOrderRecurrence(PHINode *Phi, Loop *TheLoop,
   return true;
 }
 
-/// This function returns the identity element (or neutral element) for
-/// the operation K.
-Value *RecurrenceDescriptor::getRecurrenceIdentity(RecurKind K, Type *Tp,
-                                                   FastMathFlags FMF) {
-  switch (K) {
-  case RecurKind::Xor:
-  case RecurKind::Add:
-  case RecurKind::Or:
-  case RecurKind::Mul:
-  case RecurKind::And:
-  case RecurKind::FMul:
-  case RecurKind::FAdd:
-    return ConstantExpr::getBinOpIdentity(getOpcode(K), Tp, false, FMF.noSignedZeros());
-  case RecurKind::FMulAdd:
-    return ConstantExpr::getBinOpIdentity(Instruction::FAdd, Tp, false, FMF.noSignedZeros());
-  case RecurKind::UMin:
-    return ConstantInt::get(Tp, -1, true);
-  case RecurKind::UMax:
-    return ConstantInt::get(Tp, 0);
-  case RecurKind::SMin:
-    return ConstantInt::get(Tp,
-                            APInt::getSignedMaxValue(Tp->getIntegerBitWidth()));
-  case RecurKind::SMax:
-    return ConstantInt::get(Tp,
-                            APInt::getSignedMinValue(Tp->getIntegerBitWidth()));
-  case RecurKind::FMin:
-  case RecurKind::FMax:
-    assert((FMF.noNaNs() && FMF.noSignedZeros()) &&
-           "nnan, nsz is expected to be set for FP min/max reduction.");
-    [[fallthrough]];
-  case RecurKind::FMinimum:
-  case RecurKind::FMaximum: {
-    bool Negative = K == RecurKind::FMax || K == RecurKind::FMaximum;
-    const fltSemantics &Semantics = Tp->getFltSemantics();
-    return !FMF.noInfs()
-               ? ConstantFP::getInfinity(Tp, Negative)
-               : ConstantFP::get(Tp, APFloat::getLargest(Semantics, Negative));
-  }
-  case RecurKind::IAnyOf:
-  case RecurKind::FAnyOf:
-    llvm_unreachable("No meaningful identity for recurrence kind");
-  default:
-    llvm_unreachable("Unknown recurrence kind");
-  }
-}
-
 unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
   switch (Kind) {
   case RecurKind::Add:

llvm/lib/CodeGen/ExpandVectorPredication.cpp

@@ -368,52 +368,11 @@ Value *CachingVPExpander::expandPredicationToFPCall(
 
 static Value *getNeutralReductionElement(const VPReductionIntrinsic &VPI,
                                          Type *EltTy) {
-  bool Negative = false;
-  unsigned EltBits = EltTy->getScalarSizeInBits();
-  Intrinsic::ID VID = VPI.getIntrinsicID();
-  switch (VID) {
-  default:
-    llvm_unreachable("Expecting a VP reduction intrinsic");
-  case Intrinsic::vp_reduce_add:
-  case Intrinsic::vp_reduce_or:
-  case Intrinsic::vp_reduce_xor:
-  case Intrinsic::vp_reduce_umax:
-    return Constant::getNullValue(EltTy);
-  case Intrinsic::vp_reduce_mul:
-    return ConstantInt::get(EltTy, 1, /*IsSigned*/ false);
-  case Intrinsic::vp_reduce_and:
-  case Intrinsic::vp_reduce_umin:
-    return ConstantInt::getAllOnesValue(EltTy);
-  case Intrinsic::vp_reduce_smin:
-    return ConstantInt::get(EltTy->getContext(),
-                            APInt::getSignedMaxValue(EltBits));
-  case Intrinsic::vp_reduce_smax:
-    return ConstantInt::get(EltTy->getContext(),
-                            APInt::getSignedMinValue(EltBits));
-  case Intrinsic::vp_reduce_fmax:
-  case Intrinsic::vp_reduce_fmaximum:
-    Negative = true;
-    [[fallthrough]];
-  case Intrinsic::vp_reduce_fmin:
-  case Intrinsic::vp_reduce_fminimum: {
-    bool PropagatesNaN = VID == Intrinsic::vp_reduce_fminimum ||
-                         VID == Intrinsic::vp_reduce_fmaximum;
-    FastMathFlags Flags = VPI.getFastMathFlags();
-    const fltSemantics &Semantics = EltTy->getFltSemantics();
-    return (!Flags.noNaNs() && !PropagatesNaN)
-               ? ConstantFP::getQNaN(EltTy, Negative)
-           : !Flags.noInfs()
-               ? ConstantFP::getInfinity(EltTy, Negative)
-               : ConstantFP::get(EltTy,
-                                 APFloat::getLargest(Semantics, Negative));
-  }
-  case Intrinsic::vp_reduce_fadd:
-    return ConstantExpr::getBinOpIdentity(
-        Instruction::FAdd, EltTy, false,
-        VPI.getFastMathFlags().noSignedZeros());
-  case Intrinsic::vp_reduce_fmul:
-    return ConstantFP::get(EltTy, 1.0);
-  }
+  Intrinsic::ID RdxID = *VPI.getFunctionalIntrinsicID();
+  FastMathFlags FMF;
+  if (isa<FPMathOperator>(VPI))
+    FMF = VPI.getFastMathFlags();
+  return getReductionIdentity(RdxID, EltTy, FMF);
 }
 
 Value *

llvm/lib/Transforms/Utils/LoopUtils.cpp

@@ -1207,14 +1207,62 @@ Value *llvm::createAnyOfReduction(IRBuilderBase &Builder, Value *Src,
   return Builder.CreateSelect(AnyOf, NewVal, InitVal, "rdx.select");
 }
 
+Value *llvm::getReductionIdentity(Intrinsic::ID RdxID, Type *Ty,
+                                  FastMathFlags Flags) {
+  bool Negative = false;
+  switch (RdxID) {
+  default:
+    llvm_unreachable("Expecting a reduction intrinsic");
+  case Intrinsic::vector_reduce_add:
+  case Intrinsic::vector_reduce_mul:
+  case Intrinsic::vector_reduce_or:
+  case Intrinsic::vector_reduce_xor:
+  case Intrinsic::vector_reduce_and:
+  case Intrinsic::vector_reduce_fadd:
+  case Intrinsic::vector_reduce_fmul: {
+    unsigned Opc = getArithmeticReductionInstruction(RdxID);
+    return ConstantExpr::getBinOpIdentity(Opc, Ty, false,
+                                          Flags.noSignedZeros());
+  }
+  case Intrinsic::vector_reduce_umax:
+  case Intrinsic::vector_reduce_umin:
+  case Intrinsic::vector_reduce_smin:
+  case Intrinsic::vector_reduce_smax: {
+    Intrinsic::ID ScalarID = getMinMaxReductionIntrinsicOp(RdxID);
+    return ConstantExpr::getIntrinsicIdentity(ScalarID, Ty);
+  }
+  case Intrinsic::vector_reduce_fmax:
+  case Intrinsic::vector_reduce_fmaximum:
+    Negative = true;
+    [[fallthrough]];
+  case Intrinsic::vector_reduce_fmin:
+  case Intrinsic::vector_reduce_fminimum: {
+    bool PropagatesNaN = RdxID == Intrinsic::vector_reduce_fminimum ||
+                         RdxID == Intrinsic::vector_reduce_fmaximum;
+    const fltSemantics &Semantics = Ty->getFltSemantics();
+    return (!Flags.noNaNs() && !PropagatesNaN)
+               ? ConstantFP::getQNaN(Ty, Negative)
+           : !Flags.noInfs()
+               ? ConstantFP::getInfinity(Ty, Negative)
+               : ConstantFP::get(Ty, APFloat::getLargest(Semantics, Negative));
+  }
+  }
+}
+
+Value *llvm::getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF) {
+  assert((!(K == RecurKind::FMin || K == RecurKind::FMax) ||
+          (FMF.noNaNs() && FMF.noSignedZeros())) &&
+         "nnan, nsz is expected to be set for FP min/max reduction.");
+  Intrinsic::ID RdxID = getReductionIntrinsicID(K);
+  return getReductionIdentity(RdxID, Tp, FMF);
+}
+
 Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
                                    RecurKind RdxKind) {
   auto *SrcVecEltTy = cast<VectorType>(Src->getType())->getElementType();
   auto getIdentity = [&]() {
-    Intrinsic::ID ID = getReductionIntrinsicID(RdxKind);
-    unsigned Opc = getArithmeticReductionInstruction(ID);
-    bool NSZ = Builder.getFastMathFlags().noSignedZeros();
-    return ConstantExpr::getBinOpIdentity(Opc, SrcVecEltTy, false, NSZ);
+    return getRecurrenceIdentity(RdxKind, SrcVecEltTy,
+                                 Builder.getFastMathFlags());
   };
   switch (RdxKind) {
   case RecurKind::Add:
@@ -1249,8 +1297,7 @@ Value *llvm::createSimpleReduction(VectorBuilder &VBuilder, Value *Src,
   Intrinsic::ID Id = getReductionIntrinsicID(Kind);
   auto *SrcTy = cast<VectorType>(Src->getType());
   Type *SrcEltTy = SrcTy->getElementType();
-  Value *Iden =
-      Desc.getRecurrenceIdentity(Kind, SrcEltTy, Desc.getFastMathFlags());
+  Value *Iden = getRecurrenceIdentity(Kind, SrcEltTy, Desc.getFastMathFlags());
   Value *Ops[] = {Iden, Src};
   return VBuilder.createSimpleReduction(Id, SrcTy, Ops);
 }

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -1840,8 +1840,8 @@ void VPReductionRecipe::execute(VPTransformState &State) {
       if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind))
         Start = RdxDesc.getRecurrenceStartValue();
       else
-        Start = RdxDesc.getRecurrenceIdentity(Kind, ElementTy,
-                                              RdxDesc.getFastMathFlags());
+        Start = llvm::getRecurrenceIdentity(Kind, ElementTy,
+                                            RdxDesc.getFastMathFlags());
       if (State.VF.isVector())
         Start = State.Builder.CreateVectorSplat(VecTy->getElementCount(),
                                                 Start);
@@ -3010,8 +3010,8 @@ void VPReductionPHIRecipe::execute(VPTransformState &State) {
           Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident");
     }
   } else {
-    Iden = RdxDesc.getRecurrenceIdentity(RK, VecTy->getScalarType(),
-                                         RdxDesc.getFastMathFlags());
+    Iden = llvm::getRecurrenceIdentity(RK, VecTy->getScalarType(),
+                                       RdxDesc.getFastMathFlags());
 
     if (!ScalarPHI) {
       Iden = Builder.CreateVectorSplat(State.VF, Iden);

llvm/unittests/Analysis/IVDescriptorsTest.cpp

@@ -209,10 +209,6 @@ for.end:
         EXPECT_TRUE(IsRdxPhi);
         RecurKind Kind = Rdx.getRecurrenceKind();
         EXPECT_EQ(Kind, RecurKind::FMin);
-        Type *Ty = Phi->getType();
-        Value *Id = Rdx.getRecurrenceIdentity(Kind, Ty, Rdx.getFastMathFlags());
-        // Identity value for FP min reduction is +Inf.
-        EXPECT_EQ(Id, ConstantFP::getInfinity(Ty, false /*Negative*/));
       });
 }
 
@@ -261,9 +257,5 @@ for.end:
         EXPECT_TRUE(IsRdxPhi);
         RecurKind Kind = Rdx.getRecurrenceKind();
         EXPECT_EQ(Kind, RecurKind::FMax);
-        Type *Ty = Phi->getType();
-        Value *Id = Rdx.getRecurrenceIdentity(Kind, Ty, Rdx.getFastMathFlags());
-        // Identity value for FP max reduction is -Inf.
-        EXPECT_EQ(Id, ConstantFP::getInfinity(Ty, true /*Negative*/));
       });
 }