!fixup replace MultiUse kinds with boolean flag

fhahn · fhahn · commit 0cddbac88441 · 2025-10-26T18:03:13.000Z
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -42,13 +42,9 @@ enum class RecurKind {
   And,      ///< Bitwise or logical AND of integers.
   Xor,      ///< Bitwise or logical XOR of integers.
   SMin,     ///< Signed integer min implemented in terms of select(cmp()).
-  SMinMultiUse,     ///< Signed integer min implemented in terms of select(cmp()).
   SMax,     ///< Signed integer max implemented in terms of select(cmp()).
-  SMaxMultiUse,     ///< Signed integer max implemented in terms of select(cmp()).
   UMin,     ///< Unsigned integer min implemented in terms of select(cmp()).
-  UMinMultiUse,     ///< Unsigned integer min implemented in terms of select(cmp()).
   UMax,     ///< Unsigned integer max implemented in terms of select(cmp()).
-  UMaxMultiUse,     ///< Unsigned integer max implemented in terms of select(cmp()).
   FAdd,     ///< Sum of floats.
   FMul,     ///< Product of floats.
   FMin,     ///< FP min implemented in terms of select(cmp()).
@@ -99,12 +95,15 @@ class RecurrenceDescriptor {
                        RecurKind K, FastMathFlags FMF, Instruction *ExactFP,
                        Type *RT, bool Signed, bool Ordered,
                        SmallPtrSetImpl<Instruction *> &CI,
-                       unsigned MinWidthCastToRecurTy)
+                       unsigned MinWidthCastToRecurTy, bool PhiMultiUse = false)
       : IntermediateStore(Store), StartValue(Start), LoopExitInstr(Exit),
         Kind(K), FMF(FMF), ExactFPMathInst(ExactFP), RecurrenceType(RT),
-        IsSigned(Signed), IsOrdered(Ordered),
+        IsSigned(Signed), IsOrdered(Ordered), IsPhiMultiUse(PhiMultiUse),
         MinWidthCastToRecurrenceType(MinWidthCastToRecurTy) {
     CastInsts.insert_range(CI);
+    assert(
+        (!PhiMultiUse || isMinMaxRecurrenceKind(K)) &&
+        "Only min/max recurrences are allowed to have multiple uses currently");
   }
 
   /// This POD struct holds information about a potential recurrence operation.
@@ -251,26 +250,8 @@ class RecurrenceDescriptor {
 
   /// Returns true if the recurrence kind is an integer min/max kind.
   static bool isIntMinMaxRecurrenceKind(RecurKind Kind) {
-    return Kind == RecurKind::UMin || Kind == RecurKind::UMinMultiUse ||
-           Kind == RecurKind::UMax || Kind == RecurKind::UMaxMultiUse ||
-           Kind == RecurKind::SMin || Kind == RecurKind::SMinMultiUse ||
-           Kind == RecurKind::SMax || Kind == RecurKind::SMaxMultiUse;
-  }
-
-  static RecurKind convertFromMultiUseKind(RecurKind Kind) {
-    switch (Kind) {
-    case RecurKind::UMaxMultiUse:
-      return RecurKind::UMax;
-    case RecurKind::UMinMultiUse:
-      return RecurKind::UMin;
-    case RecurKind::SMinMultiUse:
-      return RecurKind::SMin;
-    case RecurKind::SMaxMultiUse:
-      return RecurKind::SMax;
-    default:
-      return Kind;
-    }
-    llvm_unreachable("all cases must be handled above");
+    return Kind == RecurKind::UMin || Kind == RecurKind::UMax ||
+           Kind == RecurKind::SMin || Kind == RecurKind::SMax;
   }
 
   /// Returns true if the recurrence kind is a floating-point minnum/maxnum
@@ -361,6 +342,10 @@ class RecurrenceDescriptor {
   /// Expose an ordered FP reduction to the instance users.
   bool isOrdered() const { return IsOrdered; }
 
+  /// Returns true if the reduction PHI has multiple in-loop users. This is
+  /// relevant for min/max reductions that are part of a FindLastIV pattern.
+  bool isPhiMultiUse() const { return IsPhiMultiUse; }
+
   /// Attempts to find a chain of operations from Phi to LoopExitInst that can
   /// be treated as a set of reductions instructions for in-loop reductions.
   LLVM_ABI SmallVector<Instruction *, 4> getReductionOpChain(PHINode *Phi,
@@ -398,6 +383,9 @@ class RecurrenceDescriptor {
   // Currently only a non-reassociative FAdd can be considered in-order,
   // if it is also the only FAdd in the PHI's use chain.
   bool IsOrdered = false;
+  // True if the reduction PHI has multiple in-loop users. This is relevant
+  // for min/max reductions that are part of a FindLastIV pattern.
+  bool IsPhiMultiUse = false;
   // Instructions used for type-promoting the recurrence.
   SmallPtrSet<Instruction *, 8> CastInsts;
   // The minimum width used by the recurrence.
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -227,25 +227,24 @@ getMultiUseMinMax(PHINode *Phi, RecurKind Kind, Loop *TheLoop) {
 
   Value *A, *B;
   if (match(Inc, m_OneUse(m_UMin(m_Value(A), m_Value(B)))))
-    RK = RecurKind::UMinMultiUse;
+    RK = RecurKind::UMin;
   else if (match(Inc, m_OneUse(m_UMax(m_Value(A), m_Value(B)))))
-    RK = RecurKind::UMaxMultiUse;
+    RK = RecurKind::UMax;
   else if (match(Inc, m_OneUse(m_SMax(m_Value(A), m_Value(B)))))
-    RK = RecurKind::SMaxMultiUse;
+    RK = RecurKind::SMax;
   else if (match(Inc, m_OneUse(m_SMin(m_Value(A), m_Value(B)))))
-    RK = RecurKind::SMinMultiUse;
+    RK = RecurKind::SMin;
   else
     return std::nullopt;
 
-  if (RecurrenceDescriptor::convertFromMultiUseKind(RK) != Kind || A == B ||
-      (A != Phi && B != Phi))
+  if (A == B || (A != Phi && B != Phi))
     return std::nullopt;
 
   SmallPtrSet<Instruction *, 4> CastInsts;
   Value *RdxStart = Phi->getIncomingValueForBlock(TheLoop->getLoopPreheader());
   RecurrenceDescriptor RD(RdxStart, nullptr, nullptr, RK, FastMathFlags(),
-                          nullptr, Phi->getType(), false, false, CastInsts,
-                          -1U);
+                          nullptr, Phi->getType(), false, false, CastInsts, -1U,
+                          true);
   return {RD};
 }
 
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -1217,7 +1217,9 @@ llvm::canParallelizeReductionWhenUnrolling(PHINode &Phi, Loop *L,
   RecurrenceDescriptor RdxDesc;
   if (!RecurrenceDescriptor::isReductionPHI(&Phi, L, RdxDesc,
                                             /*DemandedBits=*/nullptr,
-                                            /*AC=*/nullptr, /*DT=*/nullptr, SE))
+                                            /*AC=*/nullptr, /*DT=*/nullptr,
+                                            SE) ||
+      RdxDesc.isPhiMultiUse())
     return std::nullopt;
   RecurKind RK = RdxDesc.getRecurrenceKind();
   // Skip unsupported reductions.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -878,6 +878,11 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
       Requirements->addExactFPMathInst(RedDes.getExactFPMathInst());
       AllowedExit.insert(RedDes.getLoopExitInstr());
       Reductions[Phi] = RedDes;
+      assert((!RedDes.isPhiMultiUse() ||
+              RecurrenceDescriptor::isMinMaxRecurrenceKind(
+                  RedDes.getRecurrenceKind())) &&
+             "Only min/max recurrences are allowed to have multiple uses "
+             "currently");
       return true;
     }
 
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6515,6 +6515,11 @@ void LoopVectorizationCostModel::collectInLoopReductions() {
     PHINode *Phi = Reduction.first;
     const RecurrenceDescriptor &RdxDesc = Reduction.second;
 
+    // Multi-use reductions (e.g., used in FindLastIV patterns) are handled
+    // separately and should not be considered for in-loop reductions.
+    if (RdxDesc.isPhiMultiUse())
+      continue;
+
     // We don't collect reductions that are type promoted (yet).
     if (RdxDesc.getRecurrenceType() != Phi->getType())
       continue;
@@ -8072,7 +8077,8 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
           getScalingForReduction(RdxDesc.getLoopExitInstr()).value_or(1);
       PhiRecipe = new VPReductionPHIRecipe(
           Phi, RdxDesc.getRecurrenceKind(), *StartV, CM.isInLoopReduction(Phi),
-          CM.useOrderedReductions(RdxDesc), ScaleFactor);
+          CM.useOrderedReductions(RdxDesc), ScaleFactor,
+          RdxDesc.isPhiMultiUse());
     } else if (Legal->isFixedOrderRecurrence(Phi)) {
       // TODO: Currently fixed-order recurrences are modeled as chains of
       // first-order recurrences. If there are no users of the intermediate
@@ -8429,8 +8435,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
   // Adjust the recipes for any inloop reductions.
   adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
 
-  // Try to convert remaining VPWidenPHIRecipes to reduction recipes.
-  if (!VPlanTransforms::runPass(VPlanTransforms::legalizeUnclassifiedPhis,
+  // Try to legalize reductions with multiple in-loop uses.
+  if (!VPlanTransforms::runPass(VPlanTransforms::legalizeMultiUseReductions,
                                 *Plan))
     return nullptr;
   // Apply mandatory transformation to handle FP maxnum/minnum reduction with
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -25091,10 +25091,6 @@ class HorizontalReduction {
         case RecurKind::FMinNum:
         case RecurKind::FMaximumNum:
         case RecurKind::FMinimumNum:
-        case RecurKind::UMaxMultiUse:
-        case RecurKind::UMinMultiUse:
-        case RecurKind::SMinMultiUse:
-        case RecurKind::SMaxMultiUse:
         case RecurKind::None:
           llvm_unreachable("Unexpected reduction kind for repeated scalar.");
         }
@@ -25236,10 +25232,6 @@ class HorizontalReduction {
     case RecurKind::FMinNum:
     case RecurKind::FMaximumNum:
     case RecurKind::FMinimumNum:
-    case RecurKind::UMaxMultiUse:
-    case RecurKind::UMinMultiUse:
-    case RecurKind::SMinMultiUse:
-    case RecurKind::SMaxMultiUse:
     case RecurKind::None:
       llvm_unreachable("Unexpected reduction kind for repeated scalar.");
     }
@@ -25346,10 +25338,6 @@ class HorizontalReduction {
     case RecurKind::FMinNum:
     case RecurKind::FMaximumNum:
     case RecurKind::FMinimumNum:
-    case RecurKind::UMaxMultiUse:
-    case RecurKind::UMinMultiUse:
-    case RecurKind::SMinMultiUse:
-    case RecurKind::SMaxMultiUse:
     case RecurKind::None:
       llvm_unreachable("Unexpected reduction kind for reused scalars.");
     }
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2346,6 +2346,10 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
   /// The phi is part of an ordered reduction. Requires IsInLoop to be true.
   bool IsOrdered;
 
+  /// The phi is part of a multi-use reduction (e.g., used in FindLastIV
+  /// patterns).
+  bool IsPhiMultiUse;
+
   /// When expanding the reduction PHI, the plan's VF element count is divided
   /// by this factor to form the reduction phi's VF.
   unsigned VFScaleFactor = 1;
@@ -2354,9 +2358,10 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
   /// Create a new VPReductionPHIRecipe for the reduction \p Phi.
   VPReductionPHIRecipe(PHINode *Phi, RecurKind Kind, VPValue &Start,
                        bool IsInLoop = false, bool IsOrdered = false,
-                       unsigned VFScaleFactor = 1)
+                       unsigned VFScaleFactor = 1, bool IsPhiMultiUse = false)
       : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start), Kind(Kind),
-        IsInLoop(IsInLoop), IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) {
+        IsInLoop(IsInLoop), IsOrdered(IsOrdered), IsPhiMultiUse(IsPhiMultiUse),
+        VFScaleFactor(VFScaleFactor) {
     assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
   }
 
@@ -2365,7 +2370,7 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
   VPReductionPHIRecipe *clone() override {
     auto *R = new VPReductionPHIRecipe(
         dyn_cast_or_null<PHINode>(getUnderlyingValue()), getRecurrenceKind(),
-        *getOperand(0), IsInLoop, IsOrdered, VFScaleFactor);
+        *getOperand(0), IsInLoop, IsOrdered, VFScaleFactor, IsPhiMultiUse);
     R->addOperand(getBackedgeValue());
     return R;
   }
@@ -2398,6 +2403,9 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
   /// Returns true, if the phi is part of an in-loop reduction.
   bool isInLoop() const { return IsInLoop; }
 
+  /// Returns true, if the phi is part of a multi-use reduction.
+  bool isPhiMultiUse() const { return IsPhiMultiUse; }
+
   /// Returns true if the recipe only uses the first lane of operand \p Op.
   bool onlyFirstLaneUsed(const VPValue *Op) const override {
     assert(is_contained(operands(), Op) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -950,23 +950,25 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
   return true;
 }
 
-bool VPlanTransforms::legalizeUnclassifiedPhis(VPlan &Plan) {
-  using namespace VPlanPatternMatch;
+bool VPlanTransforms::legalizeMultiUseReductions(VPlan &Plan) {
   for (auto &PhiR : make_early_inc_range(
            Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
     auto *MinMaxPhiR = dyn_cast<VPReductionPHIRecipe>(&PhiR);
-    if (!MinMaxPhiR || !RecurrenceDescriptor::isIntMinMaxRecurrenceKind(
-                           MinMaxPhiR->getRecurrenceKind()))
+    if (!MinMaxPhiR)
       continue;
 
-    RecurKind RdxKind = RecurrenceDescriptor::convertFromMultiUseKind(
-        MinMaxPhiR->getRecurrenceKind());
-    if (RdxKind == MinMaxPhiR->getRecurrenceKind())
+    RecurKind RdxKind = MinMaxPhiR->getRecurrenceKind();
+    // TODO: check for multi-uses in VPlan directly.
+    if (!RecurrenceDescriptor::isIntMinMaxRecurrenceKind(RdxKind) ||
+        !MinMaxPhiR->isPhiMultiUse())
       continue;
 
     // One user of MinMaxPhiR is MinMaxOp, the other users must be a compare
     // that's part of a FindLastIV chain.
-    auto *MinMaxOp = cast<VPRecipeWithIRFlags>(MinMaxPhiR->getBackedgeValue());
+    auto *MinMaxOp =
+        dyn_cast<VPRecipeWithIRFlags>(MinMaxPhiR->getBackedgeValue());
+    if (!MinMaxOp)
+      return false;
     auto MinMaxUsers = to_vector(MinMaxPhiR->users());
     auto *Cmp = dyn_cast<VPRecipeWithIRFlags>(
         MinMaxUsers[0] == MinMaxOp ? MinMaxUsers[1] : MinMaxUsers[0]);
@@ -1019,28 +1021,22 @@ bool VPlanTransforms::legalizeUnclassifiedPhis(VPlan &Plan) {
     assert(!FindIVPhiR->isInLoop() && !FindIVPhiR->isOrdered() &&
            "cannot handle inloop/ordered reductions yet");
 
-    auto NewPhiR = new VPReductionPHIRecipe(
-        cast<PHINode>(MinMaxPhiR->getUnderlyingInstr()), RdxKind,
-        *MinMaxPhiR->getOperand(0), false, false, 1);
-    NewPhiR->insertBefore(MinMaxPhiR);
-    MinMaxPhiR->replaceAllUsesWith(NewPhiR);
-    NewPhiR->addOperand(MinMaxPhiR->getOperand(1));
-    MinMaxPhiR->eraseFromParent();
-
     // The reduction using MinMaxPhiR needs adjusting to compute the correct
     // result:
     //  1. We need to find the last IV for which the condition based on the
     //  min/max recurrence is true,
     //  2. Compare the partial min/max reduction result to its final value and,
     //  3. Select the lanes of the partial FindLastIV reductions which
     //  correspond to the lanes matching the min/max reduction result.
-    VPInstruction *FindIVResult = cast<VPInstruction>(
+    VPInstruction *FindIVResult = dyn_cast<VPInstruction>(
         *(Sel->user_begin() + (*Sel->user_begin() == FindIVPhiR ? 1 : 0)));
+    if (!FindIVResult)
+      return false;
     VPBuilder B(FindIVResult);
-    VPInstruction *MinMaxResult =
-        B.createNaryOp(VPInstruction::ComputeReductionResult,
-                       {NewPhiR, NewPhiR->getBackedgeValue()}, VPIRFlags(), {});
-    NewPhiR->getBackedgeValue()->replaceUsesWithIf(
+    VPInstruction *MinMaxResult = B.createNaryOp(
+        VPInstruction::ComputeReductionResult,
+        {MinMaxPhiR, MinMaxPhiR->getBackedgeValue()}, VPIRFlags(), {});
+    MinMaxPhiR->getBackedgeValue()->replaceUsesWithIf(
         MinMaxResult, [](VPUser &U, unsigned) { return isa<VPPhi>(&U); });
     auto *FinalMinMaxCmp = B.createICmp(
         CmpInst::ICMP_EQ, MinMaxResult->getOperand(1), MinMaxResult);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -143,10 +143,10 @@ struct VPlanTransforms {
           GetIntOrFpInductionDescriptor,
       const TargetLibraryInfo &TLI);
 
-  /// Try to legalize unclassified phis by converting VPWidenPHIRecipes to
-  /// min-max reductions used by FindLastIV reductions if possible. Returns
-  /// false if the VPlan contains VPWidenPHIRecipes that cannot be legalized.
-  static bool legalizeUnclassifiedPhis(VPlan &Plan);
+  /// Try to legalize reductions with multiple in-loop uses. Currently only
+  /// min/max reductions used by FindLastIV reductions are supported. Otherwise
+  /// return false.
+  static bool legalizeMultiUseReductions(VPlan &Plan);
 
   /// Try to have all users of fixed-order recurrences appear after the recipe
   /// defining their previous value, by either sinking users or hoisting recipes
