!fixup updates after merging

Author: fhahn

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -7170,7 +7170,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   // TODO: Move to VPlan transform stage once the transition to the VPlan-based
   // cost model is complete for better cost estimates.
   VPlanTransforms::runPass(VPlanTransforms::unrollByUF, BestVPlan, BestUF);
-  VPlanTransforms::runPass(VPlanTransforms::materializeBuildVectors, BestVPlan);
+  VPlanTransforms::runPass(VPlanTransforms::materializeBuildAndUnpackVectors,
+                           BestVPlan);
   VPlanTransforms::runPass(VPlanTransforms::materializeBroadcasts, BestVPlan);
   VPlanTransforms::runPass(VPlanTransforms::replicateByVF, BestVPlan, BestVF);
   bool HasBranchWeights =

llvm/lib/Transforms/Vectorize/VPlan.h

@@ -1060,7 +1060,8 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
     ResumeForEpilogue,
     /// Returns the value for vscale.
     VScale,
-    Unpack,
+    /// Extracts all lanes from its (non-scalable) vector operand.
+    UnpackVector,
   };
 
   /// Returns true if this VPInstruction generates scalar values for all lanes.
@@ -2704,6 +2705,15 @@ class LLVM_ABI_FOR_TEST VPReductionRecipe : public VPRecipeWithIRFlags {
     return R && classof(R);
   }
 
+  static inline bool classof(const VPValue *VPV) {
+    const VPRecipeBase *R = VPV->getDefiningRecipe();
+    return R && classof(R);
+  }
+
+  static inline bool classof(const VPSingleDefRecipe *R) {
+    return classof(static_cast<const VPRecipeBase *>(R));
+  }
+
   /// Generate the reduction in the loop.
   void execute(VPTransformState &State) override;
 

llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp

@@ -109,7 +109,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
   case VPInstruction::AnyOf:
   case VPInstruction::BuildStructVector:
   case VPInstruction::BuildVector:
-  case VPInstruction::Unpack:
+  case VPInstruction::UnpackVector:
     return SetResultTyFromOp();
   case VPInstruction::ExtractLane:
     return inferScalarType(R->getOperand(1));

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -506,7 +506,7 @@ unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {
   case VPInstruction::ExtractPenultimateElement:
   case VPInstruction::FirstActiveLane:
   case VPInstruction::Not:
-  case VPInstruction::Unpack:
+  case VPInstruction::UnpackVector:
     return 1;
   case Instruction::ICmp:
   case Instruction::FCmp:
@@ -1232,7 +1232,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
   case VPInstruction::StepVector:
   case VPInstruction::ReductionStartVector:
   case VPInstruction::VScale:
-  case VPInstruction::Unpack:
+  case VPInstruction::UnpackVector:
     return false;
   default:
     return true;
@@ -1399,7 +1399,7 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
   case VPInstruction::ResumeForEpilogue:
     O << "resume-for-epilogue";
     break;
-  case VPInstruction::Unpack:
+  case VPInstruction::UnpackVector:
     O << "unpack-into-scalars";
     break;
   default:

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -1230,7 +1230,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
                                                              m_VPValue(Idx)))) {
     auto *BuildVector = cast<VPInstruction>(R.getOperand(0));
     Def->replaceAllUsesWith(BuildVector->getOperand(
-        dyn_cast<ConstantInt>(Idx->getLiveInIRValue())->getZExtValue()));
+        cast<ConstantInt>(Idx->getLiveInIRValue())->getZExtValue()));
     return;
   }
 
@@ -3694,7 +3694,7 @@ void VPlanTransforms::materializeBackedgeTakenCount(VPlan &Plan,
   BTC->replaceAllUsesWith(TCMO);
 }
 
-void VPlanTransforms::materializeBuildVectors(VPlan &Plan) {
+void VPlanTransforms::materializeBuildAndUnpackVectors(VPlan &Plan) {
   if (Plan.hasScalarVFOnly())
     return;
 
@@ -3770,13 +3770,14 @@ void VPlanTransforms::materializeBuildVectors(VPlan &Plan) {
                     }))
           continue;
 
-        auto *Unpack = new VPInstruction(VPInstruction::Unpack, {Def});
+        auto *UnpackVector =
+            new VPInstruction(VPInstruction::UnpackVector, {Def});
         if (R.isPhi())
-          Unpack->insertBefore(*VPBB, VPBB->getFirstNonPhi());
+          UnpackVector->insertBefore(*VPBB, VPBB->getFirstNonPhi());
         else
-          Unpack->insertAfter(&R);
+          UnpackVector->insertAfter(&R);
         Def->replaceUsesWithIf(
-            Unpack,
+            UnpackVector,
             [Def, &UsesVectorOrInsideReplicateRegion](VPUser &U, unsigned) {
               return !UsesVectorOrInsideReplicateRegion(&U) &&
                      U.usesScalars(Def);

llvm/lib/Transforms/Vectorize/VPlanTransforms.h

@@ -319,8 +319,9 @@ struct VPlanTransforms {
                                             VPBasicBlock *VectorPH);
 
   /// Add explicit Build[Struct]Vector recipes that combine multiple scalar
-  /// values into single vectors.
-  static void materializeBuildVectors(VPlan &Plan);
+  /// values into single vectors and UnpackVector to extract scalars from a
+  /// vector as needed.
+  static void materializeBuildAndUnpackVectors(VPlan &Plan);
 
   /// Materialize VF and VFxUF to be computed explicitly using VPInstructions.
   static void materializeVFAndVFxUF(VPlan &Plan, VPBasicBlock *VectorPH,

llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp

@@ -472,7 +472,8 @@ cloneForLane(VPlan &Plan, VPBuilder &Builder, Type *IdxTy,
              const DenseMap<VPValue *, SmallVector<VPValue *>> &Def2LaneDefs) {
 
   VPValue *Op;
-  if (match(DefR, m_VPInstruction<VPInstruction::Unpack>(m_VPValue(Op)))) {
+  if (match(DefR,
+            m_VPInstruction<VPInstruction::UnpackVector>(m_VPValue(Op)))) {
     auto LaneDefs = Def2LaneDefs.find(Op);
     if (LaneDefs != Def2LaneDefs.end())
       return LaneDefs->second[Lane.getKnownLane()];
@@ -486,7 +487,7 @@ cloneForLane(VPlan &Plan, VPBuilder &Builder, Type *IdxTy,
   SmallVector<VPValue *> NewOps;
   for (VPValue *Op : DefR->operands()) {
     if (Lane.getKind() == VPLane::Kind::ScalableLast) {
-      match(Op, m_VPInstruction<VPInstruction::Unpack>(m_VPValue(Op)));
+      match(Op, m_VPInstruction<VPInstruction::UnpackVector>(m_VPValue(Op)));
       NewOps.push_back(
           Builder.createNaryOp(VPInstruction::ExtractLastElement, {Op}));
       continue;
@@ -562,7 +563,8 @@ void VPlanTransforms::replicateByVF(VPlan &Plan, ElementCount VF) {
           (isa<VPReplicateRecipe>(&R) &&
            cast<VPReplicateRecipe>(&R)->isSingleScalar()) ||
           (isa<VPInstruction>(&R) &&
-           !cast<VPInstruction>(&R)->doesGeneratePerAllLanes() && cast<VPInstruction>(&R)->getOpcode() != VPInstruction::Unpack))
+           !cast<VPInstruction>(&R)->doesGeneratePerAllLanes() &&
+           cast<VPInstruction>(&R)->getOpcode() != VPInstruction::UnpackVector))
         continue;
 
       auto *DefR = cast<VPRecipeWithIRFlags>(&R);

llvm/lib/Transforms/Vectorize/VPlanUtils.h

@@ -84,9 +84,11 @@ inline bool isSingleScalar(const VPValue *VPV) {
     return VPI->isSingleScalar() || VPI->isVectorToScalar() ||
            (PreservesUniformity(VPI->getOpcode()) &&
             all_of(VPI->operands(), isSingleScalar));
+  if (auto *Reduce = dyn_cast<VPReductionRecipe>(VPV))
+    return true;
 
   // VPExpandSCEVRecipes must be placed in the entry and are alway uniform.
-  return isa<VPExpandSCEVRecipe, VPPhi>(VPV);
+  return isa<VPExpandSCEVRecipe>(VPV);
 }
 
 /// Return true if \p V is a header mask in \p Plan.