diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 3cff43a510298..ea5b72c374e73 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4174,7 +4174,7 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() { // divisors. case Instruction::Select: { VPValue *VPV = VPI->getVPSingleValue(); - if (VPV->getNumUsers() == 1) { + if (VPV->hasOneUser()) { if (auto *WR = dyn_cast(*VPV->user_begin())) { switch (WR->getOpcode()) { case Instruction::UDiv: diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 16b1b539345de..f6fc1d48b2e41 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1066,25 +1066,25 @@ const VPRegionBlock *VPlan::getVectorLoopRegion() const { void VPlan::printLiveIns(raw_ostream &O) const { VPSlotTracker SlotTracker(this); - if (VF.getNumUsers() > 0) { + if (VF.hasUsers()) { O << "\nLive-in "; VF.printAsOperand(O, SlotTracker); O << " = VF"; } - if (VFxUF.getNumUsers() > 0) { + if (VFxUF.hasUsers()) { O << "\nLive-in "; VFxUF.printAsOperand(O, SlotTracker); O << " = VF * UF"; } - if (VectorTripCount.getNumUsers() > 0) { + if (VectorTripCount.hasUsers()) { O << "\nLive-in "; VectorTripCount.printAsOperand(O, SlotTracker); O << " = vector-trip-count"; } - if (BackedgeTakenCount && BackedgeTakenCount->getNumUsers()) { + if (BackedgeTakenCount && BackedgeTakenCount->hasUsers()) { O << "\nLive-in "; BackedgeTakenCount->printAsOperand(O, SlotTracker); O << " = backedge-taken count"; @@ -1502,9 +1502,9 @@ void VPSlotTracker::assignName(const VPValue *V) { } void VPSlotTracker::assignNames(const VPlan &Plan) { - if (Plan.VF.getNumUsers() > 0) + if (Plan.VF.hasUsers()) assignName(&Plan.VF); - if (Plan.VFxUF.getNumUsers() > 0) + if (Plan.VFxUF.hasUsers()) assignName(&Plan.VFxUF); assignName(&Plan.VectorTripCount); if (Plan.BackedgeTakenCount) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 53291a931530f..f098c7a774350 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -4218,7 +4218,7 @@ class VPlan { /// Resets the trip count for the VPlan. The caller must make sure all uses of /// the original trip count have been replaced. void resetTripCount(VPValue *NewTripCount) { - assert(TripCount && NewTripCount && TripCount->getNumUsers() == 0 && + assert(TripCount && NewTripCount && TripCount->hasNoUsers() && "TripCount must be set when resetting"); TripCount = NewTripCount; } diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index cef91c15dd873..6a4de832e25b0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -888,7 +888,7 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) { if (VecV == RdxResult) continue; if (auto *DerivedIV = dyn_cast(VecV)) { - if (DerivedIV->getNumUsers() == 1 && + if (DerivedIV->hasOneUser() && DerivedIV->getOperand(1) == &Plan.getVectorTripCount()) { auto *NewSel = Builder.createSelect(AnyNaN, Plan.getCanonicalIV(), &Plan.getVectorTripCount()); diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index bf51489543098..acf71d6ce5a7f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -2285,7 +2285,7 @@ InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF, TTI::CastContextHint CCH = TTI::CastContextHint::None; // For Trunc/FPTrunc, get the context from the only user. if ((Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) && - !hasMoreThanOneUniqueUser() && getNumUsers() > 0) { + hasOneUser()) { if (auto *StoreRecipe = dyn_cast(*user_begin())) CCH = ComputeCCH(StoreRecipe); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp index bab7a9e4205f7..9d82c39baf2ba 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp @@ -156,8 +156,7 @@ bool VPlanSlp::areVectorizable(ArrayRef Operands) const { return false; } - if (any_of(Operands, - [](VPValue *Op) { return Op->hasMoreThanOneUniqueUser(); })) { + if (any_of(Operands, [](VPValue *Op) { return !Op->hasNoOrOneUser(); })) { LLVM_DEBUG(dbgs() << "VPSLP: Some operands have multiple users.\n"); return false; } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index ecb0f093e772d..e80b9d05a343b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -316,7 +316,7 @@ static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan) { }); // Remove phi recipes that are unused after merging the regions. - if (Phi1ToMove.getVPSingleValue()->getNumUsers() == 0) { + if (Phi1ToMove.getVPSingleValue()->hasNoUsers()) { Phi1ToMove.eraseFromParent(); continue; } @@ -363,7 +363,7 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe, Plan.createVPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask); VPPredInstPHIRecipe *PHIRecipe = nullptr; - if (PredRecipe->getNumUsers() != 0) { + if (!PredRecipe->hasNoUsers()) { PHIRecipe = new VPPredInstPHIRecipe(RecipeWithoutMask, RecipeWithoutMask->getDebugLoc()); PredRecipe->replaceAllUsesWith(PHIRecipe); @@ -546,8 +546,7 @@ static bool isDeadRecipe(VPRecipeBase &R) { return false; // Recipe is dead if no user keeps the recipe alive. - return all_of(R.definedValues(), - [](VPValue *V) { return V->getNumUsers() == 0; }); + return all_of(R.definedValues(), [](VPValue *V) { return V->hasNoUsers(); }); } void VPlanTransforms::removeDeadRecipes(VPlan &Plan) { @@ -563,11 +562,11 @@ void VPlanTransforms::removeDeadRecipes(VPlan &Plan) { // Check if R is a dead VPPhi <-> update cycle and remove it. auto *PhiR = dyn_cast(&R); - if (!PhiR || PhiR->getNumOperands() != 2 || PhiR->getNumUsers() != 1) + if (!PhiR || PhiR->getNumOperands() != 2 || !PhiR->hasOneUser()) continue; VPValue *Incoming = PhiR->getOperand(1); if (*PhiR->user_begin() != Incoming->getDefiningRecipe() || - Incoming->getNumUsers() != 1) + !Incoming->hasOneUser()) continue; PhiR->replaceAllUsesWith(PhiR->getOperand(0)); PhiR->eraseFromParent(); @@ -658,7 +657,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) { auto *RepR = dyn_cast(U); // Skip recipes that shouldn't be narrowed. if (!Def || !isa(Def) || - Def->getNumUsers() == 0 || !Def->getUnderlyingValue() || + Def->hasNoUsers() || !Def->getUnderlyingValue() || (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))) continue; @@ -1103,8 +1102,8 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { m_LogicalAnd(m_Deferred(X), m_VPValue(Z)))) && // Simplify only if one of the operands has one use to avoid creating an // extra recipe. - (!Def->getOperand(0)->hasMoreThanOneUniqueUser() || - !Def->getOperand(1)->hasMoreThanOneUniqueUser())) + (Def->getOperand(0)->hasNoOrOneUser() || + Def->getOperand(1)->hasNoOrOneUser())) return Def->replaceAllUsesWith( Builder.createLogicalAnd(X, Builder.createOr(Y, Z))); @@ -1130,7 +1129,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { // further. if (match(Def, m_LogicalAnd(m_LogicalAnd(m_VPValue(X), m_VPValue(Y)), m_VPValue(Z))) && - X->hasMoreThanOneUniqueUser()) + !X->hasNoOrOneUser()) return Def->replaceAllUsesWith( Builder.createLogicalAnd(X, Builder.createLogicalAnd(Y, Z))); @@ -1383,7 +1382,7 @@ static void simplifyBlends(VPlan &Plan) { // TODO: Find the most expensive mask that can be deadcoded, or a mask // that's used by multiple blends where it can be removed from them all. VPValue *Mask = Blend->getMask(I); - if (Mask->getNumUsers() == 1 && !match(Mask, m_False())) { + if (Mask->hasOneUser() && !match(Mask, m_False())) { StartIndex = I; break; } @@ -1419,7 +1418,7 @@ static void simplifyBlends(VPlan &Plan) { NewBlend->setOperand(0, Inc1); NewBlend->setOperand(1, Inc0); NewBlend->setOperand(2, NewMask); - if (OldMask->getNumUsers() == 0) + if (OldMask->hasNoUsers()) cast(OldMask)->eraseFromParent(); } } @@ -1466,8 +1465,7 @@ static bool optimizeVectorInductionWidthForTCAndVFUF(VPlan &Plan, // Currently only handle canonical IVs as it is trivial to replace the start // and stop values, and we currently only perform the optimization when the // IV has a single use. - if (!WideIV || !WideIV->isCanonical() || - WideIV->hasMoreThanOneUniqueUser() || + if (!WideIV || !WideIV->isCanonical() || !WideIV->hasNoOrOneUser() || NewIVTy == WideIV->getScalarType()) continue; @@ -2611,7 +2609,7 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { ToErase.push_back(CurRecipe); } // Remove dead EVL mask. - if (EVLMask->getNumUsers() == 0) + if (EVLMask->hasNoUsers()) ToErase.push_back(EVLMask->getDefiningRecipe()); for (VPRecipeBase *R : reverse(ToErase)) { @@ -3590,7 +3588,7 @@ void VPlanTransforms::materializeBroadcasts(VPlan &Plan) { #endif SmallVector VPValues; - if (Plan.getOrCreateBackedgeTakenCount()->getNumUsers() > 0) + if (Plan.getOrCreateBackedgeTakenCount()->hasUsers()) VPValues.push_back(Plan.getOrCreateBackedgeTakenCount()); append_range(VPValues, Plan.getLiveIns()); for (VPRecipeBase &R : *Plan.getEntry()) @@ -3659,7 +3657,7 @@ void VPlanTransforms::materializeConstantVectorTripCount( void VPlanTransforms::materializeBackedgeTakenCount(VPlan &Plan, VPBasicBlock *VectorPH) { VPValue *BTC = Plan.getOrCreateBackedgeTakenCount(); - if (BTC->getNumUsers() == 0) + if (BTC->hasNoUsers()) return; VPBuilder Builder(VectorPH, VectorPH->begin()); @@ -3725,7 +3723,7 @@ void VPlanTransforms::materializeVectorTripCount(VPlan &Plan, assert(VectorTC.isLiveIn() && "vector-trip-count must be a live-in"); // There's nothing to do if there are no users of the vector trip count or its // IR value has already been set. - if (VectorTC.getNumUsers() == 0 || VectorTC.getLiveInIRValue()) + if (VectorTC.hasNoUsers() || VectorTC.getLiveInIRValue()) return; VPValue *TC = Plan.getTripCount(); @@ -3790,7 +3788,7 @@ void VPlanTransforms::materializeVFAndVFxUF(VPlan &Plan, VPBasicBlock *VectorPH, // If there are no users of the runtime VF, compute VFxUF by constant folding // the multiplication of VF and UF. - if (VF.getNumUsers() == 0) { + if (VF.hasNoUsers()) { VPValue *RuntimeVFxUF = Builder.createElementCount(TCTy, VFEC * Plan.getUF()); VFxUF.replaceAllUsesWith(RuntimeVFxUF); diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp index 7a63d20825a31..ed1bd7606ec8a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp @@ -535,7 +535,7 @@ void VPlanTransforms::replicateByVF(VPlan &Plan, ElementCount VF) { continue; VPBuilder Builder(RepR); - if (RepR->getNumUsers() == 0) { + if (RepR->hasNoUsers()) { if (isa(RepR->getUnderlyingInstr()) && vputils::isSingleScalar(RepR->getOperand(1))) { // Stores to invariant addresses need to store the last lane only. diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index 85c6c2c8d7965..fc62a0015595f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -56,6 +56,8 @@ class LLVM_ABI_FOR_TEST VPValue { SmallVector Users; + unsigned getNumUsers() const { return Users.size(); } + protected: // Hold the underlying Value, if any, attached to this VPValue. Value *UnderlyingVal; @@ -110,7 +112,6 @@ class LLVM_ABI_FOR_TEST VPValue { void dump() const; #endif - unsigned getNumUsers() const { return Users.size(); } void addUser(VPUser &User) { Users.push_back(&User); } /// Remove a single \p User from the list of users. @@ -136,17 +137,12 @@ class LLVM_ABI_FOR_TEST VPValue { return const_user_range(user_begin(), user_end()); } + bool hasUsers() const { return !Users.empty(); } + bool hasNoUsers() const { return !hasUsers(); } + bool hasOneUser() const { return !hasNoUsers() && all_equal(Users); } + /// Returns true if the value has more than one unique user. - bool hasMoreThanOneUniqueUser() const { - if (getNumUsers() == 0) - return false; - - // Check if all users match the first user. - auto Current = std::next(user_begin()); - while (Current != user_end() && *user_begin() == *Current) - Current++; - return Current != user_end(); - } + bool hasNoOrOneUser() const { return hasNoUsers() || hasOneUser(); } void replaceAllUsesWith(VPValue *New); diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 99f3bc367a548..af8a24a50b442 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -198,8 +198,8 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const { } // EVLIVIncrement is only used by EVLIV & BranchOnCount. // Having more than two users is unexpected. - if ((I->getNumUsers() != 1) && - (I->getNumUsers() != 2 || none_of(I->users(), [&I](VPUser *U) { + if ((size(I->users()) != 1) && + (size(I->users()) != 2 || none_of(I->users(), [&I](VPUser *U) { using namespace llvm::VPlanPatternMatch; return match(U, m_BranchOnCount(m_Specific(I), m_VPValue())); }))) { diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index db64c755d005f..3cdcc7c517faa 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -137,38 +137,38 @@ TEST_F(VPInstructionTest, setOperand) { VPValue *VPV1 = getPlan().getOrAddLiveIn(ConstantInt::get(Int32, 1)); VPValue *VPV2 = getPlan().getOrAddLiveIn(ConstantInt::get(Int32, 2)); VPInstruction *I1 = new VPInstruction(0, {VPV1, VPV2}); - EXPECT_EQ(1u, VPV1->getNumUsers()); + EXPECT_TRUE(VPV1->hasOneUser()); EXPECT_EQ(I1, *VPV1->user_begin()); - EXPECT_EQ(1u, VPV2->getNumUsers()); + EXPECT_TRUE(VPV2->hasOneUser()); EXPECT_EQ(I1, *VPV2->user_begin()); // Replace operand 0 (VPV1) with VPV3. VPValue *VPV3 = getPlan().getOrAddLiveIn(ConstantInt::get(Int32, 3)); I1->setOperand(0, VPV3); - EXPECT_EQ(0u, VPV1->getNumUsers()); - EXPECT_EQ(1u, VPV2->getNumUsers()); + EXPECT_TRUE(VPV1->hasNoUsers()); + EXPECT_TRUE(VPV2->hasOneUser()); EXPECT_EQ(I1, *VPV2->user_begin()); - EXPECT_EQ(1u, VPV3->getNumUsers()); + EXPECT_TRUE(VPV3->hasOneUser()); EXPECT_EQ(I1, *VPV3->user_begin()); // Replace operand 1 (VPV2) with VPV3. I1->setOperand(1, VPV3); - EXPECT_EQ(0u, VPV1->getNumUsers()); - EXPECT_EQ(0u, VPV2->getNumUsers()); - EXPECT_EQ(2u, VPV3->getNumUsers()); + EXPECT_TRUE(VPV1->hasNoUsers()); + EXPECT_TRUE(VPV2->hasNoUsers()); + EXPECT_EQ(2u, size(VPV3->users())); EXPECT_EQ(I1, *VPV3->user_begin()); EXPECT_EQ(I1, *std::next(VPV3->user_begin())); // Replace operand 0 (VPV3) with VPV4. VPValue *VPV4 = getPlan().getOrAddLiveIn(ConstantInt::get(Int32, 4)); I1->setOperand(0, VPV4); - EXPECT_EQ(1u, VPV3->getNumUsers()); + EXPECT_TRUE(VPV3->hasOneUser()); EXPECT_EQ(I1, *VPV3->user_begin()); EXPECT_EQ(I1, *VPV4->user_begin()); // Replace operand 1 (VPV3) with VPV4. I1->setOperand(1, VPV4); - EXPECT_EQ(0u, VPV3->getNumUsers()); + EXPECT_TRUE(VPV3->hasNoUsers()); EXPECT_EQ(I1, *VPV4->user_begin()); EXPECT_EQ(I1, *std::next(VPV4->user_begin())); @@ -186,34 +186,34 @@ TEST_F(VPInstructionTest, replaceAllUsesWith) { VPV1->replaceAllUsesWith(VPV3); EXPECT_EQ(VPV3, I1->getOperand(0)); EXPECT_EQ(VPV2, I1->getOperand(1)); - EXPECT_EQ(0u, VPV1->getNumUsers()); - EXPECT_EQ(1u, VPV2->getNumUsers()); + EXPECT_TRUE(VPV1->hasNoUsers()); + EXPECT_TRUE(VPV2->hasOneUser()); EXPECT_EQ(I1, *VPV2->user_begin()); - EXPECT_EQ(1u, VPV3->getNumUsers()); + EXPECT_TRUE(VPV3->hasOneUser()); EXPECT_EQ(I1, *VPV3->user_begin()); // Replace all uses of VPV2 with VPV3. VPV2->replaceAllUsesWith(VPV3); EXPECT_EQ(VPV3, I1->getOperand(0)); EXPECT_EQ(VPV3, I1->getOperand(1)); - EXPECT_EQ(0u, VPV1->getNumUsers()); - EXPECT_EQ(0u, VPV2->getNumUsers()); - EXPECT_EQ(2u, VPV3->getNumUsers()); + EXPECT_TRUE(VPV1->hasNoUsers()); + EXPECT_TRUE(VPV2->hasNoUsers()); + EXPECT_EQ(2u, size(VPV3->users())); EXPECT_EQ(I1, *VPV3->user_begin()); // Replace all uses of VPV3 with VPV1. VPV3->replaceAllUsesWith(VPV1); EXPECT_EQ(VPV1, I1->getOperand(0)); EXPECT_EQ(VPV1, I1->getOperand(1)); - EXPECT_EQ(2u, VPV1->getNumUsers()); + EXPECT_EQ(2u, size(VPV1->users())); EXPECT_EQ(I1, *VPV1->user_begin()); - EXPECT_EQ(0u, VPV2->getNumUsers()); - EXPECT_EQ(0u, VPV3->getNumUsers()); + EXPECT_TRUE(VPV2->hasNoUsers()); + EXPECT_TRUE(VPV3->hasNoUsers()); VPInstruction *I2 = new VPInstruction(0, {VPV1, VPV2}); - EXPECT_EQ(3u, VPV1->getNumUsers()); + EXPECT_EQ(3u, size(VPV1->users())); VPV1->replaceAllUsesWith(VPV3); - EXPECT_EQ(3u, VPV3->getNumUsers()); + EXPECT_EQ(3u, size(VPV3->users())); delete I1; delete I2; @@ -225,15 +225,15 @@ TEST_F(VPInstructionTest, releaseOperandsAtDeletion) { VPValue *VPV2 = getPlan().getOrAddLiveIn(ConstantInt::get(Int32, 1)); VPInstruction *I1 = new VPInstruction(0, {VPV1, VPV2}); - EXPECT_EQ(1u, VPV1->getNumUsers()); + EXPECT_TRUE(VPV1->hasOneUser()); EXPECT_EQ(I1, *VPV1->user_begin()); - EXPECT_EQ(1u, VPV2->getNumUsers()); + EXPECT_TRUE(VPV2->hasOneUser()); EXPECT_EQ(I1, *VPV2->user_begin()); delete I1; - EXPECT_EQ(0u, VPV1->getNumUsers()); - EXPECT_EQ(0u, VPV2->getNumUsers()); + EXPECT_TRUE(VPV1->hasNoUsers()); + EXPECT_TRUE(VPV2->hasNoUsers()); } using VPBasicBlockTest = VPlanTestBase;