@@ -1744,7 +1744,7 @@ namespace slpvectorizer {
17441744
17451745/// Bottom Up SLP Vectorizer.
17461746class BoUpSLP {
1747- struct TreeEntry;
1747+ class TreeEntry;
17481748 class ScheduleEntity;
17491749 class ScheduleData;
17501750 class ScheduleBundle;
@@ -3607,7 +3607,8 @@ class BoUpSLP {
36073607 /// opportunities.
36083608 void reorderGatherNode(TreeEntry &TE);
36093609
3610- struct TreeEntry {
3610+ class TreeEntry {
3611+ public:
36113612 using VecTreeTy = SmallVector<std::unique_ptr<TreeEntry>, 8>;
36123613 TreeEntry(VecTreeTy &Container) : Container(Container) {}
36133614
@@ -3774,6 +3775,9 @@ class BoUpSLP {
37743775 /// Interleaving factor for interleaved loads Vectorize nodes.
37753776 unsigned InterleaveFactor = 0;
37763777
3778+ /// True if the node does not require scheduling.
3779+ bool DoesNotNeedToSchedule = false;
3780+
37773781 /// Set this bundle's \p OpIdx'th operand to \p OpVL.
37783782 void setOperand(unsigned OpIdx, ArrayRef<Value *> OpVL) {
37793783 if (Operands.size() < OpIdx + 1)
@@ -3791,6 +3795,16 @@ class BoUpSLP {
37913795 /// Sets interleaving factor for the interleaving nodes.
37923796 void setInterleave(unsigned Factor) { InterleaveFactor = Factor; }
37933797
3798+ /// Marks the node as one that does not require scheduling.
3799+ void setDoesNotNeedToSchedule() {
3800+ assert(::doesNotNeedToSchedule(Scalars) &&
3801+ "Expected to not need scheduling");
3802+ DoesNotNeedToSchedule = true;
3803+ }
3804+ /// Returns true if the node is marked as one that does not require
3805+ /// scheduling.
3806+ bool doesNotNeedToSchedule() const { return DoesNotNeedToSchedule; }
3807+
37943808 /// Set this bundle's operands from \p Operands.
37953809 void setOperands(ArrayRef<ValueList> Operands) {
37963810 for (unsigned I : seq<unsigned>(Operands.size()))
@@ -4107,6 +4121,8 @@ class BoUpSLP {
41074121 }
41084122 }
41094123 } else if (!Last->isGather()) {
4124+ if (doesNotNeedToSchedule(VL))
4125+ Last->setDoesNotNeedToSchedule();
41104126 SmallPtrSet<Value *, 4> Processed;
41114127 for (Value *V : VL) {
41124128 if (isa<PoisonValue>(V))
@@ -4124,7 +4140,7 @@ class BoUpSLP {
41244140 // Update the scheduler bundle to point to this TreeEntry.
41254141 assert((!Bundle.getBundle().empty() || isa<PHINode>(S.getMainOp()) ||
41264142 isVectorLikeInstWithConstOps(S.getMainOp()) ||
4127- doesNotNeedToSchedule(VL )) &&
4143+ Last-> doesNotNeedToSchedule()) &&
41284144 "Bundle and VL out of sync");
41294145 if (!Bundle.getBundle().empty()) {
41304146#if !defined(NDEBUG) || defined(EXPENSIVE_CHECKS)
@@ -15331,8 +15347,8 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
1533115347 }
1533215348
1533315349 if (!TEUseEI.UserTE->isGather() && !UserPHI &&
15334- doesNotNeedToSchedule( TEUseEI.UserTE->Scalars ) !=
15335- doesNotNeedToSchedule( UseEI.UserTE->Scalars ) &&
15350+ TEUseEI.UserTE->doesNotNeedToSchedule( ) !=
15351+ UseEI.UserTE->doesNotNeedToSchedule( ) &&
1533615352 is_contained(UseEI.UserTE->Scalars, TEInsertPt))
1533715353 continue;
1533815354 // Check if the user node of the TE comes after user node of TEPtr,
@@ -16127,7 +16143,7 @@ void BoUpSLP::setInsertPointAfterBundle(const TreeEntry *E) {
1612716143 }
1612816144 if (IsPHI ||
1612916145 (!E->isGather() && E->State != TreeEntry::SplitVectorize &&
16130- doesNotNeedToSchedule( E->Scalars )) ||
16146+ E->doesNotNeedToSchedule( )) ||
1613116147 (GatheredLoadsEntriesFirst.has_value() &&
1613216148 E->Idx >= *GatheredLoadsEntriesFirst && !E->isGather() &&
1613316149 E->getOpcode() == Instruction::Load)) {
@@ -19803,7 +19819,7 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
1980319819 if (ScheduleData *SD = BS->getScheduleData(I)) {
1980419820 [[maybe_unused]] ArrayRef<TreeEntry *> SDTEs = getTreeEntries(I);
1980519821 assert((isVectorLikeInstWithConstOps(SD->getInst()) || SDTEs.empty() ||
19806- doesNotNeedToSchedule( SDTEs.front()->Scalars )) &&
19822+ SDTEs.front()->doesNotNeedToSchedule( )) &&
1980719823 "scheduler and vectorizer bundle mismatch");
1980819824 SD->setSchedulingPriority(Idx++);
1980919825 continue;
0 commit comments