@@ -5578,16 +5578,14 @@ class BoUpSLP {
55785578 // Decrement the unscheduled counter and insert to ready list if
55795579 // ready.
55805580 auto DecrUnschedForInst = [&](Instruction *I, TreeEntry *UserTE,
5581- unsigned OpIdx, bool FirstRun = false ) {
5581+ unsigned OpIdx) {
55825582 if (!ScheduleCopyableDataMap.empty()) {
55835583 const EdgeInfo EI = {UserTE, OpIdx};
55845584 if (ScheduleCopyableData *CD = getScheduleCopyableData(EI, I)) {
55855585 DecrUnsched(CD, /*IsControl=*/false);
55865586 return;
55875587 }
55885588 }
5589- if (!FirstRun)
5590- return;
55915589 auto It = OperandsUses.find(I);
55925590 assert(It != OperandsUses.end() && "Operand not found");
55935591 if (It->second > 0) {
@@ -5604,48 +5602,37 @@ class BoUpSLP {
56045602 break;
56055603 // Need to search for the lane since the tree entry can be
56065604 // reordered.
5607- auto *It = find(Bundle->getTreeEntry()->Scalars, In);
5608- bool FirstRun = true;
5609- do {
5610- int Lane =
5611- std::distance(Bundle->getTreeEntry()->Scalars.begin(), It);
5612- assert(Lane >= 0 && "Lane not set");
5613- if (isa<StoreInst>(In) &&
5614- !Bundle->getTreeEntry()->ReorderIndices.empty())
5615- Lane = Bundle->getTreeEntry()->ReorderIndices[Lane];
5616- assert(Lane < static_cast<int>(
5617- Bundle->getTreeEntry()->Scalars.size()) &&
5618- "Couldn't find extract lane");
5619-
5620- // Since vectorization tree is being built recursively this
5621- // assertion ensures that the tree entry has all operands set
5622- // before reaching this code. Couple of exceptions known at the
5623- // moment are extracts where their second (immediate) operand is
5624- // not added. Since immediates do not affect scheduler behavior
5625- // this is considered okay.
5626- assert(In &&
5627- (isa<ExtractValueInst, ExtractElementInst, CallBase>(In) ||
5628- In->getNumOperands() ==
5629- Bundle->getTreeEntry()->getNumOperands() ||
5630- Bundle->getTreeEntry()->isCopyableElement(In)) &&
5631- "Missed TreeEntry operands?");
5632-
5633- for (unsigned OpIdx :
5634- seq<unsigned>(Bundle->getTreeEntry()->getNumOperands()))
5635- if (auto *I = dyn_cast<Instruction>(
5636- Bundle->getTreeEntry()->getOperand(OpIdx)[Lane])) {
5637- LLVM_DEBUG(dbgs() << "SLP: check for readiness (def): "
5638- << *I << "\n");
5639- DecrUnschedForInst(I, Bundle->getTreeEntry(), OpIdx,
5640- FirstRun);
5641- }
5642- // If parent node is schedulable, it will be handle correctly.
5643- if (!Bundle->getTreeEntry()->doesNotNeedToSchedule())
5644- break;
5645- It = std::find(std::next(It),
5646- Bundle->getTreeEntry()->Scalars.end(), In);
5647- FirstRun = false;
5648- } while (It != Bundle->getTreeEntry()->Scalars.end());
5605+ int Lane = std::distance(Bundle->getTreeEntry()->Scalars.begin(),
5606+ find(Bundle->getTreeEntry()->Scalars, In));
5607+ assert(Lane >= 0 && "Lane not set");
5608+ if (isa<StoreInst>(In) &&
5609+ !Bundle->getTreeEntry()->ReorderIndices.empty())
5610+ Lane = Bundle->getTreeEntry()->ReorderIndices[Lane];
5611+ assert(Lane < static_cast<int>(
5612+ Bundle->getTreeEntry()->Scalars.size()) &&
5613+ "Couldn't find extract lane");
5614+
5615+ // Since vectorization tree is being built recursively this
5616+ // assertion ensures that the tree entry has all operands set before
5617+ // reaching this code. Couple of exceptions known at the moment are
5618+ // extracts where their second (immediate) operand is not added.
5619+ // Since immediates do not affect scheduler behavior this is
5620+ // considered okay.
5621+ assert(In &&
5622+ (isa<ExtractValueInst, ExtractElementInst, CallBase>(In) ||
5623+ In->getNumOperands() ==
5624+ Bundle->getTreeEntry()->getNumOperands() ||
5625+ Bundle->getTreeEntry()->isCopyableElement(In)) &&
5626+ "Missed TreeEntry operands?");
5627+
5628+ for (unsigned OpIdx :
5629+ seq<unsigned>(Bundle->getTreeEntry()->getNumOperands()))
5630+ if (auto *I = dyn_cast<Instruction>(
5631+ Bundle->getTreeEntry()->getOperand(OpIdx)[Lane])) {
5632+ LLVM_DEBUG(dbgs() << "SLP: check for readiness (def): " << *I
5633+ << "\n");
5634+ DecrUnschedForInst(I, Bundle->getTreeEntry(), OpIdx);
5635+ }
56495636 }
56505637 } else {
56515638 // If BundleMember is a stand-alone instruction, no operand reordering
0 commit comments