@@ -125,36 +125,80 @@ SchedBundle *Scheduler::createBundle(ArrayRef<Instruction *> Instrs) {
125125void Scheduler::eraseBundle (SchedBundle *SB) { Bndls.erase (SB); }
126126
127127bool Scheduler::tryScheduleUntil (ArrayRef<Instruction *> Instrs) {
128- // Use a set of instructions, instead of `Instrs` for fast lookups.
129- DenseSet<Instruction *> InstrsToDefer (Instrs.begin (), Instrs.end ());
130- // This collects the nodes that correspond to instructions found in `Instrs`
131- // that have just become ready. These nodes won't be scheduled right away.
132- SmallVector<DGNode *, 8 > DeferredNodes;
133-
128+ // Create a bundle for Instrs. If it turns out the schedule is infeasible we
129+ // will dismantle it.
130+ auto *InstrsSB = createBundle (Instrs);
134131 // Keep scheduling ready nodes until we either run out of ready nodes (i.e.,
135132 // ReadyList is empty), or all nodes that correspond to `Instrs` (the nodes of
136133 // which are collected in DeferredNodes) are all ready to schedule.
137- while (!ReadyList.empty ()) {
138- auto *ReadyN = ReadyList.pop ();
139- if (InstrsToDefer.contains (ReadyN->getInstruction ())) {
140- // If the ready instruction is one of those in `Instrs`, then we don't
141- // schedule it right away. Instead we defer it until we can schedule it
142- // along with the rest of the instructions in `Instrs`, at the same
143- // time in a single scheduling bundle.
144- DeferredNodes.push_back (ReadyN);
145- bool ReadyToScheduleDeferred = DeferredNodes.size () == Instrs.size ();
146- if (ReadyToScheduleDeferred) {
147- scheduleAndUpdateReadyList (*createBundle (Instrs));
134+ SmallVector<DGNode *> Retry;
135+ bool KeepScheduling = true ;
136+ while (KeepScheduling) {
137+ enum class TryScheduleRes {
138+ Success, // /> We successfully scheduled the bundle.
139+ Failure, // /> We failed to schedule the bundle.
140+ Finished, // /> We successfully scheduled the bundle and it is the last
141+ // / bundle to be scheduled.
142+ };
143+ // / TryScheduleNode() attempts to schedule all DAG nodes in the bundle that
144+ // / ReadyN is in. If it's not in a bundle it will create a singleton bundle
145+ // / and will try to schedule it.
146+ auto TryScheduleBndl = [this , InstrsSB](DGNode *ReadyN) -> TryScheduleRes {
147+ auto *SB = ReadyN->getSchedBundle ();
148+ if (SB == nullptr ) {
149+ // If ReadyN does not belong to a bundle, create a singleton bundle
150+ // and schedule it.
151+ auto *SingletonSB = createBundle ({ReadyN->getInstruction ()});
152+ scheduleAndUpdateReadyList (*SingletonSB);
153+ return TryScheduleRes::Success;
154+ }
155+ if (SB->ready ()) {
156+ // Remove the rest of the bundle from the ready list.
157+ // TODO: Perhaps change the Scheduler + ReadyList to operate on
158+ // SchedBundles instead of DGNodes.
159+ for (auto *N : *SB) {
160+ if (N != ReadyN)
161+ ReadyList.remove (N);
162+ }
163+ // If all nodes in the bundle are ready.
164+ scheduleAndUpdateReadyList (*SB);
165+ if (SB == InstrsSB)
166+ // We just scheduled InstrsSB bundle, so we are done scheduling.
167+ return TryScheduleRes::Finished;
168+ return TryScheduleRes::Success;
169+ }
170+ return TryScheduleRes::Failure;
171+ };
172+ while (!ReadyList.empty ()) {
173+ auto *ReadyN = ReadyList.pop ();
174+ auto Res = TryScheduleBndl (ReadyN);
175+ switch (Res) {
176+ case TryScheduleRes::Success:
177+ // We successfully scheduled ReadyN, keep scheduling.
178+ continue ;
179+ case TryScheduleRes::Failure:
180+ // We failed to schedule ReadyN, defer it to later and keep scheduling
181+ // other ready instructions.
182+ Retry.push_back (ReadyN);
183+ continue ;
184+ case TryScheduleRes::Finished:
185+ // We successfully scheduled the instruction bundle, so we are done.
148186 return true ;
149187 }
150- } else {
151- // If the ready instruction is not found in `Instrs`, then we wrap it in a
152- // scheduling bundle and schedule it right away.
153- scheduleAndUpdateReadyList (*createBundle ({ReadyN->getInstruction ()}));
188+ llvm_unreachable (" Unhandled TrySchedule() result"
189+ }
190+ // Try to schedule nodes from the Retry list.
191+ KeepScheduling = false ;
192+ for (auto *N : make_early_inc_range (Retry)) {
193+ auto Res = TryScheduleBndl (N);
194+ if (Res == TryScheduleRes::Success) {
195+ Retry.erase (find (Retry, N));
196+ KeepScheduling = true ;
197+ }
154198 }
155199 }
156- assert (DeferredNodes. size () != Instrs. size () &&
157- " We should have succesfully scheduled and early-returned! "
200+
201+ eraseBundle (InstrsSB );
158202 return false ;
159203}
160204
@@ -275,6 +319,7 @@ bool Scheduler::trySchedule(ArrayRef<Instruction *> Instrs) {
275319 // If one or more instrs are already scheduled we need to destroy the
276320 // top-most part of the schedule that includes the instrs in the bundle and
277321 // re-schedule.
322+ DAG.extend (Instrs);
278323 trimSchedule (Instrs);
279324 ScheduleTopItOpt = std::next (VecUtils::getLowest (Instrs)->getIterator ());
280325 return tryScheduleUntil (Instrs);
0 commit comments