diff --git a/llvm/include/llvm/Transforms/Utils/LockstepReverseIterator.h b/llvm/include/llvm/Transforms/Utils/LockstepReverseIterator.h new file mode 100644 index 0000000000000..cc9d717054528 --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/LockstepReverseIterator.h @@ -0,0 +1,155 @@ +//===- LockstepReverseIterator.h ------------------------------*- C++ -*---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_UTILS_LOCKSTEPREVERSEITERATOR_H +#define LLVM_TRANSFORMS_UTILS_LOCKSTEPREVERSEITERATOR_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Instruction.h" + +namespace llvm { + +struct NoActiveBlocksOption {}; + +struct ActiveBlocksOption { + SmallSetVector ActiveBlocks; + SmallSetVector &getActiveBlocks() { return ActiveBlocks; } + ActiveBlocksOption() = default; +}; + +/// Iterates through instructions in a set of blocks in reverse order from the +/// first non-terminator. For example (assume all blocks have size n): +/// LockstepReverseIterator I([B1, B2, B3]); +/// *I-- = [B1[n], B2[n], B3[n]]; +/// *I-- = [B1[n-1], B2[n-1], B3[n-1]]; +/// *I-- = [B1[n-2], B2[n-2], B3[n-2]]; +/// ... +/// +/// The iterator continues processing until all blocks have been exhausted if \p +/// EarlyFailure is explicitly set to \c false. Use \c getActiveBlocks() to +/// determine which blocks are still going and the order they appear in the list +/// returned by operator*. +template +class LockstepReverseIterator + : private std::conditional_t { +private: + using Base = std::conditional_t; + ArrayRef Blocks; + SmallVector Insts; + bool Fail; + +public: + LockstepReverseIterator(ArrayRef Blocks) : Blocks(Blocks) { + reset(); + } + + void reset() { + Fail = false; + if constexpr (!EarlyFailure) { + this->ActiveBlocks.clear(); + for (BasicBlock *BB : Blocks) + this->ActiveBlocks.insert(BB); + } + Insts.clear(); + for (BasicBlock *BB : Blocks) { + Instruction *Prev = BB->getTerminator()->getPrevNonDebugInstruction(); + if (!Prev) { + // Block wasn't big enough - only contained a terminator. + if constexpr (EarlyFailure) { + Fail = true; + return; + } else { + this->ActiveBlocks.remove(BB); + continue; + } + } + Insts.push_back(Prev); + } + if (Insts.empty()) + Fail = true; + } + + bool isValid() const { return !Fail; } + ArrayRef operator*() const { return Insts; } + + // Note: This needs to return a SmallSetVector as the elements of + // ActiveBlocks will be later copied to Blocks using std::copy. The + // resultant order of elements in Blocks needs to be deterministic. + // Using SmallPtrSet instead causes non-deterministic order while + // copying. And we cannot simply sort Blocks as they need to match the + // corresponding Values. + SmallSetVector &getActiveBlocks() { + return Base::getActiveBlocks(); + } + + void restrictToBlocks(SmallSetVector &Blocks) { + static_assert(!EarlyFailure, "Unknown method"); + for (auto It = Insts.begin(); It != Insts.end();) { + if (!Blocks.contains((*It)->getParent())) { + this->ActiveBlocks.remove((*It)->getParent()); + It = Insts.erase(It); + } else { + ++It; + } + } + } + + LockstepReverseIterator &operator--() { + if (Fail) + return *this; + SmallVector NewInsts; + for (Instruction *Inst : Insts) { + Instruction *Prev = Inst->getPrevNonDebugInstruction(); + if (!Prev) { + if constexpr (!EarlyFailure) { + this->ActiveBlocks.remove(Inst->getParent()); + } else { + Fail = true; + return *this; + } + } else { + NewInsts.push_back(Prev); + } + } + if (NewInsts.empty()) + Fail = true; + else + Insts = NewInsts; + return *this; + } + + LockstepReverseIterator &operator++() { + static_assert(EarlyFailure, "Unknown method"); + if (Fail) + return *this; + SmallVector NewInsts; + for (Instruction *Inst : Insts) { + Instruction *Next = Inst->getNextNonDebugInstruction(); + // Already at end of block. + if (!Next) { + Fail = true; + return *this; + } + NewInsts.push_back(Next); + } + if (NewInsts.empty()) + Fail = true; + else + Insts = NewInsts; + return *this; + } +}; + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_UTILS_LOCKSTEPREVERSEITERATOR_H diff --git a/llvm/lib/Transforms/Scalar/GVNSink.cpp b/llvm/lib/Transforms/Scalar/GVNSink.cpp index 730f5cd0f8d0d..8fc09db668fe4 100644 --- a/llvm/lib/Transforms/Scalar/GVNSink.cpp +++ b/llvm/lib/Transforms/Scalar/GVNSink.cpp @@ -65,6 +65,7 @@ #include "llvm/Transforms/Scalar/GVNExpression.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Local.h" +#include "llvm/Transforms/Utils/LockstepReverseIterator.h" #include #include #include @@ -96,87 +97,6 @@ static bool isMemoryInst(const Instruction *I) { (isa(I) && !cast(I)->doesNotAccessMemory()); } -/// Iterates through instructions in a set of blocks in reverse order from the -/// first non-terminator. For example (assume all blocks have size n): -/// LockstepReverseIterator I([B1, B2, B3]); -/// *I-- = [B1[n], B2[n], B3[n]]; -/// *I-- = [B1[n-1], B2[n-1], B3[n-1]]; -/// *I-- = [B1[n-2], B2[n-2], B3[n-2]]; -/// ... -/// -/// It continues until all blocks have been exhausted. Use \c getActiveBlocks() -/// to -/// determine which blocks are still going and the order they appear in the -/// list returned by operator*. -class LockstepReverseIterator { - ArrayRef Blocks; - SmallSetVector ActiveBlocks; - SmallVector Insts; - bool Fail; - -public: - LockstepReverseIterator(ArrayRef Blocks) : Blocks(Blocks) { - reset(); - } - - void reset() { - Fail = false; - ActiveBlocks.clear(); - for (BasicBlock *BB : Blocks) - ActiveBlocks.insert(BB); - Insts.clear(); - for (BasicBlock *BB : Blocks) { - if (BB->size() <= 1) { - // Block wasn't big enough - only contained a terminator. - ActiveBlocks.remove(BB); - continue; - } - Insts.push_back(BB->getTerminator()->getPrevNonDebugInstruction()); - } - if (Insts.empty()) - Fail = true; - } - - bool isValid() const { return !Fail; } - ArrayRef operator*() const { return Insts; } - - // Note: This needs to return a SmallSetVector as the elements of - // ActiveBlocks will be later copied to Blocks using std::copy. The - // resultant order of elements in Blocks needs to be deterministic. - // Using SmallPtrSet instead causes non-deterministic order while - // copying. And we cannot simply sort Blocks as they need to match the - // corresponding Values. - SmallSetVector &getActiveBlocks() { return ActiveBlocks; } - - void restrictToBlocks(SmallSetVector &Blocks) { - for (auto II = Insts.begin(); II != Insts.end();) { - if (!Blocks.contains((*II)->getParent())) { - ActiveBlocks.remove((*II)->getParent()); - II = Insts.erase(II); - } else { - ++II; - } - } - } - - void operator--() { - if (Fail) - return; - SmallVector NewInsts; - for (auto *Inst : Insts) { - if (Inst == &Inst->getParent()->front()) - ActiveBlocks.remove(Inst->getParent()); - else - NewInsts.push_back(Inst->getPrevNonDebugInstruction()); - } - if (NewInsts.empty()) { - Fail = true; - return; - } - Insts = NewInsts; - } -}; - //===----------------------------------------------------------------------===// /// Candidate solution for sinking. There may be different ways to @@ -634,9 +554,11 @@ class GVNSink { /// The main heuristic function. Analyze the set of instructions pointed to by /// LRI and return a candidate solution if these instructions can be sunk, or /// std::nullopt otherwise. - std::optional analyzeInstructionForSinking( - LockstepReverseIterator &LRI, unsigned &InstNum, unsigned &MemoryInstNum, - ModelledPHISet &NeededPHIs, SmallPtrSetImpl &PHIContents); + std::optional + analyzeInstructionForSinking(LockstepReverseIterator &LRI, + unsigned &InstNum, unsigned &MemoryInstNum, + ModelledPHISet &NeededPHIs, + SmallPtrSetImpl &PHIContents); /// Create a ModelledPHI for each PHI in BB, adding to PHIs. void analyzeInitialPHIs(BasicBlock *BB, ModelledPHISet &PHIs, @@ -675,7 +597,7 @@ class GVNSink { }; std::optional -GVNSink::analyzeInstructionForSinking(LockstepReverseIterator &LRI, +GVNSink::analyzeInstructionForSinking(LockstepReverseIterator &LRI, unsigned &InstNum, unsigned &MemoryInstNum, ModelledPHISet &NeededPHIs, @@ -827,7 +749,7 @@ unsigned GVNSink::sinkBB(BasicBlock *BBEnd) { return BB->getTerminator()->getNumSuccessors() != 1; }); - LockstepReverseIterator LRI(Preds); + LockstepReverseIterator LRI(Preds); SmallVector Candidates; unsigned InstNum = 0, MemoryInstNum = 0; ModelledPHISet NeededPHIs; diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index 1991ec82d1e1e..7440ca46c8d27 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -74,6 +74,7 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Local.h" +#include "llvm/Transforms/Utils/LockstepReverseIterator.h" #include "llvm/Transforms/Utils/ValueMapper.h" #include #include @@ -2326,81 +2327,6 @@ static void sinkLastInstruction(ArrayRef Blocks) { } } -namespace { - - // LockstepReverseIterator - Iterates through instructions - // in a set of blocks in reverse order from the first non-terminator. - // For example (assume all blocks have size n): - // LockstepReverseIterator I([B1, B2, B3]); - // *I-- = [B1[n], B2[n], B3[n]]; - // *I-- = [B1[n-1], B2[n-1], B3[n-1]]; - // *I-- = [B1[n-2], B2[n-2], B3[n-2]]; - // ... - class LockstepReverseIterator { - ArrayRef Blocks; - SmallVector Insts; - bool Fail; - - public: - LockstepReverseIterator(ArrayRef Blocks) : Blocks(Blocks) { - reset(); - } - - void reset() { - Fail = false; - Insts.clear(); - for (auto *BB : Blocks) { - Instruction *Inst = BB->getTerminator(); - for (Inst = Inst->getPrevNode(); Inst && isa(Inst);) - Inst = Inst->getPrevNode(); - if (!Inst) { - // Block wasn't big enough. - Fail = true; - return; - } - Insts.push_back(Inst); - } - } - - bool isValid() const { - return !Fail; - } - - void operator--() { - if (Fail) - return; - for (auto *&Inst : Insts) { - for (Inst = Inst->getPrevNode(); Inst && isa(Inst);) - Inst = Inst->getPrevNode(); - // Already at beginning of block. - if (!Inst) { - Fail = true; - return; - } - } - } - - void operator++() { - if (Fail) - return; - for (auto *&Inst : Insts) { - for (Inst = Inst->getNextNode(); Inst && isa(Inst);) - Inst = Inst->getNextNode(); - // Already at end of block. - if (!Inst) { - Fail = true; - return; - } - } - } - - ArrayRef operator * () const { - return Insts; - } - }; - -} // end anonymous namespace - /// Check whether BB's predecessors end with unconditional branches. If it is /// true, sink any common code from the predecessors to BB. static bool sinkCommonCodeFromPredecessors(BasicBlock *BB, @@ -2477,7 +2403,7 @@ static bool sinkCommonCodeFromPredecessors(BasicBlock *BB, int ScanIdx = 0; SmallPtrSet InstructionsToSink; - LockstepReverseIterator LRI(UnconditionalPreds); + LockstepReverseIterator LRI(UnconditionalPreds); while (LRI.isValid() && canSinkInstructions(*LRI, PHIOperands)) { LLVM_DEBUG(dbgs() << "SINK: instruction can be sunk: " << *(*LRI)[0] @@ -2507,7 +2433,7 @@ static bool sinkCommonCodeFromPredecessors(BasicBlock *BB, // Okay, we *could* sink last ScanIdx instructions. But how many can we // actually sink before encountering instruction that is unprofitable to // sink? - auto ProfitableToSinkInstruction = [&](LockstepReverseIterator &LRI) { + auto ProfitableToSinkInstruction = [&](LockstepReverseIterator &LRI) { unsigned NumPHIInsts = 0; for (Use &U : (*LRI)[0]->operands()) { auto It = PHIOperands.find(&U);