diff --git a/llvm/include/llvm/Transforms/Utils/PredicateInfo.h b/llvm/include/llvm/Transforms/Utils/PredicateInfo.h index 3df3495f84470..e3b394127b75a 100644 --- a/llvm/include/llvm/Transforms/Utils/PredicateInfo.h +++ b/llvm/include/llvm/Transforms/Utils/PredicateInfo.h @@ -67,7 +67,7 @@ class Value; class IntrinsicInst; class raw_ostream; -enum PredicateType { PT_Branch, PT_Assume, PT_Switch }; +enum PredicateType { PT_Branch, PT_Assume, PT_Switch, PT_PHI }; /// Constraint for a predicate of the form "cmp Pred Op, OtherOp", where Op /// is the value the constraint applies to (the bitcast result). @@ -171,6 +171,19 @@ class PredicateSwitch : public PredicateWithEdge { } }; +class PredicatePHI : public PredicateBase { +public: + BasicBlock *PHIBlock; + SmallVector, 4> IncomingPredicates; + + PredicatePHI(Value *Op, BasicBlock *PHIBB) + : PredicateBase(PT_PHI, Op, nullptr), PHIBlock(PHIBB) {} + PredicatePHI() = delete; + static bool classof(const PredicateBase *PB) { return PB->Type == PT_PHI; } + + LLVM_ABI std::optional getConstraint() const; +}; + /// Encapsulates PredicateInfo, including all data associated with memory /// accesses. class PredicateInfo { diff --git a/llvm/lib/Transforms/Utils/PredicateInfo.cpp b/llvm/lib/Transforms/Utils/PredicateInfo.cpp index 13c7ad2927d1c..20adf5656d3ef 100644 --- a/llvm/lib/Transforms/Utils/PredicateInfo.cpp +++ b/llvm/lib/Transforms/Utils/PredicateInfo.cpp @@ -15,7 +15,9 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/Analysis/AssumptionCache.h" +#include "llvm/Analysis/IteratedDominanceFrontier.h" #include "llvm/IR/AssemblyAnnotationWriter.h" +#include "llvm/IR/CFG.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstIterator.h" @@ -213,6 +215,8 @@ class PredicateInfoBuilder { // whether it returned a valid result. DenseMap ValueInfoNums; + DenseMap> PHICandidates; + BumpPtrAllocator &Allocator; ValueInfo &getOrCreateValueInfo(Value *); @@ -224,6 +228,13 @@ class PredicateInfoBuilder { SmallVectorImpl &OpsToRename); void processSwitch(SwitchInst *, BasicBlock *, SmallVectorImpl &OpsToRename); + void identifyPHICandidates(SmallVectorImpl &OpsToRename); + void needsPHIInsertion(Value *Op, + const SmallPtrSetImpl &DefiningBlocks, + SmallPtrSetImpl &PHIBlocks); + void insertPredicatePHIs(Value *Op, BasicBlock *PHIBlock, + SmallVectorImpl &OpsToRename); + void processPredicatePHIs(SmallVectorImpl &OpsToRename); void renameUses(SmallVectorImpl &OpsToRename); void addInfoFor(SmallVectorImpl &OpsToRename, Value *Op, PredicateBase *PB); @@ -461,6 +472,103 @@ void PredicateInfoBuilder::processSwitch( } } +void PredicateInfoBuilder::identifyPHICandidates( + SmallVectorImpl &OpsToRename) { + for (Value *Op : OpsToRename) { + const auto &ValueInfo = getValueInfo(Op); + SmallPtrSet DefiningBlocks; + for (const auto *PInfo : ValueInfo.Infos) { + if (auto *PBranch = dyn_cast(PInfo)) { + DefiningBlocks.insert(PBranch->From); + } else if (auto *PSwitch = dyn_cast(PInfo)) { + DefiningBlocks.insert(PSwitch->From); + } + } + + if (DefiningBlocks.size() > 1) { + SmallPtrSet PHIBlocks; + needsPHIInsertion(Op, DefiningBlocks, PHIBlocks); + for (BasicBlock *PHIBlock : PHIBlocks) { + PHICandidates[PHIBlock].push_back(Op); + } + } + } +} + +void PredicateInfoBuilder::needsPHIInsertion( + Value *Op, const SmallPtrSetImpl &DefiningBlocks, + SmallPtrSetImpl &PHIBlocks) { + IDFCalculator IDF(DT); + IDF.setDefiningBlocks(DefiningBlocks); + + SmallPtrSet LiveInBlocks; + for (auto CurUser = Op->users().begin(); CurUser != Op->users().end(); + ++CurUser) { + if (auto *I = dyn_cast(*CurUser)) { + LiveInBlocks.insert(I->getParent()); + } + } + IDF.setLiveInBlocks(LiveInBlocks); + + SmallVector IDFBlocks; + IDF.calculate(IDFBlocks); + + for (const auto &IDFBlock : IDFBlocks) { + PHIBlocks.insert(std::move(IDFBlock)); + } +} + +void PredicateInfoBuilder::insertPredicatePHIs( + Value *Op, BasicBlock *PHIBlock, SmallVectorImpl &OpsToRename) { + IRBuilder<> Builder(&PHIBlock->front()); + + PHINode *PHI = Builder.CreatePHI(Op->getType(), pred_size(PHIBlock), + Op->getName() + ".predicate.phi"); + PredicatePHI *PPHI = new (Allocator) PredicatePHI(Op, PHIBlock); + PPHI->RenamedOp = PHI; + + const auto &ValueInfo = getValueInfo(Op); + + for (BasicBlock *Pred : predecessors(PHIBlock)) { + Value *IncomingValue = nullptr; + for (const auto *PInfo : ValueInfo.Infos) { + if (auto *PBranch = dyn_cast(PInfo)) { + if (PBranch->From == Pred && PBranch->To == PHIBlock) { + PPHI->IncomingPredicates.push_back( + {Pred, const_cast(PInfo)}); + IncomingValue = PBranch->OriginalOp; + } + } else if (auto *PSwitch = dyn_cast(PInfo)) { + if (PSwitch->From == Pred && PSwitch->To == PHIBlock) { + PPHI->IncomingPredicates.push_back( + {Pred, const_cast(PInfo)}); + IncomingValue = PSwitch->OriginalOp; + } + } + } + if (IncomingValue) { + PHI->addIncoming(IncomingValue, Pred); + } else { + PHI->eraseFromParent(); + PPHI->RenamedOp = nullptr; + PPHI->IncomingPredicates.clear(); + return; + } + } + + addInfoFor(OpsToRename, Op, PPHI); +} + +void PredicateInfoBuilder::processPredicatePHIs( + SmallVectorImpl &OpsToRename) { + for (const auto &PHICandidate : PHICandidates) { + BasicBlock *PHIBlock = PHICandidate.first; + for (Value *Op : PHICandidate.second) { + insertPredicatePHIs(Op, PHIBlock, OpsToRename); + } + } +} + // Build predicate info for our function void PredicateInfoBuilder::buildPredicateInfo() { DT.updateDFSNumbers(); @@ -487,6 +595,10 @@ void PredicateInfoBuilder::buildPredicateInfo() { if (DT.isReachableFromEntry(II->getParent())) processAssume(II, II->getParent(), OpsToRename); } + + identifyPHICandidates(OpsToRename); + processPredicatePHIs(OpsToRename); + // Now rename all our operations. renameUses(OpsToRename); } @@ -530,6 +642,13 @@ Value *PredicateInfoBuilder::materializeStack(unsigned int &Counter, Op->getName() + "." + Twine(Counter++)); PI.PredicateMap.insert({PIC, ValInfo}); Result.Def = PIC; + } else if (isa(ValInfo)) { + auto *PPHI = dyn_cast(ValInfo); + IRBuilder<> B(&*PPHI->PHIBlock->getFirstInsertionPt()); + BitCastInst *PIC = + CreateSSACopy(&*PPHI->PHIBlock->getFirstInsertionPt(), Op); + PI.PredicateMap.insert({PIC, ValInfo}); + Result.Def = PIC; } else { auto *PAssume = dyn_cast(ValInfo); assert(PAssume && @@ -617,6 +736,15 @@ void PredicateInfoBuilder::renameUses(SmallVectorImpl &OpsToRename) { OrderedUses.push_back(VD); } } + } else if (const auto *PPHI = dyn_cast(PossibleCopy)) { + VD.LocalNum = LN_First; + auto *DomNode = DT.getNode(PPHI->PHIBlock); + if (DomNode) { + VD.DFSIn = DomNode->getDFSNumIn(); + VD.DFSOut = DomNode->getDFSNumOut(); + VD.PInfo = PossibleCopy; + OrderedUses.push_back(VD); + } } } @@ -755,10 +883,31 @@ std::optional PredicateBase::getConstraint() const { } return {{CmpInst::ICMP_EQ, cast(this)->CaseValue}}; + case PT_PHI: + return cast(this)->getConstraint(); } llvm_unreachable("Unknown predicate type"); } +std::optional PredicatePHI::getConstraint() const { + if (IncomingPredicates.empty()) + return std::nullopt; + + auto FirstConstraint = IncomingPredicates[0].second->getConstraint(); + if (!FirstConstraint) + return std::nullopt; + + for (size_t I = 1; I < IncomingPredicates.size(); ++I) { + auto Constraint = IncomingPredicates[I].second->getConstraint(); + if (!Constraint || Constraint->Predicate != FirstConstraint->Predicate || + Constraint->OtherOp != FirstConstraint->OtherOp) { + return std::nullopt; + } + } + + return FirstConstraint; +} + void PredicateInfo::verifyPredicateInfo() const {} // Replace bitcasts created by PredicateInfo with their operand. @@ -821,6 +970,10 @@ class PredicateInfoAnnotatedWriter : public AssemblyAnnotationWriter { } else if (const auto *PA = dyn_cast(PI)) { OS << "; assume predicate info {" << " Comparison:" << *PA->Condition; + } else if (const auto *PP = dyn_cast(PI)) { + OS << "; phi predicate info { PHIBlock: "; + PP->PHIBlock->printAsOperand(OS); + OS << " IncomingEdges: " << PP->IncomingPredicates.size(); } OS << ", RenamedOp: "; PI->RenamedOp->printAsOperand(OS, false); diff --git a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll index b4f8abbf83a53..3b194a88c7ea0 100644 --- a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll +++ b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll @@ -68,10 +68,10 @@ define i32 @foo(ptr nocapture %var1, ptr nocapture readnone %var2, ptr nocapture ; CHECK-NEXT: [[ADD8]] = add nsw i32 [[ADD86]], [[ADD]] ; CHECK-NEXT: [[INC]] = add nuw i32 [[J_113]], 1 ; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i32 [[INC]], [[ITR]] -; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_BODY3]], label [[FOR_INC11_LOOPEXIT_LOOPEXIT5:%.*]], !llvm.loop [[LOOP7:![0-9]+]] +; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_BODY3]], label [[FOR_INC11_LOOPEXIT_LOOPEXIT7:%.*]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK: for.inc11.loopexit.loopexit: ; CHECK-NEXT: br label [[FOR_INC11_LOOPEXIT:%.*]] -; CHECK: for.inc11.loopexit.loopexit5: +; CHECK: for.inc11.loopexit.loopexit7: ; CHECK-NEXT: [[ADD8_LCSSA:%.*]] = phi i32 [ [[ADD8]], [[FOR_BODY3]] ] ; CHECK-NEXT: store i32 [[ADD8_LCSSA]], ptr [[ARRAYIDX7]], align 4, !alias.scope [[META2]] ; CHECK-NEXT: br label [[FOR_INC11_LOOPEXIT]] diff --git a/llvm/test/Transforms/NewGVN/phi-predicate-opt.ll b/llvm/test/Transforms/NewGVN/phi-predicate-opt.ll new file mode 100644 index 0000000000000..1164cce689f02 --- /dev/null +++ b/llvm/test/Transforms/NewGVN/phi-predicate-opt.ll @@ -0,0 +1,39 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -passes=newgvn -S | FileCheck %s + +@p = local_unnamed_addr global i32 0 + +define noundef i32 @h5diff(i32 %0, i1 %1) local_unnamed_addr #0 { +; CHECK-LABEL: @h5diff( +; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[TMP0:%.*]], 0 +; CHECK-NEXT: br i1 [[TMP1:%.*]], label [[TMP3:%.*]], label [[TMP4:%.*]] +; CHECK: 3: +; CHECK-NEXT: store i32 1, ptr @p, align 4 +; CHECK-NEXT: br i1 [[COND]], label [[TMP5:%.*]], label [[COMMON_RET:%.*]] +; CHECK: common.ret: +; CHECK-NEXT: ret i32 0 +; CHECK: 4: +; CHECK-NEXT: store i32 2, ptr @p, align 4 +; CHECK-NEXT: br i1 [[COND]], label [[TMP5]], label [[COMMON_RET]] +; CHECK: 5: +; CHECK-NEXT: store i32 0, ptr @p, align 4 +; CHECK-NEXT: br label [[COMMON_RET]] +; + %cond = icmp eq i32 %0, 0 + br i1 %1, label %3, label %4 + +3: + store i32 1, ptr @p, align 4 + br i1 %cond, label %5, label %common.ret + +common.ret: + ret i32 0 + +4: + store i32 2, ptr @p, align 4 + br i1 %cond, label %5, label %common.ret + +5: + store i32 %0, ptr @p, align 4 + br label %common.ret +} diff --git a/llvm/test/Transforms/NewGVN/pr33116.ll b/llvm/test/Transforms/NewGVN/pr33116.ll index a0dc3b2382f91..3d272d3efaaf3 100644 --- a/llvm/test/Transforms/NewGVN/pr33116.ll +++ b/llvm/test/Transforms/NewGVN/pr33116.ll @@ -9,9 +9,9 @@ define void @b(i1 %arg) { ; CHECK: while.d: ; CHECK-NEXT: br label [[F:%.*]] ; CHECK: f: -; CHECK-NEXT: br i1 %arg, label [[IF_E:%.*]], label [[C]] +; CHECK-NEXT: br i1 [[ARG:%.*]], label [[IF_E:%.*]], label [[C]] ; CHECK: c: -; CHECK-NEXT: br i1 %arg, label [[IF_G:%.*]], label [[IF_E]] +; CHECK-NEXT: br i1 [[ARG]], label [[IF_G:%.*]], label [[IF_E]] ; CHECK: if.g: ; CHECK-NEXT: store i32 undef, ptr @a, align 4 ; CHECK-NEXT: br label [[WHILE_D]] diff --git a/llvm/test/Transforms/NewGVN/pr33204.ll b/llvm/test/Transforms/NewGVN/pr33204.ll index a57544cc7f557..6a76444420f93 100644 --- a/llvm/test/Transforms/NewGVN/pr33204.ll +++ b/llvm/test/Transforms/NewGVN/pr33204.ll @@ -24,9 +24,9 @@ define void @hoge(i32 %arg, i1 %arg2) { ; CHECK-NEXT: unreachable ; CHECK: bb6: ; CHECK-NEXT: store i32 [[TMP]], ptr @global.1, align 4, !h [[META0]] -; CHECK-NEXT: br i1 %arg2, label [[BB7:%.*]], label [[BB1]] +; CHECK-NEXT: br i1 [[ARG2:%.*]], label [[BB7:%.*]], label [[BB1]] ; CHECK: bb7: -; CHECK-NEXT: br i1 %arg2, label [[BB10:%.*]], label [[BB8:%.*]] +; CHECK-NEXT: br i1 [[ARG2]], label [[BB10:%.*]], label [[BB8:%.*]] ; CHECK: bb8: ; CHECK-NEXT: br i1 false, label [[BB9:%.*]], label [[BB3:%.*]] ; CHECK: bb9: diff --git a/llvm/test/Transforms/NewGVN/pr33720.ll b/llvm/test/Transforms/NewGVN/pr33720.ll index 5621148a12068..ddd4317a6d2b5 100644 --- a/llvm/test/Transforms/NewGVN/pr33720.ll +++ b/llvm/test/Transforms/NewGVN/pr33720.ll @@ -8,11 +8,11 @@ define void @patatino(i1 %arg) { ; CHECK-LABEL: @patatino( ; CHECK-NEXT: entry: -; CHECK-NEXT: br i1 %arg, label [[IF_END24:%.*]], label [[FOR_COND16:%.*]] +; CHECK-NEXT: br i1 [[ARG:%.*]], label [[IF_END24:%.*]], label [[FOR_COND16:%.*]] ; CHECK: for.cond2thread-pre-split: ; CHECK-NEXT: br i1 false, label [[FOR_BODY:%.*]], label [[FOR_COND8_PREHEADER:%.*]] ; CHECK: for.cond8.preheader: -; CHECK-NEXT: br i1 %arg, label [[L1:%.*]], label [[FOR_COND11THREAD_PRE_SPLIT_LR_PH:%.*]] +; CHECK-NEXT: br i1 [[ARG]], label [[L1:%.*]], label [[FOR_COND11THREAD_PRE_SPLIT_LR_PH:%.*]] ; CHECK: for.cond11thread-pre-split.lr.ph: ; CHECK-NEXT: br label [[L1]] ; CHECK: for.body: @@ -32,7 +32,7 @@ define void @patatino(i1 %arg) { ; CHECK-NEXT: br label [[FOR_BODY]] ; CHECK: for.cond16: ; CHECK-NEXT: [[J_0:%.*]] = phi ptr [ @f, [[ENTRY:%.*]] ], [ poison, [[FOR_COND20:%.*]] ], [ @e, [[FOR_COND16]] ] -; CHECK-NEXT: br i1 %arg, label [[FOR_COND20]], label [[FOR_COND16]] +; CHECK-NEXT: br i1 [[ARG]], label [[FOR_COND20]], label [[FOR_COND16]] ; CHECK: for.cond20: ; CHECK-NEXT: [[J_2:%.*]] = phi ptr [ [[J_0]], [[FOR_COND16]] ], [ poison, [[IF_END24]] ] ; CHECK-NEXT: br i1 true, label [[IF_END24]], label [[FOR_COND16]] diff --git a/llvm/test/Transforms/Util/PredicateInfo/phi-predicate.ll b/llvm/test/Transforms/Util/PredicateInfo/phi-predicate.ll new file mode 100644 index 0000000000000..0b001d4e4b8fb --- /dev/null +++ b/llvm/test/Transforms/Util/PredicateInfo/phi-predicate.ll @@ -0,0 +1,43 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -passes=print-predicateinfo < %s 2>&1 >/dev/null | FileCheck %s + +@p = local_unnamed_addr global i32 0 + +define noundef i32 @h5diff(i32 %0, i1 %1) local_unnamed_addr #0 { +; CHECK-LABEL: @h5diff( +; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[TMP0:%.*]], 0 +; CHECK-NEXT: br i1 [[TMP1:%.*]], label [[TMP3:%.*]], label [[TMP4:%.*]] +; CHECK: 3: +; CHECK-NEXT: store i32 1, ptr @p, align 4 +; CHECK: [[DOT0:%.*]] = bitcast i32 [[TMP0]] to i32 +; CHECK-NEXT: br i1 [[COND]], label [[TMP5:%.*]], label [[COMMON_RET:%.*]] +; CHECK: common.ret: +; CHECK-NEXT: ret i32 0 +; CHECK: 4: +; CHECK-NEXT: store i32 2, ptr @p, align 4 +; CHECK: [[DOT1:%.*]] = bitcast i32 [[TMP0]] to i32 +; CHECK-NEXT: br i1 [[COND]], label [[TMP5]], label [[COMMON_RET]] +; CHECK: 5: +; CHECK-NEXT: [[DOTPREDICATE_PHI:%.*]] = phi i32 [ [[DOT1]], [[TMP4]] ], [ [[DOT0]], [[TMP3]] ] +; CHECK: [[TMP6:%.*]] = bitcast i32 [[TMP0]] to i32 +; CHECK-NEXT: store i32 [[TMP6]], ptr @p, align 4 +; CHECK-NEXT: br label [[COMMON_RET]] +; + %cond = icmp eq i32 %0, 0 + br i1 %1, label %3, label %4 + +3: + store i32 1, ptr @p, align 4 + br i1 %cond, label %5, label %common.ret + +common.ret: + ret i32 0 + +4: + store i32 2, ptr @p, align 4 + br i1 %cond, label %5, label %common.ret + +5: + store i32 %0, ptr @p, align 4 + br label %common.ret +}