diff --git a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h index fb02997371bfb..51d7a729a7bde 100644 --- a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h +++ b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h @@ -13,6 +13,7 @@ #ifndef LLVM_ANALYSIS_BRANCHPROBABILITYINFO_H #define LLVM_ANALYSIS_BRANCHPROBABILITYINFO_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/DenseSet.h" @@ -181,7 +182,7 @@ class BranchProbabilityInfo { /// can be used when updating the CFG to update the branch probability /// information. void setEdgeProbability(const BasicBlock *Src, - const SmallVectorImpl &Probs); + ArrayRef Probs); /// Copy outgoing edge probabilities from \p Src to \p Dst. /// @@ -406,6 +407,7 @@ class BranchProbabilityInfo { /// Based on computed weights by \p computeEstimatedBlockWeight set /// probabilities on branches. bool calcEstimatedHeuristics(const BasicBlock *BB); + bool calcFixedWeights(const BasicBlock *BB); bool calcMetadataWeights(const BasicBlock *BB); bool calcPointerHeuristics(const BasicBlock *BB); bool calcZeroHeuristics(const BasicBlock *BB, const TargetLibraryInfo *TLI); diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp index b45deccd913db..bc229977970a7 100644 --- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp +++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp @@ -870,6 +870,20 @@ void BranchProbabilityInfo::computeEestimateBlockWeight( } while (!BlockWorkList.empty() || !LoopWorkList.empty()); } +bool BranchProbabilityInfo::calcFixedWeights(const BasicBlock *BB) { + const Instruction *Terminator = BB->getTerminator(); + if (const InvokeInst *Invoke = dyn_cast(Terminator)) { + assert(Invoke->getNormalDest() == Invoke->getSuccessor(0) && + Invoke->getUnwindDest() == Invoke->getSuccessor(1) && + "unexpected successor ordering"); + const BranchProbability BP[] = {BranchProbability::getOne(), + BranchProbability::getZero()}; + setEdgeProbability(BB, BP); + return true; + } + return false; +} + // Calculate edge probabilities based on block's estimated weight. // Note that gathered weights were not scaled for loops. Thus edges entering // and exiting loops requires special processing. @@ -1130,7 +1144,7 @@ BranchProbabilityInfo::getEdgeProbability(const BasicBlock *Src, /// Set the edge probability for all edges at once. void BranchProbabilityInfo::setEdgeProbability( - const BasicBlock *Src, const SmallVectorImpl &Probs) { + const BasicBlock *Src, ArrayRef Probs) { assert(Src->getTerminator()->getNumSuccessors() == Probs.size()); eraseBlock(Src); // Erase stale data if any. if (Probs.size() == 0) @@ -1256,6 +1270,8 @@ void BranchProbabilityInfo::calculate(const Function &F, const LoopInfo &LoopI, continue; if (calcMetadataWeights(BB)) continue; + if (calcFixedWeights(BB)) + continue; if (calcEstimatedHeuristics(BB)) continue; if (calcPointerHeuristics(BB)) diff --git a/llvm/test/Analysis/BranchProbabilityInfo/basic.ll b/llvm/test/Analysis/BranchProbabilityInfo/basic.ll index 682de24ac5828..90d34a418d98d 100644 --- a/llvm/test/Analysis/BranchProbabilityInfo/basic.ll +++ b/llvm/test/Analysis/BranchProbabilityInfo/basic.ll @@ -243,8 +243,8 @@ entry: if.then: invoke i32 @InvokeCall() to label %invoke.cont unwind label %lpad -; CHECK: edge if.then -> invoke.cont probability is 0x7fff8000 / 0x80000000 = 100.00% [HOT edge] -; CHECK: edge if.then -> lpad probability is 0x00008000 / 0x80000000 = 0.00% +; CHECK: edge if.then -> invoke.cont probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] +; CHECK: edge if.then -> lpad probability is 0x00000000 / 0x80000000 = 0.00% invoke.cont: call void @ColdFunc() #0 @@ -270,8 +270,8 @@ if.then: invoke i32 @InvokeCall() to label %invoke.cont unwind label %lpad ; The cold call heuristic should not kick in when the cold callsite is in EH path. -; CHECK: edge if.then -> invoke.cont probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge] -; CHECK: edge if.then -> lpad probability is 0x00000800 / 0x80000000 = 0.00% +; CHECK: edge if.then -> invoke.cont probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] +; CHECK: edge if.then -> lpad probability is 0x00000000 / 0x80000000 = 0.00% invoke.cont: br label %if.end @@ -298,8 +298,8 @@ if.then: to label %invoke.cont unwind label %lpad ; Regardless of cold calls, edge weights from a invoke instruction should be ; determined by the invoke heuristic. -; CHECK: edge if.then -> invoke.cont probability is 0x7fff8000 / 0x80000000 = 100.00% [HOT edge] -; CHECK: edge if.then -> lpad probability is 0x00008000 / 0x80000000 = 0.00% +; CHECK: edge if.then -> invoke.cont probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] +; CHECK: edge if.then -> lpad probability is 0x00000000 / 0x80000000 = 0.00% invoke.cont: call void @ColdFunc() #0 @@ -318,13 +318,13 @@ if.end: ; CHECK-LABEL: test_invoke_code_profiled define void @test_invoke_code_profiled(i1 %c) personality ptr @__gxx_personality_v0 { entry: -; CHECK: edge entry -> invoke.to0 probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge] -; CHECK: edge entry -> lpad probability is 0x00000800 / 0x80000000 = 0.00% +; CHECK: edge entry -> invoke.to0 probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] +; CHECK: edge entry -> lpad probability is 0x00000000 / 0x80000000 = 0.00% invoke i32 @InvokeCall() to label %invoke.to0 unwind label %lpad invoke.to0: -; CHECK: edge invoke.to0 -> invoke.to1 probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge] -; CHECK: edge invoke.to0 -> lpad probability is 0x00000800 / 0x80000000 = 0.00% +; CHECK: edge invoke.to0 -> invoke.to1 probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] +; CHECK: edge invoke.to0 -> lpad probability is 0x00000000 / 0x80000000 = 0.00% invoke i32 @InvokeCall() to label %invoke.to1 unwind label %lpad, !prof !{!"branch_weights", i32 444} diff --git a/llvm/test/Analysis/BranchProbabilityInfo/deopt-invoke.ll b/llvm/test/Analysis/BranchProbabilityInfo/deopt-invoke.ll index ae46ed7851f89..1679d0903ef90 100644 --- a/llvm/test/Analysis/BranchProbabilityInfo/deopt-invoke.ll +++ b/llvm/test/Analysis/BranchProbabilityInfo/deopt-invoke.ll @@ -11,8 +11,8 @@ declare void @cold() cold define void @test1(i32 %0) personality ptr @"personality_function" !prof !1 { ;CHECK: edge entry -> unreached probability is 0x00000001 / 0x80000000 = 0.00% ;CHECK: edge entry -> invoke probability is 0x7fffffff / 0x80000000 = 100.00% [HOT edge] -;CHECK: edge invoke -> invoke.cont.unreached probability is 0x00000000 / 0x80000000 = 0.00% -;CHECK: edge invoke -> land.pad probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] +;CHECK: edge invoke -> invoke.cont.unreached probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] +;CHECK: edge invoke -> land.pad probability is 0x00000000 / 0x80000000 = 0.00% ;CHECK: edge land.pad -> exit probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] entry: @@ -41,8 +41,8 @@ exit: define void @test2(i32 %0) personality ptr @"personality_function" { ;CHECK: edge entry -> unreached probability is 0x00000000 / 0x80000000 = 0.00% ;CHECK: edge entry -> invoke probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] -;CHECK: edge invoke -> invoke.cont.cold probability is 0x7fff8000 / 0x80000000 = 100.00% [HOT edge] -;CHECK: edge invoke -> land.pad probability is 0x00008000 / 0x80000000 = 0.00% +;CHECK: edge invoke -> invoke.cont.cold probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] +;CHECK: edge invoke -> land.pad probability is 0x00000000 / 0x80000000 = 0.00% ;CHECK: edge land.pad -> exit probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] entry: @@ -71,8 +71,8 @@ exit: define void @test3(i32 %0) personality ptr @"personality_function" { ;CHECK: edge entry -> unreached probability is 0x00000000 / 0x80000000 = 0.00% ;CHECK: edge entry -> invoke probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] -;CHECK: edge invoke -> invoke.cont.cold probability is 0x7fff8000 / 0x80000000 = 100.00% [HOT edge] -;CHECK: edge invoke -> land.pad probability is 0x00008000 / 0x80000000 = 0.00% +;CHECK: edge invoke -> invoke.cont.cold probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] +;CHECK: edge invoke -> land.pad probability is 0x00000000 / 0x80000000 = 0.00% ;CHECK: edge land.pad -> exit probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] entry: br i1 undef, label %unreached, label %invoke diff --git a/llvm/test/Analysis/BranchProbabilityInfo/loop.ll b/llvm/test/Analysis/BranchProbabilityInfo/loop.ll index c2aa705d34957..562d979ed0b7e 100644 --- a/llvm/test/Analysis/BranchProbabilityInfo/loop.ll +++ b/llvm/test/Analysis/BranchProbabilityInfo/loop.ll @@ -499,8 +499,8 @@ loop: %i.0 = phi i32 [ 0, %entry ], [ %inc, %invoke.cont ] invoke i32 @InvokeCall() to label %invoke.cont unwind label %lpad -; CHECK: edge loop -> invoke.cont probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge] -; CHECK: edge loop -> lpad probability is 0x00000800 / 0x80000000 = 0.00% +; CHECK: edge loop -> invoke.cont probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] +; CHECK: edge loop -> lpad probability is 0x00000000 / 0x80000000 = 0.00% invoke.cont: %inc = add nsw i32 %i.0, 1 diff --git a/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll b/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll index e4bcdfc082ff9..028f1070b8dc5 100644 --- a/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll +++ b/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll @@ -118,8 +118,8 @@ if.then: ; preds = %entry %exception = call ptr @__cxa_allocate_exception(i64 1) #0 invoke i32 @smallFunction(i32 %idx) to label %invoke.cont unwind label %lpad -; CHECK: edge if.then -> invoke.cont probability is 0x40000000 / 0x80000000 = 50.00% -; CHECK: edge if.then -> lpad probability is 0x40000000 / 0x80000000 = 50.00% +; CHECK: edge if.then -> invoke.cont probability is 0x80000000 / 0x80000000 = 100.00% +; CHECK: edge if.then -> lpad probability is 0x00000000 / 0x80000000 = 0.00% invoke.cont: ; preds = %if.then call void @__cxa_throw(ptr %exception, ptr @_ZTIi, ptr null) #1