Skip to content

Commit 3e9069e

Browse files
MatzeBMatzeB
committed
BranchProbabilityInfo: Report branch_weights 1, 0 for invoke
The `calcEstimatedHeuristics` used to report probabilities of invoke exception paths as `BlockExecWeight::UNWIND` in relation to `BlockExecWeight::DEFAULT`. This is "just" a 0xfffff : 1 ratio which can lead to overestimated block counts for loops with high execution counts. For example for a loop with a trip count of 5 million the exception loop exit was estimated as 4 times more likely than the regular loop exit. This change circumvents the limited range of `calcEstimatedHeuristics` by hardcoding the probabilities of exception edge as 1, 0. This also matches the behavior when loading PGO profiles.
1 parent 0d77978 commit 3e9069e

File tree

6 files changed

+40
-22
lines changed

6 files changed

+40
-22
lines changed

‎llvm/include/llvm/Analysis/BranchProbabilityInfo.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#ifndef LLVM_ANALYSIS_BRANCHPROBABILITYINFO_H
1414
#define LLVM_ANALYSIS_BRANCHPROBABILITYINFO_H
1515

16+
#include "llvm/ADT/ArrayRef.h"
1617
#include "llvm/ADT/DenseMap.h"
1718
#include "llvm/ADT/DenseMapInfo.h"
1819
#include "llvm/ADT/DenseSet.h"
@@ -181,7 +182,7 @@ class BranchProbabilityInfo {
181182
/// can be used when updating the CFG to update the branch probability
182183
/// information.
183184
void setEdgeProbability(const BasicBlock *Src,
184-
const SmallVectorImpl<BranchProbability> &Probs);
185+
ArrayRef<BranchProbability> Probs);
185186

186187
/// Copy outgoing edge probabilities from \p Src to \p Dst.
187188
///
@@ -406,6 +407,7 @@ class BranchProbabilityInfo {
406407
/// Based on computed weights by \p computeEstimatedBlockWeight set
407408
/// probabilities on branches.
408409
bool calcEstimatedHeuristics(const BasicBlock *BB);
410+
bool calcFixedWeights(const BasicBlock *BB);
409411
bool calcMetadataWeights(const BasicBlock *BB);
410412
bool calcPointerHeuristics(const BasicBlock *BB);
411413
bool calcZeroHeuristics(const BasicBlock *BB, const TargetLibraryInfo *TLI);

‎llvm/lib/Analysis/BranchProbabilityInfo.cpp‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -870,6 +870,20 @@ void BranchProbabilityInfo::computeEestimateBlockWeight(
870870
} while (!BlockWorkList.empty() || !LoopWorkList.empty());
871871
}
872872

873+
bool BranchProbabilityInfo::calcFixedWeights(const BasicBlock *BB) {
874+
const Instruction *Terminator = BB->getTerminator();
875+
if (const InvokeInst *Invoke = dyn_cast<InvokeInst>(Terminator)) {
876+
assert(Invoke->getNormalDest() == Invoke->getSuccessor(0) &&
877+
Invoke->getUnwindDest() == Invoke->getSuccessor(1) &&
878+
"unexpected successor ordering");
879+
const BranchProbability BP[] = {BranchProbability::getOne(),
880+
BranchProbability::getZero()};
881+
setEdgeProbability(BB, BP);
882+
return true;
883+
}
884+
return false;
885+
}
886+
873887
// Calculate edge probabilities based on block's estimated weight.
874888
// Note that gathered weights were not scaled for loops. Thus edges entering
875889
// and exiting loops requires special processing.
@@ -1130,7 +1144,7 @@ BranchProbabilityInfo::getEdgeProbability(const BasicBlock *Src,
11301144

11311145
/// Set the edge probability for all edges at once.
11321146
void BranchProbabilityInfo::setEdgeProbability(
1133-
const BasicBlock *Src, const SmallVectorImpl<BranchProbability> &Probs) {
1147+
const BasicBlock *Src, ArrayRef<BranchProbability> Probs) {
11341148
assert(Src->getTerminator()->getNumSuccessors() == Probs.size());
11351149
eraseBlock(Src); // Erase stale data if any.
11361150
if (Probs.size() == 0)
@@ -1256,6 +1270,8 @@ void BranchProbabilityInfo::calculate(const Function &F, const LoopInfo &LoopI,
12561270
continue;
12571271
if (calcMetadataWeights(BB))
12581272
continue;
1273+
if (calcFixedWeights(BB))
1274+
continue;
12591275
if (calcEstimatedHeuristics(BB))
12601276
continue;
12611277
if (calcPointerHeuristics(BB))

‎llvm/test/Analysis/BranchProbabilityInfo/basic.ll‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -243,8 +243,8 @@ entry:
243243
if.then:
244244
invoke i32 @InvokeCall()
245245
to label %invoke.cont unwind label %lpad
246-
; CHECK: edge if.then -> invoke.cont probability is 0x7fff8000 / 0x80000000 = 100.00% [HOT edge]
247-
; CHECK: edge if.then -> lpad probability is 0x00008000 / 0x80000000 = 0.00%
246+
; CHECK: edge if.then -> invoke.cont probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
247+
; CHECK: edge if.then -> lpad probability is 0x00000000 / 0x80000000 = 0.00%
248248

249249
invoke.cont:
250250
call void @ColdFunc() #0
@@ -270,8 +270,8 @@ if.then:
270270
invoke i32 @InvokeCall()
271271
to label %invoke.cont unwind label %lpad
272272
; The cold call heuristic should not kick in when the cold callsite is in EH path.
273-
; CHECK: edge if.then -> invoke.cont probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge]
274-
; CHECK: edge if.then -> lpad probability is 0x00000800 / 0x80000000 = 0.00%
273+
; CHECK: edge if.then -> invoke.cont probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
274+
; CHECK: edge if.then -> lpad probability is 0x00000000 / 0x80000000 = 0.00%
275275

276276
invoke.cont:
277277
br label %if.end
@@ -298,8 +298,8 @@ if.then:
298298
to label %invoke.cont unwind label %lpad
299299
; Regardless of cold calls, edge weights from a invoke instruction should be
300300
; determined by the invoke heuristic.
301-
; CHECK: edge if.then -> invoke.cont probability is 0x7fff8000 / 0x80000000 = 100.00% [HOT edge]
302-
; CHECK: edge if.then -> lpad probability is 0x00008000 / 0x80000000 = 0.00%
301+
; CHECK: edge if.then -> invoke.cont probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
302+
; CHECK: edge if.then -> lpad probability is 0x00000000 / 0x80000000 = 0.00%
303303

304304
invoke.cont:
305305
call void @ColdFunc() #0
@@ -318,13 +318,13 @@ if.end:
318318
; CHECK-LABEL: test_invoke_code_profiled
319319
define void @test_invoke_code_profiled(i1 %c) personality ptr @__gxx_personality_v0 {
320320
entry:
321-
; CHECK: edge entry -> invoke.to0 probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge]
322-
; CHECK: edge entry -> lpad probability is 0x00000800 / 0x80000000 = 0.00%
321+
; CHECK: edge entry -> invoke.to0 probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
322+
; CHECK: edge entry -> lpad probability is 0x00000000 / 0x80000000 = 0.00%
323323
invoke i32 @InvokeCall() to label %invoke.to0 unwind label %lpad
324324

325325
invoke.to0:
326-
; CHECK: edge invoke.to0 -> invoke.to1 probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge]
327-
; CHECK: edge invoke.to0 -> lpad probability is 0x00000800 / 0x80000000 = 0.00%
326+
; CHECK: edge invoke.to0 -> invoke.to1 probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
327+
; CHECK: edge invoke.to0 -> lpad probability is 0x00000000 / 0x80000000 = 0.00%
328328
invoke i32 @InvokeCall() to label %invoke.to1 unwind label %lpad,
329329
!prof !{!"branch_weights", i32 444}
330330

‎llvm/test/Analysis/BranchProbabilityInfo/deopt-invoke.ll‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ declare void @cold() cold
1111
define void @test1(i32 %0) personality ptr @"personality_function" !prof !1 {
1212
;CHECK: edge entry -> unreached probability is 0x00000001 / 0x80000000 = 0.00%
1313
;CHECK: edge entry -> invoke probability is 0x7fffffff / 0x80000000 = 100.00% [HOT edge]
14-
;CHECK: edge invoke -> invoke.cont.unreached probability is 0x00000000 / 0x80000000 = 0.00%
15-
;CHECK: edge invoke -> land.pad probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
14+
;CHECK: edge invoke -> invoke.cont.unreached probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
15+
;CHECK: edge invoke -> land.pad probability is 0x00000000 / 0x80000000 = 0.00%
1616
;CHECK: edge land.pad -> exit probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
1717

1818
entry:
@@ -41,8 +41,8 @@ exit:
4141
define void @test2(i32 %0) personality ptr @"personality_function" {
4242
;CHECK: edge entry -> unreached probability is 0x00000000 / 0x80000000 = 0.00%
4343
;CHECK: edge entry -> invoke probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
44-
;CHECK: edge invoke -> invoke.cont.cold probability is 0x7fff8000 / 0x80000000 = 100.00% [HOT edge]
45-
;CHECK: edge invoke -> land.pad probability is 0x00008000 / 0x80000000 = 0.00%
44+
;CHECK: edge invoke -> invoke.cont.cold probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
45+
;CHECK: edge invoke -> land.pad probability is 0x00000000 / 0x80000000 = 0.00%
4646
;CHECK: edge land.pad -> exit probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
4747

4848
entry:
@@ -71,8 +71,8 @@ exit:
7171
define void @test3(i32 %0) personality ptr @"personality_function" {
7272
;CHECK: edge entry -> unreached probability is 0x00000000 / 0x80000000 = 0.00%
7373
;CHECK: edge entry -> invoke probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
74-
;CHECK: edge invoke -> invoke.cont.cold probability is 0x7fff8000 / 0x80000000 = 100.00% [HOT edge]
75-
;CHECK: edge invoke -> land.pad probability is 0x00008000 / 0x80000000 = 0.00%
74+
;CHECK: edge invoke -> invoke.cont.cold probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
75+
;CHECK: edge invoke -> land.pad probability is 0x00000000 / 0x80000000 = 0.00%
7676
;CHECK: edge land.pad -> exit probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
7777
entry:
7878
br i1 undef, label %unreached, label %invoke

‎llvm/test/Analysis/BranchProbabilityInfo/loop.ll‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -499,8 +499,8 @@ loop:
499499
%i.0 = phi i32 [ 0, %entry ], [ %inc, %invoke.cont ]
500500
invoke i32 @InvokeCall()
501501
to label %invoke.cont unwind label %lpad
502-
; CHECK: edge loop -> invoke.cont probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge]
503-
; CHECK: edge loop -> lpad probability is 0x00000800 / 0x80000000 = 0.00%
502+
; CHECK: edge loop -> invoke.cont probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
503+
; CHECK: edge loop -> lpad probability is 0x00000000 / 0x80000000 = 0.00%
504504

505505
invoke.cont:
506506
%inc = add nsw i32 %i.0, 1

‎llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -118,8 +118,8 @@ if.then: ; preds = %entry
118118
%exception = call ptr @__cxa_allocate_exception(i64 1) #0
119119
invoke i32 @smallFunction(i32 %idx)
120120
to label %invoke.cont unwind label %lpad
121-
; CHECK: edge if.then -> invoke.cont probability is 0x40000000 / 0x80000000 = 50.00%
122-
; CHECK: edge if.then -> lpad probability is 0x40000000 / 0x80000000 = 50.00%
121+
; CHECK: edge if.then -> invoke.cont probability is 0x80000000 / 0x80000000 = 100.00%
122+
; CHECK: edge if.then -> lpad probability is 0x00000000 / 0x80000000 = 0.00%
123123

124124
invoke.cont: ; preds = %if.then
125125
call void @__cxa_throw(ptr %exception, ptr @_ZTIi, ptr null) #1

0 commit comments

Comments
 (0)