Skip to content

Commit 49ebb71

Browse files
mtrofinmtrofin
committed
[SLU][profcheck] create likely branch weights for guard->branch
1 parent 0762b0d commit 49ebb71

File tree

2 files changed

+139
-54
lines changed

2 files changed

+139
-54
lines changed

‎llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
#include "llvm/IR/Instruction.h"
4141
#include "llvm/IR/Instructions.h"
4242
#include "llvm/IR/IntrinsicInst.h"
43+
#include "llvm/IR/MDBuilder.h"
4344
#include "llvm/IR/Module.h"
4445
#include "llvm/IR/PatternMatch.h"
4546
#include "llvm/IR/ProfDataUtils.h"
@@ -2829,9 +2830,14 @@ static BranchInst *turnGuardIntoBranch(IntrinsicInst *GI, Loop &L,
28292830
MSSAU->getMemorySSA()->verifyMemorySSA();
28302831

28312832
DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
2832-
Instruction *DeoptBlockTerm =
2833-
SplitBlockAndInsertIfThen(GI->getArgOperand(0), GI, true,
2834-
GI->getMetadata(LLVMContext::MD_prof), &DTU, &LI);
2833+
// llvm.experimental.guard doesn't have branch weights. We can assume,
2834+
// however, that the deopt path is unlikely.
2835+
Instruction *DeoptBlockTerm = SplitBlockAndInsertIfThen(
2836+
GI->getArgOperand(0), GI, true,
2837+
!ProfcheckDisableMetadataFixes && EstimateProfile
2838+
? MDBuilder(GI->getContext()).createUnlikelyBranchWeights()
2839+
: nullptr,
2840+
&DTU, &LI);
28352841
BranchInst *CheckBI = cast<BranchInst>(CheckBB->getTerminator());
28362842
// SplitBlockAndInsertIfThen inserts control flow that branches to
28372843
// DeoptBlockTerm if the condition is true. We want the opposite.

‎llvm/test/Transforms/SimpleLoopUnswitch/guards.ll‎

Lines changed: 130 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,34 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --version 5
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --check-globals all --version 5
22
; RUN: opt -passes='loop(simple-loop-unswitch<nontrivial>),verify<loops>' -simple-loop-unswitch-guards -S < %s | FileCheck %s
33
; RUN: opt -passes='simple-loop-unswitch<nontrivial>' -simple-loop-unswitch-guards -S < %s | FileCheck %s
44
; RUN: opt -passes='loop-mssa(simple-loop-unswitch<nontrivial>),verify<loops>' -simple-loop-unswitch-guards -verify-memoryssa -verify-loop-info -S < %s | FileCheck %s
55

66
declare void @llvm.experimental.guard(i1, ...)
77

8-
define void @test_simple_case(i1 %cond, i32 %N) {
9-
; CHECK-LABEL: @test_simple_case(
8+
define void @test_simple_case(i1 %cond, i32 %N) !prof !0 {
9+
; CHECK-LABEL: define void @test_simple_case(i1 %cond, i32 %N) !prof !0 {
1010
; CHECK-NEXT: entry:
11-
; CHECK-NEXT: br i1 [[COND:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
11+
; CHECK-NEXT: br i1 %cond, label %entry.split.us, label %entry.split, !prof !1
1212
; CHECK: entry.split.us:
13-
; CHECK-NEXT: br label [[LOOP_US:%.*]]
13+
; CHECK-NEXT: br label %loop.us
1414
; CHECK: loop.us:
15-
; CHECK-NEXT: [[IV_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ], [ [[IV_NEXT_US:%.*]], [[GUARDED_US:%.*]] ]
16-
; CHECK-NEXT: br label [[GUARDED_US]]
15+
; CHECK-NEXT: %iv.us = phi i32 [ 0, %entry.split.us ], [ %iv.next.us, %guarded.us ]
16+
; CHECK-NEXT: br label %guarded.us
1717
; CHECK: guarded.us:
18-
; CHECK-NEXT: [[IV_NEXT_US]] = add i32 [[IV_US]], 1
19-
; CHECK-NEXT: [[LOOP_COND_US:%.*]] = icmp slt i32 [[IV_NEXT_US]], [[N:%.*]]
20-
; CHECK-NEXT: br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[EXIT_SPLIT_US:%.*]]
18+
; CHECK-NEXT: %iv.next.us = add i32 %iv.us, 1
19+
; CHECK-NEXT: %loop.cond.us = icmp slt i32 %iv.next.us, %N
20+
; CHECK-NEXT: br i1 %loop.cond.us, label %loop.us, label %exit.split.us
21+
; CHECK: exit.split.us:
22+
; CHECK-NEXT: br label %exit
23+
; CHECK: entry.split:
24+
; CHECK-NEXT: br label %loop
25+
; CHECK: loop:
26+
; CHECK-NEXT: br label %deopt
2127
; CHECK: deopt:
2228
; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
2329
; CHECK-NEXT: unreachable
30+
; CHECK: exit:
31+
; CHECK-NEXT: ret void
2432
;
2533

2634
entry:
@@ -38,25 +46,39 @@ exit:
3846
}
3947

4048
define void @test_two_guards(i1 %cond1, i1 %cond2, i32 %N) {
41-
; CHECK-LABEL: @test_two_guards(
49+
; CHECK-LABEL: define void @test_two_guards(i1 %cond1, i1 %cond2, i32 %N) {
4250
; CHECK-NEXT: entry:
43-
; CHECK-NEXT: br i1 [[COND1:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
51+
; CHECK-NEXT: br i1 %cond1, label %entry.split.us, label %entry.split, !prof !1
4452
; CHECK: entry.split.us:
45-
; CHECK-NEXT: br i1 [[COND2:%.*]], label [[ENTRY_SPLIT_US_SPLIT_US:%.*]], label [[ENTRY_SPLIT_US_SPLIT:%.*]]
53+
; CHECK-NEXT: br i1 %cond2, label %entry.split.us.split.us, label %entry.split.us.split, !prof !1
4654
; CHECK: entry.split.us.split.us:
47-
; CHECK-NEXT: br label [[LOOP_US_US:%.*]]
55+
; CHECK-NEXT: br label %loop.us.us
4856
; CHECK: loop.us.us:
49-
; CHECK-NEXT: [[IV_US_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US_SPLIT_US]] ], [ [[IV_NEXT_US_US:%.*]], [[GUARDED_US2:%.*]] ]
50-
; CHECK-NEXT: br label [[GUARDED_US_US:%.*]]
57+
; CHECK-NEXT: %iv.us.us = phi i32 [ 0, %entry.split.us.split.us ], [ %iv.next.us.us, %guarded.us2 ]
58+
; CHECK-NEXT: br label %guarded.us.us
5159
; CHECK: guarded.us.us:
52-
; CHECK-NEXT: br label [[GUARDED_US2]]
60+
; CHECK-NEXT: br label %guarded.us2
5361
; CHECK: guarded.us2:
54-
; CHECK-NEXT: [[IV_NEXT_US_US]] = add i32 [[IV_US_US]], 1
55-
; CHECK-NEXT: [[LOOP_COND_US_US:%.*]] = icmp slt i32 [[IV_NEXT_US_US]], [[N:%.*]]
56-
; CHECK-NEXT: br i1 [[LOOP_COND_US_US]], label [[LOOP_US_US]], label [[EXIT_SPLIT_US_SPLIT_US:%.*]]
62+
; CHECK-NEXT: %iv.next.us.us = add i32 %iv.us.us, 1
63+
; CHECK-NEXT: %loop.cond.us.us = icmp slt i32 %iv.next.us.us, %N
64+
; CHECK-NEXT: br i1 %loop.cond.us.us, label %loop.us.us, label %exit.split.us.split.us
65+
; CHECK: exit.split.us.split.us:
66+
; CHECK-NEXT: br label %exit.split.us
67+
; CHECK: entry.split.us.split:
68+
; CHECK-NEXT: br label %loop.us
69+
; CHECK: loop.us:
70+
; CHECK-NEXT: br label %guarded.us
71+
; CHECK: guarded.us:
72+
; CHECK-NEXT: br label %deopt1
5773
; CHECK: deopt1:
5874
; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
5975
; CHECK-NEXT: unreachable
76+
; CHECK: exit.split.us:
77+
; CHECK-NEXT: br label %exit
78+
; CHECK: entry.split:
79+
; CHECK-NEXT: br label %loop
80+
; CHECK: loop:
81+
; CHECK-NEXT: br label %deopt
6082
; CHECK: deopt:
6183
; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
6284
; CHECK-NEXT: unreachable
@@ -80,35 +102,45 @@ exit:
80102
}
81103

82104
define void @test_conditional_guards(i1 %cond, i32 %N) {
83-
; CHECK-LABEL: @test_conditional_guards(
105+
; CHECK-LABEL: define void @test_conditional_guards(i1 %cond, i32 %N) {
84106
; CHECK-NEXT: entry:
85-
; CHECK-NEXT: [[FROZEN:%.+]] = freeze i1 [[COND:%.*]]
86-
; CHECK-NEXT: br i1 [[FROZEN]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
107+
; CHECK-NEXT: %cond.fr = freeze i1 %cond
108+
; CHECK-NEXT: br i1 %cond.fr, label %entry.split.us, label %entry.split, !prof !1
87109
; CHECK: entry.split.us:
88-
; CHECK-NEXT: br label [[LOOP_US:%.*]]
110+
; CHECK-NEXT: br label %loop.us
89111
; CHECK: loop.us:
90-
; CHECK-NEXT: [[IV_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ], [ [[IV_NEXT_US:%.*]], [[BACKEDGE_US:%.*]] ]
91-
; CHECK-NEXT: [[CONDITION_US:%.*]] = icmp eq i32 [[IV_US]], 123
92-
; CHECK-NEXT: br i1 [[CONDITION_US]], label [[GUARD_US:%.*]], label [[BACKEDGE_US]]
112+
; CHECK-NEXT: %iv.us = phi i32 [ 0, %entry.split.us ], [ %iv.next.us, %backedge.us ]
113+
; CHECK-NEXT: %condition.us = icmp eq i32 %iv.us, 123
114+
; CHECK-NEXT: br i1 %condition.us, label %guard.us, label %backedge.us
93115
; CHECK: guard.us:
94-
; CHECK-NEXT: br label [[GUARDED_US:%.*]]
116+
; CHECK-NEXT: br label %guarded.us
95117
; CHECK: backedge.us:
96-
; CHECK-NEXT: [[IV_NEXT_US]] = add i32 [[IV_US]], 1
97-
; CHECK-NEXT: [[LOOP_COND_US:%.*]] = icmp slt i32 [[IV_NEXT_US]], [[N:%.*]]
98-
; CHECK-NEXT: br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[EXIT_SPLIT_US:%.*]]
118+
; CHECK-NEXT: %iv.next.us = add i32 %iv.us, 1
119+
; CHECK-NEXT: %loop.cond.us = icmp slt i32 %iv.next.us, %N
120+
; CHECK-NEXT: br i1 %loop.cond.us, label %loop.us, label %exit.split.us
121+
; CHECK: guarded.us:
122+
; CHECK-NEXT: br label %backedge.us
123+
; CHECK: exit.split.us:
124+
; CHECK-NEXT: br label %exit
125+
; CHECK: entry.split:
126+
; CHECK-NEXT: br label %loop
99127
; CHECK: loop:
100-
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
101-
; CHECK-NEXT: [[CONDITION:%.*]] = icmp eq i32 [[IV]], 123
102-
; CHECK-NEXT: br i1 [[CONDITION]], label [[GUARD:%.*]], label [[BACKEDGE]]
128+
; CHECK-NEXT: %iv = phi i32 [ 0, %entry.split ], [ %iv.next, %backedge ]
129+
; CHECK-NEXT: %condition = icmp eq i32 %iv, 123
130+
; CHECK-NEXT: br i1 %condition, label %guard, label %backedge
103131
; CHECK: guard:
104-
; CHECK-NEXT: br label [[DEOPT:%.*]]
132+
; CHECK-NEXT: br label %deopt
105133
; CHECK: deopt:
106134
; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
107135
; CHECK-NEXT: unreachable
108136
; CHECK: backedge:
109-
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
110-
; CHECK-NEXT: [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], [[N]]
111-
; CHECK-NEXT: br i1 [[LOOP_COND]], label %loop, label [[EXIT_SPLIT:%.*]]
137+
; CHECK-NEXT: %iv.next = add i32 %iv, 1
138+
; CHECK-NEXT: %loop.cond = icmp slt i32 %iv.next, %N
139+
; CHECK-NEXT: br i1 %loop.cond, label %loop, label %exit.split
140+
; CHECK: exit.split:
141+
; CHECK-NEXT: br label %exit
142+
; CHECK: exit:
143+
; CHECK-NEXT: ret void
112144
;
113145

114146
entry:
@@ -135,7 +167,7 @@ exit:
135167
define void @test_nested_loop(i1 %cond, i32 %N, i1 %arg) {
136168
; CHECK-LABEL: define void @test_nested_loop(i1 %cond, i32 %N, i1 %arg) {
137169
; CHECK-NEXT: entry:
138-
; CHECK-NEXT: br i1 %cond, label %entry.split, label %outer_loop.split
170+
; CHECK-NEXT: br i1 %cond, label %entry.split, label %outer_loop.split, !prof !1
139171
; CHECK: entry.split:
140172
; CHECK-NEXT: br i1 %arg, label %entry.split.split.us, label %entry.split.split
141173
; CHECK: entry.split.split.us:
@@ -204,17 +236,49 @@ exit:
204236
}
205237

206238
define void @test_sibling_loops(i1 %cond1, i1 %cond2, i32 %N) {
207-
; CHECK-LABEL: @test_sibling_loops(
239+
; CHECK-LABEL: define void @test_sibling_loops(i1 %cond1, i1 %cond2, i32 %N) {
208240
; CHECK-NEXT: entry:
209-
; CHECK-NEXT: br i1 [[COND1:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
210-
; CHECK: [[IV1_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ], [ [[IV1_NEXT_US:%.*]], [[GUARDED_US:%.*]] ]
211-
; CHECK-NEXT: br label [[GUARDED_US]]
212-
; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
241+
; CHECK-NEXT: br i1 %cond1, label %entry.split.us, label %entry.split, !prof !1
242+
; CHECK: entry.split.us:
243+
; CHECK-NEXT: br label %loop1.us
244+
; CHECK: loop1.us:
245+
; CHECK-NEXT: %iv1.us = phi i32 [ 0, %entry.split.us ], [ %iv1.next.us, %guarded.us ]
246+
; CHECK-NEXT: br label %guarded.us
247+
; CHECK: guarded.us:
248+
; CHECK-NEXT: %iv1.next.us = add i32 %iv1.us, 1
249+
; CHECK-NEXT: %loop1.cond.us = icmp slt i32 %iv1.next.us, %N
250+
; CHECK-NEXT: br i1 %loop1.cond.us, label %loop1.us, label %between.split.us
251+
; CHECK: between.split.us:
252+
; CHECK-NEXT: br label %between
253+
; CHECK: entry.split:
254+
; CHECK-NEXT: br label %loop1
255+
; CHECK: loop1:
256+
; CHECK-NEXT: br label %deopt
257+
; CHECK: deopt:
258+
; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
213259
; CHECK-NEXT: unreachable
214-
; CHECK: [[IV2_US:%.*]] = phi i32 [ 0, [[BETWEEN:%.*]] ], [ [[IV1_NEXT_US2:%.*]], [[GUARDED_US2:%.*]] ]
215-
; CHECK-NEXT: br label [[GUARDED_US2]]
216-
; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
260+
; CHECK: between:
261+
; CHECK-NEXT: br i1 %cond2, label %between.split.us2, label %between.split, !prof !1
262+
; CHECK: between.split.us2:
263+
; CHECK-NEXT: br label %loop2.us
264+
; CHECK: loop2.us:
265+
; CHECK-NEXT: %iv2.us = phi i32 [ 0, %between.split.us2 ], [ %iv2.next.us, %guarded.us3 ]
266+
; CHECK-NEXT: br label %guarded.us3
267+
; CHECK: guarded.us3:
268+
; CHECK-NEXT: %iv2.next.us = add i32 %iv2.us, 1
269+
; CHECK-NEXT: %loop2.cond.us = icmp slt i32 %iv2.next.us, %N
270+
; CHECK-NEXT: br i1 %loop2.cond.us, label %loop2.us, label %exit.split.us
271+
; CHECK: exit.split.us:
272+
; CHECK-NEXT: br label %exit
273+
; CHECK: between.split:
274+
; CHECK-NEXT: br label %loop2
275+
; CHECK: loop2:
276+
; CHECK-NEXT: br label %deopt1
277+
; CHECK: deopt1:
278+
; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
217279
; CHECK-NEXT: unreachable
280+
; CHECK: exit:
281+
; CHECK-NEXT: ret void
218282
;
219283

220284
entry:
@@ -242,11 +306,20 @@ exit:
242306
}
243307

244308
; Check that we don't do anything because of cleanuppad.
245-
; CHECK-LABEL: @test_cleanuppad(
246-
; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
247-
; CHECK-NOT: call void (i1, ...) @llvm.experimental.guard(
248309
define void @test_cleanuppad(i1 %cond, i32 %N) personality ptr @__CxxFrameHandler3 {
249-
310+
; CHECK-LABEL: define void @test_cleanuppad(i1 %cond, i32 %N) personality ptr @__CxxFrameHandler3 {
311+
; CHECK-NEXT: entry:
312+
; CHECK-NEXT: br label %loop
313+
; CHECK: loop:
314+
; CHECK-NEXT: %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
315+
; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
316+
; CHECK-NEXT: %iv.next = add i32 %iv, 1
317+
; CHECK-NEXT: invoke void @may_throw(i32 %iv)
318+
; CHECK-NEXT: to label %loop unwind label %exit
319+
; CHECK: exit:
320+
; CHECK-NEXT: %cp = cleanuppad within none []
321+
; CHECK-NEXT: cleanupret from %cp unwind to caller
322+
;
250323
entry:
251324
br label %loop
252325

@@ -264,3 +337,9 @@ exit:
264337

265338
declare void @may_throw(i32 %i)
266339
declare i32 @__CxxFrameHandler3(...)
340+
341+
!0 = !{!"function_entry_count", i32 10}
342+
;.
343+
; CHECK: !0 = !{!"function_entry_count", i32 10}
344+
; CHECK: !1 = !{!"branch_weights", i32 1048575, i32 1}
345+
;.

0 commit comments

Comments
 (0)