Skip to content

Commit d842a71

Browse files
mtrofinmtrofin
committed
[SLU][profcheck] Use the original branch weigths in buildPartialInvariantUnswitchConditionalBranch 
1 parent e63f0f5 commit d842a71

File tree

2 files changed

+56
-36
lines changed

2 files changed

+56
-36
lines changed

‎llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -337,7 +337,7 @@ static void buildPartialUnswitchConditionalBranch(
337337
static void buildPartialInvariantUnswitchConditionalBranch(
338338
BasicBlock &BB, ArrayRef<Value *> ToDuplicate, bool Direction,
339339
BasicBlock &UnswitchedSucc, BasicBlock &NormalSucc, Loop &L,
340-
MemorySSAUpdater *MSSAU) {
340+
MemorySSAUpdater *MSSAU, const BranchInst &OriginalBranch) {
341341
ValueToValueMapTy VMap;
342342
for (auto *Val : reverse(ToDuplicate)) {
343343
Instruction *Inst = cast<Instruction>(Val);
@@ -377,8 +377,17 @@ static void buildPartialInvariantUnswitchConditionalBranch(
377377
IRBuilder<> IRB(&BB);
378378
IRB.SetCurrentDebugLocation(DebugLoc::getCompilerGenerated());
379379
Value *Cond = VMap[ToDuplicate[0]];
380-
IRB.CreateCondBr(Cond, Direction ? &UnswitchedSucc : &NormalSucc,
381-
Direction ? &NormalSucc : &UnswitchedSucc);
380+
auto *ProfData =
381+
!ProfcheckDisableMetadataFixes &&
382+
ToDuplicate[0] == skipTrivialSelect(OriginalBranch.getCondition())
383+
? OriginalBranch.getMetadata(LLVMContext::MD_prof)
384+
: nullptr;
385+
auto *BR =
386+
IRB.CreateCondBr(Cond, Direction ? &UnswitchedSucc : &NormalSucc,
387+
Direction ? &NormalSucc : &UnswitchedSucc, ProfData);
388+
if (!ProfData)
389+
setExplicitlyUnknownBranchWeightsIfProfiled(
390+
*BR, *BR->getParent()->getParent(), DEBUG_TYPE);
382391
}
383392

384393
/// Rewrite the PHI nodes in an unswitched loop exit basic block.
@@ -2515,7 +2524,7 @@ static void unswitchNontrivialInvariants(
25152524
// the branch in the split block.
25162525
if (PartiallyInvariant)
25172526
buildPartialInvariantUnswitchConditionalBranch(
2518-
*SplitBB, Invariants, Direction, *ClonedPH, *LoopPH, L, MSSAU);
2527+
*SplitBB, Invariants, Direction, *ClonedPH, *LoopPH, L, MSSAU, *BI);
25192528
else {
25202529
buildPartialUnswitchConditionalBranch(
25212530
*SplitBB, Invariants, Direction, *ClonedPH, *LoopPH,

‎llvm/test/Transforms/SimpleLoopUnswitch/partial-unswitch.ll‎

Lines changed: 43 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
22
; RUN: opt -passes='loop-mssa(simple-loop-unswitch<nontrivial>),verify<loops>' -S < %s | FileCheck %s
33

44
declare void @clobber()
55

6-
define i32 @partial_unswitch_true_successor(ptr %ptr, i32 %N) {
6+
define i32 @partial_unswitch_true_successor(ptr %ptr, i32 %N) !prof !0 {
77
; CHECK-LABEL: @partial_unswitch_true_successor(
88
; CHECK-NEXT: entry:
99
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[PTR:%.*]], align 4
1010
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 100
11-
; CHECK-NEXT: br i1 [[TMP1]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
11+
; CHECK-NEXT: br i1 [[TMP1]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]], !prof [[PROF1:![0-9]+]]
1212
; CHECK: entry.split.us:
1313
; CHECK-NEXT: br label [[LOOP_HEADER_US:%.*]]
1414
; CHECK: loop.header.us:
@@ -19,7 +19,7 @@ define i32 @partial_unswitch_true_successor(ptr %ptr, i32 %N) {
1919
; CHECK: loop.latch.us:
2020
; CHECK-NEXT: [[C_US:%.*]] = icmp ult i32 [[IV_US]], [[N:%.*]]
2121
; CHECK-NEXT: [[IV_NEXT_US]] = add i32 [[IV_US]], 1
22-
; CHECK-NEXT: br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]]
22+
; CHECK-NEXT: br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]], !prof [[PROF2:![0-9]+]]
2323
; CHECK: exit.split.us:
2424
; CHECK-NEXT: br label [[EXIT:%.*]]
2525
; CHECK: entry.split:
@@ -28,7 +28,7 @@ define i32 @partial_unswitch_true_successor(ptr %ptr, i32 %N) {
2828
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
2929
; CHECK-NEXT: [[LV:%.*]] = load i32, ptr [[PTR]], align 4
3030
; CHECK-NEXT: [[SC:%.*]] = icmp eq i32 [[LV]], 100
31-
; CHECK-NEXT: br i1 [[SC]], label [[NOCLOBBER:%.*]], label [[CLOBBER:%.*]]
31+
; CHECK-NEXT: br i1 [[SC]], label [[NOCLOBBER:%.*]], label [[CLOBBER:%.*]], !prof [[PROF1]]
3232
; CHECK: noclobber:
3333
; CHECK-NEXT: br label [[LOOP_LATCH]]
3434
; CHECK: clobber:
@@ -37,7 +37,7 @@ define i32 @partial_unswitch_true_successor(ptr %ptr, i32 %N) {
3737
; CHECK: loop.latch:
3838
; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
3939
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
40-
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
40+
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !prof [[PROF2]], !llvm.loop [[LOOP3:![0-9]+]]
4141
; CHECK: exit.split:
4242
; CHECK-NEXT: br label [[EXIT]]
4343
; CHECK: exit:
@@ -50,7 +50,7 @@ loop.header:
5050
%iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
5151
%lv = load i32, ptr %ptr
5252
%sc = icmp eq i32 %lv, 100
53-
br i1 %sc, label %noclobber, label %clobber
53+
br i1 %sc, label %noclobber, label %clobber, !prof !1
5454

5555
noclobber:
5656
br label %loop.latch
@@ -62,7 +62,7 @@ clobber:
6262
loop.latch:
6363
%c = icmp ult i32 %iv, %N
6464
%iv.next = add i32 %iv, 1
65-
br i1 %c, label %loop.header, label %exit
65+
br i1 %c, label %loop.header, label %exit, !prof !2
6666

6767
exit:
6868
ret i32 10
@@ -102,7 +102,7 @@ define i32 @partial_unswitch_false_successor(ptr %ptr, i32 %N) {
102102
; CHECK: loop.latch:
103103
; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
104104
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
105-
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP2:![0-9]+]]
105+
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP5:![0-9]+]]
106106
; CHECK: exit.split:
107107
; CHECK-NEXT: br label [[EXIT]]
108108
; CHECK: exit:
@@ -171,7 +171,7 @@ define i32 @partial_unswtich_gep_load_icmp(ptr %ptr, i32 %N) {
171171
; CHECK: loop.latch:
172172
; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
173173
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
174-
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP3:![0-9]+]]
174+
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP6:![0-9]+]]
175175
; CHECK: exit.split:
176176
; CHECK-NEXT: br label [[EXIT]]
177177
; CHECK: exit:
@@ -246,7 +246,7 @@ define i32 @partial_unswitch_reduction_phi(ptr %ptr, i32 %N) {
246246
; CHECK-NEXT: [[RED_NEXT]] = phi i32 [ [[ADD_5]], [[CLOBBER]] ], [ [[ADD_10]], [[NOCLOBBER]] ]
247247
; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
248248
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
249-
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP4:![0-9]+]]
249+
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP7:![0-9]+]]
250250
; CHECK: exit.split:
251251
; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i32 [ [[RED_NEXT]], [[LOOP_LATCH]] ]
252252
; CHECK-NEXT: br label [[EXIT]]
@@ -325,7 +325,7 @@ define i32 @partial_unswitch_true_successor_noclobber(ptr noalias %ptr.1, ptr no
325325
; CHECK: loop.latch:
326326
; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
327327
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
328-
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP5:![0-9]+]]
328+
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP8:![0-9]+]]
329329
; CHECK: exit.split:
330330
; CHECK-NEXT: br label [[EXIT]]
331331
; CHECK: exit:
@@ -637,7 +637,7 @@ define i32 @partial_unswitch_true_successor_preheader_insertion(ptr %ptr, i32 %N
637637
; CHECK: loop.latch:
638638
; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
639639
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
640-
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_LOOPEXIT_SPLIT:%.*]], !llvm.loop [[LOOP6:![0-9]+]]
640+
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_LOOPEXIT_SPLIT:%.*]], !llvm.loop [[LOOP9:![0-9]+]]
641641
; CHECK: exit.loopexit.split:
642642
; CHECK-NEXT: br label [[EXIT_LOOPEXIT]]
643643
; CHECK: exit.loopexit:
@@ -713,7 +713,7 @@ define i32 @partial_unswitch_true_successor_insert_point(ptr %ptr, i32 %N) {
713713
; CHECK: loop.latch:
714714
; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
715715
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
716-
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP7:![0-9]+]]
716+
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP10:![0-9]+]]
717717
; CHECK: exit.split:
718718
; CHECK-NEXT: br label [[EXIT]]
719719
; CHECK: exit:
@@ -784,7 +784,7 @@ define i32 @partial_unswitch_true_successor_hoist_invariant(ptr %ptr, i32 %N) {
784784
; CHECK: loop.latch:
785785
; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
786786
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
787-
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP8:![0-9]+]]
787+
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP11:![0-9]+]]
788788
; CHECK: exit.split:
789789
; CHECK-NEXT: br label [[EXIT]]
790790
; CHECK: exit:
@@ -1073,7 +1073,7 @@ define i32 @partial_unswitch_true_to_latch(ptr %ptr, i32 %N) {
10731073
; CHECK: loop.latch:
10741074
; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
10751075
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
1076-
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP9:![0-9]+]]
1076+
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP12:![0-9]+]]
10771077
; CHECK: exit.split:
10781078
; CHECK-NEXT: br label [[EXIT]]
10791079
; CHECK: exit:
@@ -1138,7 +1138,7 @@ define i32 @partial_unswitch_exiting_block_with_multiple_unswitch_candidates(i32
11381138
; CHECK-NEXT: store i32 [[TMP1:%.*]], ptr [[PTR]], align 16
11391139
; CHECK-NEXT: br label [[EXITING]]
11401140
; CHECK: exiting:
1141-
; CHECK-NEXT: br i1 [[EXIT_COND]], label [[LOOP]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP10:![0-9]+]]
1141+
; CHECK-NEXT: br i1 [[EXIT_COND]], label [[LOOP]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP13:![0-9]+]]
11421142
; CHECK: exit.split:
11431143
; CHECK-NEXT: [[RET_VAL:%.*]] = phi i32 [ 1, [[EXITING]] ]
11441144
; CHECK-NEXT: br label [[EXIT]]
@@ -1249,7 +1249,7 @@ define i32 @partial_unswitch_true_successor_for_cost_calculation(ptr %ptr, i32 %
12491249
; CHECK: loop.latch:
12501250
; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
12511251
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
1252-
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP11:![0-9]+]]
1252+
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP14:![0-9]+]]
12531253
; CHECK: exit.split:
12541254
; CHECK-NEXT: br label [[EXIT]]
12551255
; CHECK: exit:
@@ -1360,7 +1360,7 @@ define i32 @partial_unswitch_true_successor_trunc(ptr %ptr, i32 %N) {
13601360
; CHECK: loop.latch:
13611361
; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
13621362
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
1363-
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP12:![0-9]+]]
1363+
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP15:![0-9]+]]
13641364
; CHECK: exit.split:
13651365
; CHECK-NEXT: br label [[EXIT]]
13661366
; CHECK: exit:
@@ -1425,7 +1425,7 @@ define i32 @partial_unswitch_false_successor_trunc(ptr %ptr, i32 %N) {
14251425
; CHECK: loop.latch:
14261426
; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
14271427
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
1428-
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP13:![0-9]+]]
1428+
; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP16:![0-9]+]]
14291429
; CHECK: exit.split:
14301430
; CHECK-NEXT: br label [[EXIT]]
14311431
; CHECK: exit:
@@ -1456,15 +1456,26 @@ exit:
14561456
ret i32 10
14571457
}
14581458

1459-
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[UNSWITCH_PARTIAL_DISABLE:![0-9]+]]}
1460-
; CHECK: [[UNSWITCH_PARTIAL_DISABLE]] = !{!"llvm.loop.unswitch.partial.disable"}
1461-
; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[UNSWITCH_PARTIAL_DISABLE]]}
1462-
; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[UNSWITCH_PARTIAL_DISABLE]]}
1463-
; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[UNSWITCH_PARTIAL_DISABLE]]}
1464-
; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[UNSWITCH_PARTIAL_DISABLE]]}
1465-
; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[UNSWITCH_PARTIAL_DISABLE]]}
1466-
; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[UNSWITCH_PARTIAL_DISABLE]]}
1467-
; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[UNSWITCH_PARTIAL_DISABLE]]}
1468-
; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[UNSWITCH_PARTIAL_DISABLE]]}
1469-
; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[UNSWITCH_PARTIAL_DISABLE]]}
1470-
; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[UNSWITCH_PARTIAL_DISABLE]]}
1459+
!0 = !{!"function_entry_count", i32 10}
1460+
!1 = !{!"branch_weights", i32 1000, i32 1}
1461+
!2 = !{!"branch_weights", i32 100, i32 3}
1462+
1463+
;.
1464+
; CHECK: [[META0:![0-9]+]] = !{!"function_entry_count", i32 10}
1465+
; CHECK: [[PROF1]] = !{!"branch_weights", i32 1000, i32 1}
1466+
; CHECK: [[PROF2]] = !{!"branch_weights", i32 100, i32 3}
1467+
; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]]}
1468+
; CHECK: [[META4]] = !{!"llvm.loop.unswitch.partial.disable"}
1469+
; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META4]]}
1470+
; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
1471+
; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]]}
1472+
; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]]}
1473+
; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]]}
1474+
; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META4]]}
1475+
; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META4]]}
1476+
; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META4]]}
1477+
; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META4]]}
1478+
; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META4]]}
1479+
; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META4]]}
1480+
; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META4]]}
1481+
;.

0 commit comments

Comments
 (0)