Skip to content

Commit 45ce887

Browse files
fhahnfhahn
authored
[LV] Don't preserve LCSSA in SCEVExpander for runtime checks. (#159556)
LV does not preserve LCSSA, it constructs it just before processing a loop to vectorize. Runtime check expressions are invariant to that loop, so expanding them should not break LCSSA form for the loop we are about to vectorize. This fixes a crash when discarding instructions generated when expanding runtime checks, if the expansion introduces LCSSA phis for values from other loops which are not in LCSSA form: we would introduce new LCSSA phis and update all outside users, some of which are not created by the expander and cannot be cleaned up. Fixes #158259. PR: #159556
1 parent da315a3 commit 45ce887

File tree

6 files changed

+100
-29
lines changed

6 files changed

+100
-29
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1762,9 +1762,10 @@ class GeneratedRTChecks {
17621762
GeneratedRTChecks(PredicatedScalarEvolution &PSE, DominatorTree *DT,
17631763
LoopInfo *LI, TargetTransformInfo *TTI,
17641764
const DataLayout &DL, TTI::TargetCostKind CostKind)
1765-
: DT(DT), LI(LI), TTI(TTI), SCEVExp(*PSE.getSE(), DL, "scev.check"),
1766-
MemCheckExp(*PSE.getSE(), DL, "scev.check"), PSE(PSE),
1767-
CostKind(CostKind) {}
1765+
: DT(DT), LI(LI), TTI(TTI),
1766+
SCEVExp(*PSE.getSE(), DL, "scev.check", /*PreserveLCSSA=*/false),
1767+
MemCheckExp(*PSE.getSE(), DL, "scev.check", /*PreserveLCSSA=*/false),
1768+
PSE(PSE), CostKind(CostKind) {}
17681769

17691770
/// Generate runtime checks in SCEVCheckBlock and MemCheckBlock, so we can
17701771
/// accurately estimate the cost of the runtime checks. The blocks are

‎llvm/test/Transforms/LoopVectorize/X86/cleanup-runtime-checks.ll‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
2+
; RUN: opt -p loop-vectorize -S %s | FileCheck %s
3+
4+
target triple = "x86_64-unknown-linux-gnu"
5+
6+
declare ptr @get()
7+
declare i1 @cond()
8+
9+
; Make sure we can clean up the created runtime checks, if vectorization isn't
10+
; profitable.
11+
define void @widget(i32 %arg, i64 %arg1, ptr %src) #0 {
12+
; CHECK-LABEL: define void @widget(
13+
; CHECK-SAME: i32 [[ARG:%.*]], i64 [[ARG1:%.*]], ptr [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
14+
; CHECK-NEXT: [[ENTRY:.*:]]
15+
; CHECK-NEXT: br label %[[LOOP_1_HEADER:.*]]
16+
; CHECK: [[LOOP_1_HEADER]]:
17+
; CHECK-NEXT: br label %[[INNER_1:.*]]
18+
; CHECK: [[INNER_1]]:
19+
; CHECK-NEXT: [[C_1:%.*]] = call i1 @cond()
20+
; CHECK-NEXT: br i1 [[C_1]], label %[[INNER_2:.*]], label %[[INNER_1]]
21+
; CHECK: [[INNER_2]]:
22+
; CHECK-NEXT: [[LOAD:%.*]] = call ptr @get()
23+
; CHECK-NEXT: [[C_2:%.*]] = call i1 @cond()
24+
; CHECK-NEXT: br i1 [[C_2]], label %[[LOOP_2_PREHEADER:.*]], label %[[LOOP_1_LATCH:.*]]
25+
; CHECK: [[LOOP_2_PREHEADER]]:
26+
; CHECK-NEXT: br label %[[LOOP_2:.*]]
27+
; CHECK: [[LOOP_1_LATCH]]:
28+
; CHECK-NEXT: br label %[[LOOP_1_HEADER]]
29+
; CHECK: [[LOOP_2]]:
30+
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[LOOP_2]] ], [ [[ARG]], %[[LOOP_2_PREHEADER]] ]
31+
; CHECK-NEXT: [[PHI8:%.*]] = phi i32 [ [[OR:%.*]], %[[LOOP_2]] ], [ 99, %[[LOOP_2_PREHEADER]] ]
32+
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i32, ptr [[SRC]], i32 [[IV]]
33+
; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP_SRC]], align 4
34+
; CHECK-NEXT: [[OR]] = or i32 [[PHI8]], [[L]]
35+
; CHECK-NEXT: store i32 [[OR]], ptr [[LOAD]], align 4
36+
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
37+
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV]], 100
38+
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_2]], !prof [[PROF0:![0-9]+]]
39+
; CHECK: [[EXIT]]:
40+
; CHECK-NEXT: ret void
41+
;
42+
entry:
43+
br label %loop.1.header
44+
45+
loop.1.header:
46+
br label %inner.1
47+
48+
inner.1:
49+
%c.1 = call i1 @cond()
50+
br i1 %c.1, label %inner.2, label %inner.1
51+
52+
inner.2:
53+
%load = call ptr @get()
54+
%c.2 = call i1 @cond()
55+
br i1 %c.2, label %loop.2, label %loop.1.latch
56+
57+
loop.1.latch:
58+
br label %loop.1.header
59+
60+
loop.2:
61+
%iv = phi i32 [ %arg, %inner.2 ], [ %iv.next, %loop.2 ]
62+
%phi8 = phi i32 [ 99, %inner.2 ], [ %or, %loop.2 ]
63+
%gep.src = getelementptr i32, ptr %src, i32 %iv
64+
%l = load i32, ptr %gep.src, align 4
65+
%or = or i32 %phi8, %l
66+
store i32 %or, ptr %load, align 4
67+
%iv.next = add i32 %iv, 1
68+
%ec = icmp eq i32 %iv, 100
69+
br i1 %ec, label %exit, label %loop.2, !prof !0
70+
71+
exit:
72+
ret void
73+
}
74+
75+
attributes #0 = { "target-features"="+avx2" }
76+
!0 = !{!"branch_weights", i32 89478484, i32 1879048192}
77+
;.
78+
; CHECK: [[PROF0]] = !{!"branch_weights", i32 89478484, i32 1879048192}
79+
;.

‎llvm/test/Transforms/LoopVectorize/pr45259.ll‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,15 @@ define i8 @widget(ptr %arr, i8 %t9) {
1010
; CHECK-NEXT: br label [[BB6:%.*]]
1111
; CHECK: bb6:
1212
; CHECK-NEXT: [[T1_0:%.*]] = phi ptr [ [[ARR]], [[BB:%.*]] ], [ null, [[BB6]] ]
13+
; CHECK-NEXT: [[T1_0_LCSSA2:%.*]] = ptrtoint ptr [[T1_0]] to i64
1314
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
1415
; CHECK-NEXT: br i1 [[C]], label [[FOR_PREHEADER:%.*]], label [[BB6]]
1516
; CHECK: for.preheader:
16-
; CHECK-NEXT: [[T1_0_LCSSA:%.*]] = phi ptr [ [[T1_0]], [[BB6]] ]
1717
; CHECK-NEXT: [[T1_0_LCSSA4:%.*]] = phi ptr [ [[T1_0]], [[BB6]] ]
1818
; CHECK-NEXT: [[T1_0_LCSSA1:%.*]] = phi ptr [ [[T1_0]], [[BB6]] ]
19-
; CHECK-NEXT: [[T1_0_LCSSA3:%.*]] = ptrtoint ptr [[T1_0_LCSSA]] to i64
20-
; CHECK-NEXT: [[T1_0_LCSSA2:%.*]] = ptrtoint ptr [[T1_0_LCSSA4]] to i64
2119
; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[ARR1]] to i32
2220
; CHECK-NEXT: [[TMP1:%.*]] = sub i32 0, [[TMP0]]
21+
; CHECK-NEXT: [[T1_0_LCSSA3:%.*]] = ptrtoint ptr [[T1_0_LCSSA4]] to i64
2322
; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[T1_0_LCSSA3]] to i32
2423
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[TMP1]], [[TMP2]]
2524
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP3]], 4

‎llvm/test/Transforms/LoopVectorize/pr47343-expander-lcssa-after-cfg-update.ll‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,18 +28,15 @@ define void @f() {
2828
; CHECK: outer.latch:
2929
; CHECK-NEXT: br label [[OUTER_HEADER]]
3030
; CHECK: outer.exit.0:
31-
; CHECK-NEXT: [[DOTLCSSA:%.*]] = phi ptr [ [[TMP0]], [[OUTER_HEADER]] ]
3231
; CHECK-NEXT: br label [[LOOP_PREHEADER:%.*]]
3332
; CHECK: outer.exit.1:
34-
; CHECK-NEXT: [[DOTLCSSA1:%.*]] = phi ptr [ [[TMP0]], [[INNER_1_LATCH]] ]
3533
; CHECK-NEXT: br label [[LOOP_PREHEADER]]
3634
; CHECK: loop.preheader:
37-
; CHECK-NEXT: [[TMP1:%.*]] = phi ptr [ [[DOTLCSSA]], [[OUTER_EXIT_0]] ], [ [[DOTLCSSA1]], [[OUTER_EXIT_1]] ]
3835
; CHECK-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
3936
; CHECK: vector.memcheck:
40-
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[TMP1]], i64 1
37+
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[TMP0]], i64 1
4138
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr @f.e, [[SCEVGEP]]
42-
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[TMP1]], getelementptr inbounds nuw (i8, ptr @f.e, i64 4)
39+
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[TMP0]], getelementptr inbounds nuw (i8, ptr @f.e, i64 4)
4340
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
4441
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
4542
; CHECK: vector.ph:
@@ -59,7 +56,7 @@ define void @f() {
5956
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ 0, [[SCALAR_PH]] ]
6057
; CHECK-NEXT: [[CONV6_US_US_US:%.*]] = zext i1 false to i32
6158
; CHECK-NEXT: store i32 [[CONV6_US_US_US]], ptr @f.e, align 1
62-
; CHECK-NEXT: store i8 10, ptr [[TMP1]], align 1
59+
; CHECK-NEXT: store i8 10, ptr [[TMP0]], align 1
6360
; CHECK-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
6461
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 500
6562
; CHECK-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]

‎llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -109,14 +109,13 @@ define void @runtime_checks_ptr_inductions(ptr %dst.1, ptr %dst.2, i1 %c) {
109109
; CHECK-NEXT: [[PTR_IV_1:%.*]] = phi ptr [ [[DST_1]], %[[ENTRY]] ], [ [[PTR_IV_1_NEXT:%.*]], %[[LOOP_1]] ]
110110
; CHECK-NEXT: [[CALL:%.*]] = call i32 @val()
111111
; CHECK-NEXT: [[SEL_DST:%.*]] = select i1 [[C]], ptr [[DST_1]], ptr [[DST_2]]
112+
; CHECK-NEXT: [[SEL_DST_LCSSA12:%.*]] = ptrtoint ptr [[SEL_DST]] to i64
112113
; CHECK-NEXT: [[PTR_IV_1_NEXT]] = getelementptr i8, ptr [[PTR_IV_1]], i64 1
113114
; CHECK-NEXT: [[EC_1:%.*]] = icmp eq i32 [[CALL]], 0
114115
; CHECK-NEXT: br i1 [[EC_1]], label %[[LOOP_2_HEADER_PREHEADER:.*]], label %[[LOOP_1]]
115116
; CHECK: [[LOOP_2_HEADER_PREHEADER]]:
116-
; CHECK-NEXT: [[SEL_DST_LCSSA1:%.*]] = phi ptr [ [[SEL_DST]], %[[LOOP_1]] ]
117117
; CHECK-NEXT: [[PTR_IV_1_LCSSA:%.*]] = phi ptr [ [[PTR_IV_1]], %[[LOOP_1]] ]
118118
; CHECK-NEXT: [[SEL_DST_LCSSA:%.*]] = phi ptr [ [[SEL_DST]], %[[LOOP_1]] ]
119-
; CHECK-NEXT: [[SEL_DST_LCSSA12:%.*]] = ptrtoint ptr [[SEL_DST_LCSSA1]] to i64
120119
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
121120
; CHECK: [[VECTOR_MEMCHECK]]:
122121
; CHECK-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[PTR_IV_1_LCSSA]] to i64
@@ -140,13 +139,13 @@ define void @runtime_checks_ptr_inductions(ptr %dst.1, ptr %dst.2, i1 %c) {
140139
; CHECK-NEXT: br label %[[SCALAR_PH]]
141140
; CHECK: [[SCALAR_PH]]:
142141
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 1023, %[[MIDDLE_BLOCK]] ], [ 1, %[[VECTOR_MEMCHECK]] ]
143-
; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[PTR_IV_1_LCSSA]], %[[VECTOR_MEMCHECK]] ]
144-
; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[SEL_DST_LCSSA]], %[[VECTOR_MEMCHECK]] ]
142+
; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[PTR_IV_1_LCSSA]], %[[VECTOR_MEMCHECK]] ]
143+
; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[SEL_DST_LCSSA]], %[[VECTOR_MEMCHECK]] ]
145144
; CHECK-NEXT: br label %[[LOOP_2_HEADER:.*]]
146145
; CHECK: [[LOOP_2_HEADER]]:
147146
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[DEC7:%.*]], %[[LOOP_2_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
148-
; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[PTR_IV_2_NEXT:%.*]], %[[LOOP_2_LATCH]] ], [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ]
149-
; CHECK-NEXT: [[PTR_IV_3:%.*]] = phi ptr [ [[PTR_IV_3_NEXT:%.*]], %[[LOOP_2_LATCH]] ], [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ]
147+
; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[PTR_IV_2_NEXT:%.*]], %[[LOOP_2_LATCH]] ], [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ]
148+
; CHECK-NEXT: [[PTR_IV_3:%.*]] = phi ptr [ [[PTR_IV_3_NEXT:%.*]], %[[LOOP_2_LATCH]] ], [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ]
150149
; CHECK-NEXT: [[EC_2:%.*]] = icmp eq i32 [[IV]], 1024
151150
; CHECK-NEXT: br i1 [[EC_2]], label %[[EXIT:.*]], label %[[LOOP_2_LATCH]]
152151
; CHECK: [[LOOP_2_LATCH]]:

‎llvm/test/Transforms/LoopVectorize/skeleton-lcssa-crash.ll‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,18 +23,16 @@ define i16 @test(ptr %arg, i64 %N) {
2323
; CHECK-NEXT: [[C_3:%.*]] = call i1 @cond()
2424
; CHECK-NEXT: br i1 [[C_3]], label [[LOOP_3_PREHEADER:%.*]], label [[INNER_LATCH:%.*]]
2525
; CHECK: loop.3.preheader:
26-
; CHECK-NEXT: [[L_1_LCSSA:%.*]] = phi ptr [ [[L_1]], [[INNER_BB]] ]
27-
; CHECK-NEXT: [[L_2_LCSSA:%.*]] = phi ptr [ [[L_2]], [[INNER_BB]] ]
2826
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
2927
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
3028
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
3129
; CHECK: vector.memcheck:
32-
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[L_2_LCSSA]], i64 2
33-
; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[L_1_LCSSA]], i64 2
30+
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[L_2]], i64 2
31+
; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[L_1]], i64 2
3432
; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[N]], 1
3533
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 4
36-
; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[L_1_LCSSA]], i64 [[TMP2]]
37-
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[L_2_LCSSA]], [[SCEVGEP6]]
34+
; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[L_1]], i64 [[TMP2]]
35+
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[L_2]], [[SCEVGEP3]]
3836
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP5]], [[SCEVGEP]]
3937
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
4038
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
@@ -67,19 +65,17 @@ define i16 @test(ptr %arg, i64 %N) {
6765
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_3]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
6866
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
6967
; CHECK-NEXT: [[C_5:%.*]] = icmp ult i64 [[IV]], [[N]]
70-
; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr inbounds i16, ptr [[L_1_LCSSA]], i64 [[IV_NEXT]]
68+
; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr inbounds i16, ptr [[L_1]], i64 [[IV_NEXT]]
7169
; CHECK-NEXT: [[LOOP_L_1:%.*]] = load i16, ptr [[GEP_1]], align 2
72-
; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr inbounds i16, ptr [[L_2_LCSSA]], i64 0
70+
; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr inbounds i16, ptr [[L_2]], i64 0
7371
; CHECK-NEXT: store i16 [[LOOP_L_1]], ptr [[GEP_2]], align 2
7472
; CHECK-NEXT: br i1 [[C_5]], label [[LOOP_3]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP8:![0-9]+]]
7573
; CHECK: exit.loopexit:
7674
; CHECK-NEXT: br label [[EXIT:%.*]]
7775
; CHECK: exit.loopexit1:
78-
; CHECK-NEXT: [[L_1_LCSSA3:%.*]] = phi ptr [ [[L_1]], [[INNER_LATCH]] ]
7976
; CHECK-NEXT: br label [[EXIT]]
8077
; CHECK: exit:
81-
; CHECK-NEXT: [[L_14:%.*]] = phi ptr [ [[L_1_LCSSA3]], [[EXIT_LOOPEXIT1]] ], [ [[L_1_LCSSA]], [[EXIT_LOOPEXIT]] ]
82-
; CHECK-NEXT: [[L_3:%.*]] = load i16, ptr [[L_14]], align 2
78+
; CHECK-NEXT: [[L_3:%.*]] = load i16, ptr [[L_1]], align 2
8379
; CHECK-NEXT: ret i16 [[L_3]]
8480
;
8581
entry:

0 commit comments

Comments
 (0)