Skip to content

Commit b4dbb1c

Browse files
fhahnfhahn
authored
[VPlan] Be more careful with CSE in replicate regions. (llvm#162110)
Recipes in replicate regions implicitly depend on the region's predicate. Limit CSE to recipes in the same block, when either recipe is in a replicate region. This allows handling VPPredInstPHIRecipe during CSE. If we perform CSE on recipes inside a replicate region, we may end up with 2 VPPredInstPHIRecipes sharing the same operand. This is incompatible with current VPPredInstPHIRecipe codegen, which re-sets the current value of its operand in VPTransformState. This can cause crashes in the added test cases. Note that this patch only modifies ::isEqual to check for replicating regions and not getHash, as CSE across replicating regions should be uncommon. Fixes llvm#157314. Fixes llvm#161974. PR: llvm#162110
1 parent 9fe1f29 commit b4dbb1c

File tree

2 files changed

+173
-1
lines changed

2 files changed

+173
-1
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2012,7 +2012,7 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
20122012
.Case<VPWidenIntrinsicRecipe>([](auto *I) {
20132013
return std::make_pair(true, I->getVectorIntrinsicID());
20142014
})
2015-
.Case<VPVectorPointerRecipe>([](auto *I) {
2015+
.Case<VPVectorPointerRecipe, VPPredInstPHIRecipe>([](auto *I) {
20162016
// For recipes that do not directly map to LLVM IR instructions,
20172017
// assign opcodes after the last VPInstruction opcode (which is also
20182018
// after the last IR Instruction opcode), based on the VPDefID.
@@ -2089,6 +2089,15 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
20892089
LFlags->getPredicate() !=
20902090
cast<VPRecipeWithIRFlags>(R)->getPredicate())
20912091
return false;
2092+
// Recipes in replicate regions implicitly depend on predicate. If either
2093+
// recipe is in a replicate region, only consider them equal if both have
2094+
// the same parent.
2095+
const VPRegionBlock *RegionL = L->getParent()->getParent();
2096+
const VPRegionBlock *RegionR = R->getParent()->getParent();
2097+
if (((RegionL && RegionL->isReplicator()) ||
2098+
(RegionR && RegionR->isReplicator())) &&
2099+
L->getParent() != R->getParent())
2100+
return false;
20922101
const VPlan *Plan = L->getParent()->getPlan();
20932102
VPTypeAnalysis TypeInfo(*Plan);
20942103
return TypeInfo.inferScalarType(L) == TypeInfo.inferScalarType(R);

‎llvm/test/Transforms/LoopVectorize/cse-replicate-regions.ll‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
2+
; RUN: opt -p loop-vectorize -force-vector-width=2 -force-widen-divrem-via-safe-divisor=false -S %s | FileCheck %s
3+
4+
define void @multiple_vppredinstphi_with_same_predicate(ptr %A, i32 %d) {
5+
; CHECK-LABEL: define void @multiple_vppredinstphi_with_same_predicate(
6+
; CHECK-SAME: ptr [[A:%.*]], i32 [[D:%.*]]) {
7+
; CHECK-NEXT: [[ENTRY:.*:]]
8+
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
9+
; CHECK: [[VECTOR_PH]]:
10+
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
11+
; CHECK: [[VECTOR_BODY]]:
12+
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE2:.*]] ]
13+
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDEX]]
14+
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
15+
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD]], zeroinitializer
16+
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0
17+
; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
18+
; CHECK: [[PRED_SDIV_IF]]:
19+
; CHECK-NEXT: [[TMP3:%.*]] = sdiv i32 -10, [[D]]
20+
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i32 0
21+
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
22+
; CHECK: [[PRED_SDIV_CONTINUE]]:
23+
; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_SDIV_IF]] ]
24+
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1
25+
; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2]]
26+
; CHECK: [[PRED_SDIV_IF1]]:
27+
; CHECK-NEXT: [[TMP7:%.*]] = sdiv i32 -10, [[D]]
28+
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP7]], i32 1
29+
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
30+
; CHECK: [[PRED_SDIV_CONTINUE2]]:
31+
; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ [[TMP5]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP8]], %[[PRED_SDIV_IF1]] ]
32+
; CHECK-NEXT: [[TMP10:%.*]] = add <2 x i32> [[TMP9]], [[TMP9]]
33+
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[TMP10]], <2 x i32> zeroinitializer
34+
; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP0]], align 4
35+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
36+
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
37+
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
38+
; CHECK: [[MIDDLE_BLOCK]]:
39+
; CHECK-NEXT: br label %[[EXIT:.*]]
40+
; CHECK: [[EXIT]]:
41+
; CHECK-NEXT: ret void
42+
;
43+
entry:
44+
br label %loop.header
45+
46+
loop.header:
47+
%iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
48+
%gep.A = getelementptr inbounds i32, ptr %A, i32 %iv
49+
%l = load i32, ptr %gep.A
50+
%c = icmp sgt i32 %l, 0
51+
br i1 %c, label %then, label %loop.latch
52+
53+
then:
54+
%div.0 = sdiv i32 -10, %d
55+
%div.1 = sdiv i32 -10, %d
56+
%add = add i32 %div.1, %div.0
57+
br label %loop.latch
58+
59+
loop.latch:
60+
%merge = phi i32 [ %add, %then ], [ 0, %loop.header ]
61+
store i32 %merge, ptr %gep.A
62+
%iv.next = add i32 %iv, 1
63+
%ec = icmp eq i32 %iv.next, 100
64+
br i1 %ec, label %exit, label %loop.header
65+
66+
exit:
67+
ret void
68+
}
69+
70+
define void @multiple_vppredinstphi_with_different_predicate(ptr %A, i32 %d) {
71+
; CHECK-LABEL: define void @multiple_vppredinstphi_with_different_predicate(
72+
; CHECK-SAME: ptr [[A:%.*]], i32 [[D:%.*]]) {
73+
; CHECK-NEXT: [[ENTRY:.*:]]
74+
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
75+
; CHECK: [[VECTOR_PH]]:
76+
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
77+
; CHECK: [[VECTOR_BODY]]:
78+
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE6:.*]] ]
79+
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDEX]]
80+
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
81+
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD]], zeroinitializer
82+
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0
83+
; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
84+
; CHECK: [[PRED_SDIV_IF]]:
85+
; CHECK-NEXT: [[TMP3:%.*]] = sdiv i32 -10, [[D]]
86+
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i32 0
87+
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
88+
; CHECK: [[PRED_SDIV_CONTINUE]]:
89+
; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_SDIV_IF]] ]
90+
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1
91+
; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
92+
; CHECK: [[PRED_SDIV_IF1]]:
93+
; CHECK-NEXT: [[TMP7:%.*]] = sdiv i32 -10, [[D]]
94+
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP7]], i32 1
95+
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
96+
; CHECK: [[PRED_SDIV_CONTINUE2]]:
97+
; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ [[TMP5]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP8]], %[[PRED_SDIV_IF1]] ]
98+
; CHECK-NEXT: [[TMP10:%.*]] = xor <2 x i1> [[TMP1]], splat (i1 true)
99+
; CHECK-NEXT: [[TMP11:%.*]] = or <2 x i1> [[TMP1]], [[TMP10]]
100+
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[TMP9]], <2 x i32> zeroinitializer
101+
; CHECK-NEXT: [[TMP12:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD]], splat (i32 20)
102+
; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP11]], <2 x i1> [[TMP12]], <2 x i1> zeroinitializer
103+
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP13]], i32 0
104+
; CHECK-NEXT: br i1 [[TMP14]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
105+
; CHECK: [[PRED_SDIV_IF3]]:
106+
; CHECK-NEXT: [[TMP15:%.*]] = sdiv i32 -10, [[D]]
107+
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <2 x i32> poison, i32 [[TMP15]], i32 0
108+
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
109+
; CHECK: [[PRED_SDIV_CONTINUE4]]:
110+
; CHECK-NEXT: [[TMP17:%.*]] = phi <2 x i32> [ poison, %[[PRED_SDIV_CONTINUE2]] ], [ [[TMP16]], %[[PRED_SDIV_IF3]] ]
111+
; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x i1> [[TMP13]], i32 1
112+
; CHECK-NEXT: br i1 [[TMP18]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6]]
113+
; CHECK: [[PRED_SDIV_IF5]]:
114+
; CHECK-NEXT: [[TMP19:%.*]] = sdiv i32 -10, [[D]]
115+
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <2 x i32> [[TMP17]], i32 [[TMP19]], i32 1
116+
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
117+
; CHECK: [[PRED_SDIV_CONTINUE6]]:
118+
; CHECK-NEXT: [[TMP21:%.*]] = phi <2 x i32> [ [[TMP17]], %[[PRED_SDIV_CONTINUE4]] ], [ [[TMP20]], %[[PRED_SDIV_IF5]] ]
119+
; CHECK-NEXT: [[PREDPHI7:%.*]] = select <2 x i1> [[TMP12]], <2 x i32> [[TMP21]], <2 x i32> zeroinitializer
120+
; CHECK-NEXT: [[TMP22:%.*]] = add <2 x i32> [[PREDPHI]], [[PREDPHI7]]
121+
; CHECK-NEXT: store <2 x i32> [[TMP22]], ptr [[TMP0]], align 4
122+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
123+
; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
124+
; CHECK-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
125+
; CHECK: [[MIDDLE_BLOCK]]:
126+
; CHECK-NEXT: br label %[[EXIT:.*]]
127+
; CHECK: [[EXIT]]:
128+
; CHECK-NEXT: ret void
129+
;
130+
entry:
131+
br label %loop.header
132+
133+
loop.header:
134+
%iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
135+
%gep.A = getelementptr inbounds i32, ptr %A, i32 %iv
136+
%l = load i32, ptr %gep.A
137+
%c.0 = icmp sgt i32 %l, 0
138+
br i1 %c.0, label %then.0, label %continue
139+
140+
then.0:
141+
%div.0 = sdiv i32 -10, %d
142+
br label %continue
143+
144+
continue:
145+
%merge.0 = phi i32 [ %div.0, %then.0 ], [ 0, %loop.header ]
146+
%c.1 = icmp sgt i32 %l, 20
147+
br i1 %c.1, label %then.1, label %loop.latch
148+
149+
then.1:
150+
%div.1 = sdiv i32 -10, %d
151+
br label %loop.latch
152+
153+
loop.latch:
154+
%merge.1 = phi i32 [ %div.1, %then.1 ], [ 0, %continue ]
155+
%add = add i32 %merge.0, %merge.1
156+
store i32 %add, ptr %gep.A
157+
%iv.next = add i32 %iv, 1
158+
%ec = icmp eq i32 %iv.next, 100
159+
br i1 %ec, label %exit, label %loop.header
160+
161+
exit:
162+
ret void
163+
}

0 commit comments

Comments
 (0)