Skip to content

Commit f01a793

Browse files
lukel97lukel97
authored
[VPlan] Replace all uses of VF when EVL tail folding. NFCI (#146339)
With EVL tail folding, any use of the VF live in should be replaced by the EVL. Otherwise, it should likely be directly emitted as a constant via VPTransformState::VF. This strengthens the EVL transformation by replacing all uses of VF with EVL and asserting that the only users are VPVectorEndPointerRecipe and VPScalarIVStepsRecipe, the latter of which is new. This should be NFC because even though we didn't previously replace the EVL of VPScalarIVStepsRecipe, it's only used when unrolling which we don't allow with EVL tail folding yet.
1 parent 652630b commit f01a793

File tree

6 files changed

+35
-50
lines changed

6 files changed

+35
-50
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2198,6 +2198,11 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
21982198
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
21992199
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
22002200

2201+
assert(all_of(Plan.getVF().users(),
2202+
IsaPred<VPVectorEndPointerRecipe, VPScalarIVStepsRecipe>) &&
2203+
"User of VF that we can't transform to EVL.");
2204+
Plan.getVF().replaceAllUsesWith(&EVL);
2205+
22012206
// Create a scalar phi to track the previous EVL if fixed-order recurrence is
22022207
// contained.
22032208
VPInstruction *PrevEVL = nullptr;
@@ -2216,11 +2221,6 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
22162221
PrevEVL = Builder.createScalarPhi({MaxEVL, &EVL}, DebugLoc(), "prev.evl");
22172222
}
22182223

2219-
for (VPUser *U : to_vector(Plan.getVF().users())) {
2220-
if (auto *R = dyn_cast<VPVectorEndPointerRecipe>(U))
2221-
R->setOperand(1, &EVL);
2222-
}
2223-
22242224
SmallVector<VPRecipeBase *> ToErase;
22252225

22262226
for (VPValue *HeaderMask : collectAllHeaderMasks(Plan)) {

‎llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,13 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
158158
})
159159
.Case<VPWidenStoreEVLRecipe, VPReductionEVLRecipe>(
160160
[&](const VPRecipeBase *S) { return VerifyEVLUse(*S, 2); })
161+
.Case<VPScalarIVStepsRecipe>([&](auto *R) {
162+
if (R->getNumOperands() != 3) {
163+
errs() << "Unrolling with EVL tail folding not yet supported\n";
164+
return false;
165+
}
166+
return VerifyEVLUse(*R, 2);
167+
})
161168
.Case<VPWidenLoadEVLRecipe, VPVectorEndPointerRecipe>(
162169
[&](const VPRecipeBase *R) { return VerifyEVLUse(*R, 1); })
163170
.Case<VPInstructionWithType>(

‎llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll‎

Lines changed: 9 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
1010
; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
1111
;
1212
; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
13-
; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
1413
; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
1514
; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
1615
; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -24,7 +23,7 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
2423
; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
2524
; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
2625
; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
27-
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]
26+
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]
2827
; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
2928
; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
3029
; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -67,7 +66,6 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
6766
; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
6867
;
6968
; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
70-
; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
7169
; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
7270
; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
7371
; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -81,7 +79,7 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
8179
; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
8280
; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
8381
; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
84-
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
82+
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
8583
; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
8684
; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
8785
; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -124,7 +122,6 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
124122
; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
125123
;
126124
; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
127-
; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
128125
; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
129126
; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
130127
; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -138,7 +135,7 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
138135
; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
139136
; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
140137
; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
141-
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
138+
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
142139
; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
143140
; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
144141
; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -181,7 +178,6 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
181178
; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
182179
;
183180
; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
184-
; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
185181
; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
186182
; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
187183
; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -195,7 +191,7 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
195191
; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
196192
; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
197193
; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
198-
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]
194+
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]
199195
; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
200196
; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
201197
; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -238,7 +234,6 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
238234
; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
239235
;
240236
; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
241-
; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
242237
; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
243238
; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
244239
; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -252,7 +247,7 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
252247
; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
253248
; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
254249
; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
255-
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
250+
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
256251
; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
257252
; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
258253
; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -290,7 +285,6 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
290285
; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
291286
;
292287
; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
293-
; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
294288
; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
295289
; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
296290
; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -304,7 +298,7 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
304298
; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
305299
; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
306300
; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
307-
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
301+
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
308302
; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
309303
; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
310304
; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -342,7 +336,6 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
342336
; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
343337
;
344338
; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
345-
; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
346339
; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
347340
; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
348341
; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -356,7 +349,7 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
356349
; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
357350
; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
358351
; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
359-
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
352+
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
360353
; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
361354
; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
362355
; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -398,7 +391,6 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
398391
; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
399392
;
400393
; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
401-
; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
402394
; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
403395
; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
404396
; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -412,7 +404,7 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
412404
; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
413405
; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
414406
; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
415-
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
407+
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
416408
; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
417409
; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
418410
; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -454,7 +446,6 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
454446
; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
455447
;
456448
; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
457-
; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
458449
; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
459450
; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
460451
; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -468,7 +459,7 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
468459
; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
469460
; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
470461
; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
471-
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
462+
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
472463
; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
473464
; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
474465
; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>

0 commit comments

Comments
 (0)