Skip to content

Commit 071392e

Browse files
fhahnfhahn
committed
!fixup address latest comment, thanks
1 parent be373bf commit 071392e

18 files changed

+99
-55
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2643,14 +2643,12 @@ void InnerLoopVectorizer::createInductionAdditionalBypassValue(
26432643
B.setFastMathFlags(II.getInductionBinOp()->getFastMathFlags());
26442644

26452645
// Compute the end value for the additional bypass.
2646-
if (MainVectorTripCount) {
2647-
B.SetInsertPoint(getAdditionalBypassBlock(),
2648-
getAdditionalBypassBlock()->getFirstInsertionPt());
2649-
EndValueFromAdditionalBypass =
2650-
emitTransformedIndex(B, MainVectorTripCount, II.getStartValue(), Step,
2651-
II.getKind(), II.getInductionBinOp());
2652-
EndValueFromAdditionalBypass->setName("ind.end");
2653-
}
2646+
B.SetInsertPoint(getAdditionalBypassBlock(),
2647+
getAdditionalBypassBlock()->getFirstInsertionPt());
2648+
EndValueFromAdditionalBypass =
2649+
emitTransformedIndex(B, MainVectorTripCount, II.getStartValue(), Step,
2650+
II.getKind(), II.getInductionBinOp());
2651+
EndValueFromAdditionalBypass->setName("ind.end");
26542652
}
26552653

26562654
// Store the bypass value here, as it needs to be added as operand to its
@@ -9722,11 +9720,12 @@ void VPDerivedIVRecipe::execute(VPTransformState &State) {
97229720
Value *Step = State.get(getStepValue(), VPLane(0));
97239721
Value *Index = State.get(getOperand(1), VPLane(0));
97249722
Value *DerivedIV = emitTransformedIndex(
9725-
State.Builder, Index, getStartValue()->getLiveInIRValue(), Step,
9726-
Kind, cast_if_present<BinaryOperator>(FPBinOp));
9723+
State.Builder, Index, getStartValue()->getLiveInIRValue(), Step, Kind,
9724+
cast_if_present<BinaryOperator>(FPBinOp));
97279725
DerivedIV->setName(Name);
9728-
// Index may only bet set to constant 0 in prepareToExecute.
9729-
assert((DerivedIV != Index || cast<ConstantInt>(Index)->isNullValue()) && "IV didn't need transforming?");
9726+
// Index may only be set to constant 0 in prepareToExecute.
9727+
assert((DerivedIV != Index || cast<ConstantInt>(Index)->isNullValue()) &&
9728+
"IV didn't need transforming?");
97309729
State.set(this, DerivedIV, VPLane(0));
97319730
}
97329731

‎llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -391,8 +391,8 @@ using VPDerivedIV_match =
391391
Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0, false, VPDerivedIVRecipe>;
392392

393393
template <typename Op0_t, typename Op1_t, typename Op2_t>
394-
inline VPDerivedIV_match<Op0_t, Op1_t, Op2_t> m_DerivedIV(const Op0_t &Op0,
395-
const Op1_t &Op1, const Op2_t &Op2) {
394+
inline VPDerivedIV_match<Op0_t, Op1_t, Op2_t>
395+
m_DerivedIV(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
396396
return VPDerivedIV_match<Op0_t, Op1_t, Op2_t>({Op0, Op1, Op2});
397397
}
398398

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1061,7 +1061,12 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
10611061
if (match(&R, m_c_Mul(m_VPValue(A), m_SpecificInt(1))))
10621062
return R.getVPSingleValue()->replaceAllUsesWith(A);
10631063

1064-
if ((match(&R, m_DerivedIV(m_SpecificInt(0), m_VPValue(A), m_SpecificInt(1))) || match(&R, m_DerivedIV(m_SpecificInt(0), m_SpecificInt(0), m_VPValue())) ) && TypeInfo.inferScalarType(R.getOperand(1)) == TypeInfo.inferScalarType(R.getVPSingleValue()))
1064+
if ((match(&R,
1065+
m_DerivedIV(m_SpecificInt(0), m_VPValue(A), m_SpecificInt(1))) ||
1066+
match(&R,
1067+
m_DerivedIV(m_SpecificInt(0), m_SpecificInt(0), m_VPValue()))) &&
1068+
TypeInfo.inferScalarType(R.getOperand(1)) ==
1069+
TypeInfo.inferScalarType(R.getVPSingleValue()))
10651070
return R.getVPSingleValue()->replaceAllUsesWith(R.getOperand(1));
10661071
}
10671072

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ target triple = "aarch64-unknown-linux-gnu"
1111
; VPLANS-LABEL: Checking a loop in 'simple_memset'
1212
; VPLANS: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF>=1' {
1313
; VPLANS-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
14+
; VPLANS-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
1415
; VPLANS-NEXT: vp<[[TC:%[0-9]+]]> = original trip-count
1516
; VPLANS-EMPTY:
1617
; VPLANS-NEXT: ir-bb<entry>:

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ target triple = "aarch64-unknown-linux-gnu"
1717
; CHECK-NEXT: Successor(s): vector.ph
1818
; CHECK-EMPTY:
1919
; CHECK-NEXT: vector.ph:
20+
; CHECK-NEXT: vp<[[END1:%.+]]> = DERIVED-IV ir<%start.1> + vp<[[VEC_TC]]> * ir<8>
21+
; CHECK-NEXT: vp<[[END2:%.+]]> = DERIVED-IV ir<%start.2> + vp<[[VEC_TC]]> * ir<1>
2022
; CHECK-NEXT: Successor(s): vector loop
2123
; CHECK-EMPTY:
2224
; CHECK-NEXT: <x1> vector loop: {

‎llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,11 @@ target triple = "aarch64-unknown-linux-gnu"
4545
; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
4646
; CHECK-EMPTY:
4747
; CHECK-NEXT: scalar.ph:
48+
; CHECK-NEXT: EMIT vp<[[RESUME:%.+]]> = resume-phi [[VTC]], ir<0>
4849
; CHECK-NEXT: Successor(s): ir-bb<for.body>
4950
; CHECK-EMPTY:
5051
; CHECK-NEXT: ir-bb<for.body>:
51-
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
52+
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] (extra operand: vp<[[RESUME]]> from scalar.ph)
5253
; CHECK: IR %exitcond = icmp eq i64 %iv.next, %N
5354
; CHECK-NEXT: No successors
5455
; CHECK-EMPTY:
@@ -90,10 +91,11 @@ target triple = "aarch64-unknown-linux-gnu"
9091
; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
9192
; CHECK-EMPTY:
9293
; CHECK-NEXT: scalar.ph:
94+
; CHECK-NEXT: EMIT vp<[[RESUME:%.+]]> = resume-phi [[VTC]], ir<0>
9395
; CHECK-NEXT: Successor(s): ir-bb<for.body>
9496
; CHECK-EMPTY:
9597
; CHECK-NEXT: ir-bb<for.body>:
96-
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
98+
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] (extra operand: vp<[[RESUME]]> from scalar.ph)
9799
; CHECK: IR %exitcond = icmp eq i64 %iv.next, %N
98100
; CHECK-NEXT: No successors
99101
; CHECK-EMPTY:

‎llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,11 @@ target triple = "aarch64-unknown-linux-gnu"
4343
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
4444
; CHECK-EMPTY:
4545
; CHECK-NEXT: scalar.ph:
46+
; CHECK-NEXT: EMIT vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
4647
; CHECK-NEXT: Successor(s): ir-bb<for.body>
4748
; CHECK-EMPTY:
4849
; CHECK-NEXT: ir-bb<for.body>:
49-
; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
50+
; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] (extra operand: vp<[[RESUME]]> from scalar.ph)
5051
; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024
5152
; CHECK-NEXT: No successors
5253
; CHECK-EMPTY:
@@ -88,10 +89,11 @@ target triple = "aarch64-unknown-linux-gnu"
8889
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
8990
; CHECK-EMPTY:
9091
; CHECK-NEXT: scalar.ph:
92+
; CHECK-NEXT: EMIT vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
9193
; CHECK-NEXT: Successor(s): ir-bb<for.body>
9294
; CHECK-EMPTY:
9395
; CHECK-NEXT: ir-bb<for.body>:
94-
; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
96+
; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] (extra operand: vp<[[RESUME]]> from scalar.ph)
9597
; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024
9698
; CHECK-NEXT: No successors
9799
; CHECK-EMPTY:
@@ -138,6 +140,7 @@ target triple = "aarch64-unknown-linux-gnu"
138140
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
139141
; CHECK-EMPTY:
140142
; CHECK-NEXT: scalar.ph:
143+
; CHECK-NEXT: EMIT vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
141144
; CHECK-NEXT: Successor(s): ir-bb<for.body>
142145
; CHECK-EMPTY:
143146
; CHECK-NEXT: ir-bb<for.body>:
@@ -183,10 +186,11 @@ target triple = "aarch64-unknown-linux-gnu"
183186
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
184187
; CHECK-EMPTY:
185188
; CHECK-NEXT: scalar.ph:
189+
; CHECK-NEXT: EMIT vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
186190
; CHECK-NEXT: Successor(s): ir-bb<for.body>
187191
; CHECK-EMPTY:
188192
; CHECK-NEXT: ir-bb<for.body>:
189-
; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
193+
; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] (extra operand: vp<[[RESUME]]> from scalar.ph)
190194
; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024
191195
; CHECK-NEXT: No successors
192196
; CHECK-EMPTY:
@@ -232,6 +236,7 @@ target triple = "aarch64-unknown-linux-gnu"
232236
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
233237
; CHECK-EMPTY:
234238
; CHECK-NEXT: scalar.ph:
239+
; CHECK-NEXT: EMIT vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
235240
; CHECK-NEXT: Successor(s): ir-bb<for.body>
236241
; CHECK-EMPTY:
237242
; CHECK-NEXT: ir-bb<for.body>:
@@ -277,10 +282,11 @@ target triple = "aarch64-unknown-linux-gnu"
277282
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
278283
; CHECK-EMPTY:
279284
; CHECK-NEXT: scalar.ph:
285+
; CHECK-NEXT: EMIT vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
280286
; CHECK-NEXT: Successor(s): ir-bb<for.body>
281287
; CHECK-EMPTY:
282288
; CHECK-NEXT: ir-bb<for.body>:
283-
; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
289+
; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] (extra operand: vp<[[RESUME]]> from scalar.ph)
284290
; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024
285291
; CHECK-NEXT: No successors
286292
; CHECK-EMPTY:

‎llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,11 @@ target triple = "arm64-apple-ios"
4141
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
4242
; CHECK-EMPTY:
4343
; CHECK-NEXT: scalar.ph:
44+
; CHECK-NEXT: EMIT vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
4445
; CHECK-NEXT: Successor(s): ir-bb<loop>
4546
; CHECK-EMPTY:
4647
; CHECK-NEXT: ir-bb<loop>:
47-
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
48+
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<[[RESUME]]> from scalar.ph)
4849
; CHECK: IR %cmp = icmp ne i64 %iv.next, 1024
4950
; CHECK-NEXT: No successors
5051
; CHECK-EMPTY:
@@ -86,10 +87,11 @@ target triple = "arm64-apple-ios"
8687
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
8788
; CHECK-EMPTY:
8889
; CHECK-NEXT: scalar.ph:
90+
; CHECK-NEXT: EMIT vp<[[RESUME:%.+]]> = resume-phi vp<[[VTC]]>, ir<0>
8991
; CHECK-NEXT: Successor(s): ir-bb<loop>
9092
; CHECK-EMPTY:
9193
; CHECK-NEXT: ir-bb<loop>:
92-
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
94+
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<[[RESUME]]> from scalar.ph)
9395
; CHECK: IR %cmp = icmp ne i64 %iv.next, 1024
9496
; CHECK-NEXT: No successors
9597
; CHECK-EMPTY:

‎llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ define i32 @conversion_cost1(i32 %n, ptr nocapture %A, ptr nocapture %B) nounwin
4242
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
4343
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
4444
; CHECK: vec.epilog.ph:
45-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ 3, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
4645
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
46+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ 3, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
4747
; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[TMP3]], 4
4848
; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF2]]
4949
; CHECK-NEXT: [[IND_END4:%.*]] = add i64 3, [[N_VEC3]]

‎llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,8 @@ define void @fp_iv_loop1(ptr noalias nocapture %A, i32 %N) #0 {
6161
; AUTO_VEC-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp eq i64 [[N_VEC_REMAINING]], 0
6262
; AUTO_VEC-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[FOR_BODY]], label [[VEC_EPILOG_PH]]
6363
; AUTO_VEC: vec.epilog.ph:
64-
; AUTO_VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi float [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+00, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
6564
; AUTO_VEC-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
65+
; AUTO_VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi float [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+00, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
6666
; AUTO_VEC-NEXT: [[N_VEC3:%.*]] = and i64 [[ZEXT]], 2147483644
6767
; AUTO_VEC-NEXT: [[DOTCAST5:%.*]] = uitofp nneg i64 [[N_VEC3]] to float
6868
; AUTO_VEC-NEXT: [[TMP7:%.*]] = fmul fast float [[DOTCAST5]], 5.000000e-01
@@ -441,8 +441,8 @@ define void @fadd_reassoc_FMF(ptr nocapture %p, i32 %N) {
441441
; AUTO_VEC-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp eq i64 [[N_VEC_REMAINING]], 0
442442
; AUTO_VEC-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[FOR_BODY]], label [[VEC_EPILOG_PH]]
443443
; AUTO_VEC: vec.epilog.ph:
444-
; AUTO_VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi float [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+00, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
445444
; AUTO_VEC-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
445+
; AUTO_VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi float [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+00, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
446446
; AUTO_VEC-NEXT: [[N_VEC6:%.*]] = and i64 [[TMP0]], 4294967292
447447
; AUTO_VEC-NEXT: [[DOTCAST8:%.*]] = uitofp nneg i64 [[N_VEC6]] to float
448448
; AUTO_VEC-NEXT: [[TMP12:%.*]] = fmul reassoc float [[DOTCAST8]], 4.200000e+01

0 commit comments

Comments
 (0)