Skip to content

Commit 0376602

Browse files
fhahnfhahn
committed
[VPlan] Truncate/Extend ComputeReductionResult at construction (NFC).
Instead of looking up the narrower reduction type via getRecurrenceType we can generate the needed extend directly at constructiond re-use the truncated value from the loop.
1 parent 749c7c5 commit 0376602

File tree

6 files changed

+36
-55
lines changed

6 files changed

+36
-55
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -7255,7 +7255,10 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
72557255
// Get the VPInstruction computing the reduction result in the middle block.
72567256
// The first operand may not be from the middle block if it is not connected
72577257
// to the scalar preheader. In that case, there's nothing to fix.
7258-
auto *EpiRedResult = dyn_cast<VPInstruction>(EpiResumePhiR->getOperand(0));
7258+
VPValue *Incoming = EpiResumePhiR->getOperand(0);
7259+
match(Incoming, VPlanPatternMatch::m_ZExtOrSExt(
7260+
VPlanPatternMatch::m_VPValue(Incoming)));
7261+
auto *EpiRedResult = dyn_cast<VPInstruction>(Incoming);
72597262
if (!EpiRedResult ||
72607263
(EpiRedResult->getOpcode() != VPInstruction::ComputeAnyOfResult &&
72617264
EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult &&
@@ -9206,28 +9209,6 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
92069209
PhiR->setOperand(1, NewExitingVPV);
92079210
}
92089211

9209-
// If the vector reduction can be performed in a smaller type, we truncate
9210-
// then extend the loop exit value to enable InstCombine to evaluate the
9211-
// entire expression in the smaller type.
9212-
if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
9213-
!RecurrenceDescriptor::isAnyOfRecurrenceKind(
9214-
RdxDesc.getRecurrenceKind())) {
9215-
assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
9216-
Type *RdxTy = RdxDesc.getRecurrenceType();
9217-
auto *Trunc =
9218-
new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
9219-
auto *Extnd =
9220-
RdxDesc.isSigned()
9221-
? new VPWidenCastRecipe(Instruction::SExt, Trunc, PhiTy)
9222-
: new VPWidenCastRecipe(Instruction::ZExt, Trunc, PhiTy);
9223-
9224-
Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
9225-
Extnd->insertAfter(Trunc);
9226-
if (PhiR->getOperand(1) == NewExitingVPV)
9227-
PhiR->setOperand(1, Extnd->getVPSingleValue());
9228-
NewExitingVPV = Extnd;
9229-
}
9230-
92319212
// We want code in the middle block to appear to execute on the location of
92329213
// the scalar loop's latch terminator because: (a) it is all compiler
92339214
// generated, (b) these instructions are always executed after evaluating
@@ -9266,6 +9247,31 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
92669247
Builder.createNaryOp(VPInstruction::ComputeReductionResult,
92679248
{PhiR, NewExitingVPV}, Flags, ExitDL);
92689249
}
9250+
// If the vector reduction can be performed in a smaller type, we truncate
9251+
// then extend the loop exit value to enable InstCombine to evaluate the
9252+
// entire expression in the smaller type.
9253+
if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
9254+
!RecurrenceDescriptor::isAnyOfRecurrenceKind(
9255+
RdxDesc.getRecurrenceKind())) {
9256+
assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
9257+
Type *RdxTy = RdxDesc.getRecurrenceType();
9258+
auto *Trunc =
9259+
new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
9260+
Instruction::CastOps ExtendOpc =
9261+
RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
9262+
auto *Extnd = new VPWidenCastRecipe(ExtendOpc, Trunc, PhiTy);
9263+
Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
9264+
Extnd->insertAfter(Trunc);
9265+
if (PhiR->getOperand(1) == NewExitingVPV)
9266+
PhiR->setOperand(1, Extnd->getVPSingleValue());
9267+
9268+
// Update ComputeReductionResult with the truncated exiting value and
9269+
// extend its result.
9270+
FinalReductionResult->setOperand(1, Trunc);
9271+
FinalReductionResult =
9272+
Builder.createScalarCast(ExtendOpc, FinalReductionResult, PhiTy, {});
9273+
}
9274+
92699275
// Update all users outside the vector region. Also replace redundant
92709276
// ExtractLastElement.
92719277
for (auto *U : to_vector(OrigExitingVPV->users())) {

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp‎

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -771,7 +771,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
771771
assert(!RecurrenceDescriptor::isFindIVRecurrenceKind(RK) &&
772772
"should be handled by ComputeFindIVResult");
773773

774-
Type *ResultTy = State.TypeAnalysis.inferScalarType(this);
775774
// The recipe's operands are the reduction phi, followed by one operand for
776775
// each part of the reduction.
777776
unsigned UF = getNumOperands() - 1;
@@ -783,15 +782,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
783782
if (hasFastMathFlags())
784783
Builder.setFastMathFlags(getFastMathFlags());
785784

786-
// If the vector reduction can be performed in a smaller type, we truncate
787-
// then extend the loop exit value to enable InstCombine to evaluate the
788-
// entire expression in the smaller type.
789-
// TODO: Handle this in truncateToMinBW.
790-
if (State.VF.isVector() && ResultTy != RdxDesc.getRecurrenceType()) {
791-
Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), State.VF);
792-
for (unsigned Part = 0; Part < UF; ++Part)
793-
RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
794-
}
795785
// Reduce all of the unrolled parts into a single vector.
796786
Value *ReducedPartRdx = RdxParts[0];
797787
if (PhiR->isOrdered()) {
@@ -816,13 +806,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
816806
// All ops in the reduction inherit fast-math-flags from the recurrence
817807
// descriptor.
818808
ReducedPartRdx = createSimpleReduction(Builder, ReducedPartRdx, RK);
819-
820-
// If the reduction can be performed in a smaller type, we need to extend
821-
// the reduction to the wider type before we branch to the original loop.
822-
if (ResultTy != RdxDesc.getRecurrenceType())
823-
ReducedPartRdx = RdxDesc.isSigned()
824-
? Builder.CreateSExt(ReducedPartRdx, ResultTy)
825-
: Builder.CreateZExt(ReducedPartRdx, ResultTy);
826809
}
827810

828811
return ReducedPartRdx;

‎llvm/test/Transforms/LoopVectorize/X86/cost-model.ll‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1167,8 +1167,7 @@ define i32 @narrowed_reduction(ptr %a, i1 %cmp) #0 {
11671167
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
11681168
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
11691169
; CHECK: middle.block:
1170-
; CHECK-NEXT: [[TMP10:%.*]] = trunc <16 x i32> [[TMP7]] to <16 x i1>
1171-
; CHECK-NEXT: [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP10]])
1170+
; CHECK-NEXT: [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]])
11721171
; CHECK-NEXT: [[TMP21:%.*]] = zext i1 [[TMP20]] to i32
11731172
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[VEC_EPILOG_PH]]
11741173
; CHECK: scalar.ph:

‎llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -208,8 +208,7 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
208208
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
209209
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
210210
; CHECK: middle.block:
211-
; CHECK-NEXT: [[TMP9:%.*]] = trunc <4 x i32> [[TMP7]] to <4 x i16>
212-
; CHECK-NEXT: [[TMP10:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP9]])
211+
; CHECK-NEXT: [[TMP10:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP6]])
213212
; CHECK-NEXT: [[TMP11:%.*]] = zext i16 [[TMP10]] to i32
214213
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
215214
; CHECK: vec.epilog.iter.check:
@@ -234,8 +233,7 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
234233
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT4]], 256
235234
; CHECK-NEXT: br i1 [[TMP21]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
236235
; CHECK: vec.epilog.middle.block:
237-
; CHECK-NEXT: [[TMP22:%.*]] = trunc <4 x i32> [[TMP20]] to <4 x i16>
238-
; CHECK-NEXT: [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP22]])
236+
; CHECK-NEXT: [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP19]])
239237
; CHECK-NEXT: [[TMP24:%.*]] = zext i16 [[TMP23]] to i32
240238
; CHECK-NEXT: br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
241239
; CHECK: vec.epilog.scalar.ph:

‎llvm/test/Transforms/LoopVectorize/reduction-small-size.ll‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,7 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n) {
2525
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
2626
; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
2727
; CHECK: middle.block:
28-
; CHECK-NEXT: [[TMP6:%.*]] = trunc <4 x i32> [[TMP4]] to <4 x i8>
29-
; CHECK-NEXT: [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP6]])
28+
; CHECK-NEXT: [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP3]])
3029
; CHECK-NEXT: [[TMP8:%.*]] = zext i8 [[TMP7]] to i32
3130
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
3231
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -104,8 +103,7 @@ define i8 @PR34687_no_undef(i1 %c, i32 %x, i32 %n) {
104103
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
105104
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
106105
; CHECK: middle.block:
107-
; CHECK-NEXT: [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i8>
108-
; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP8]])
106+
; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP5]])
109107
; CHECK-NEXT: [[TMP10:%.*]] = zext i8 [[TMP9]] to i32
110108
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
111109
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -183,8 +181,7 @@ define i32 @PR35734(i32 %x, i32 %y) {
183181
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
184182
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
185183
; CHECK: middle.block:
186-
; CHECK-NEXT: [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i1>
187-
; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> [[TMP8]])
184+
; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> [[TMP5]])
188185
; CHECK-NEXT: [[TMP10:%.*]] = sext i1 [[TMP9]] to i32
189186
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
190187
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]

‎llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,7 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
4343
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
4444
; CHECK-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4545
; CHECK: middle.block:
46-
; CHECK-NEXT: [[TMP37:%.*]] = trunc <vscale x 8 x i32> [[TMP34]] to <vscale x 8 x i8>
47-
; CHECK-NEXT: [[TMP38:%.*]] = trunc <vscale x 8 x i32> [[TMP36]] to <vscale x 8 x i8>
48-
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 8 x i8> [[TMP38]], [[TMP37]]
46+
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 8 x i8> [[TMP35]], [[TMP33]]
4947
; CHECK-NEXT: [[TMP39:%.*]] = call i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8> [[BIN_RDX]])
5048
; CHECK-NEXT: [[TMP40:%.*]] = zext i8 [[TMP39]] to i32
5149
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 256, [[N_VEC]]

0 commit comments

Comments
 (0)