Skip to content

Commit 4e5650a

Browse files
fhahnfhahn
committed
[VPlan] Truncate/Extend ComputeReductionResult at construction (NFC).
Instead of looking up the narrower reduction type via getRecurrenceType we can generate the needed extend directly at constructiond re-use the truncated value from the loop.
1 parent 0175005 commit 4e5650a

File tree

6 files changed

+40
-54
lines changed

6 files changed

+40
-54
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 32 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -7278,6 +7278,13 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
72787278
// created a bc.merge.rdx Phi after the main vector body. Ensure that we carry
72797279
// over the incoming values correctly.
72807280
using namespace VPlanPatternMatch;
7281+
if (EpiRedResult->getNumUsers() == 1 &&
7282+
isa<VPInstructionWithType>(*EpiRedResult->user_begin())) {
7283+
EpiRedResult = cast<VPInstructionWithType>(*EpiRedResult->user_begin());
7284+
assert((EpiRedResult->getOpcode() == Instruction::SExt ||
7285+
EpiRedResult->getOpcode() == Instruction::ZExt) &&
7286+
"can only have SExt/ZExt users");
7287+
}
72817288
assert(count_if(EpiRedResult->users(), IsaPred<VPPhi>) == 1 &&
72827289
"ResumePhi must have a single user");
72837290
auto *EpiResumePhiVPI =
@@ -9212,28 +9219,6 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
92129219
PhiR->setOperand(1, NewExitingVPV);
92139220
}
92149221

9215-
// If the vector reduction can be performed in a smaller type, we truncate
9216-
// then extend the loop exit value to enable InstCombine to evaluate the
9217-
// entire expression in the smaller type.
9218-
if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
9219-
!RecurrenceDescriptor::isAnyOfRecurrenceKind(
9220-
RdxDesc.getRecurrenceKind())) {
9221-
assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
9222-
Type *RdxTy = RdxDesc.getRecurrenceType();
9223-
auto *Trunc =
9224-
new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
9225-
auto *Extnd =
9226-
RdxDesc.isSigned()
9227-
? new VPWidenCastRecipe(Instruction::SExt, Trunc, PhiTy)
9228-
: new VPWidenCastRecipe(Instruction::ZExt, Trunc, PhiTy);
9229-
9230-
Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
9231-
Extnd->insertAfter(Trunc);
9232-
if (PhiR->getOperand(1) == NewExitingVPV)
9233-
PhiR->setOperand(1, Extnd->getVPSingleValue());
9234-
NewExitingVPV = Extnd;
9235-
}
9236-
92379222
// We want code in the middle block to appear to execute on the location of
92389223
// the scalar loop's latch terminator because: (a) it is all compiler
92399224
// generated, (b) these instructions are always executed after evaluating
@@ -9273,6 +9258,31 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
92739258
Builder.createNaryOp(VPInstruction::ComputeReductionResult,
92749259
{PhiR, NewExitingVPV}, Flags, ExitDL);
92759260
}
9261+
// If the vector reduction can be performed in a smaller type, we truncate
9262+
// then extend the loop exit value to enable InstCombine to evaluate the
9263+
// entire expression in the smaller type.
9264+
if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
9265+
!RecurrenceDescriptor::isAnyOfRecurrenceKind(
9266+
RdxDesc.getRecurrenceKind())) {
9267+
assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
9268+
Type *RdxTy = RdxDesc.getRecurrenceType();
9269+
auto *Trunc =
9270+
new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
9271+
Instruction::CastOps ExtendOpc =
9272+
RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
9273+
auto *Extnd = new VPWidenCastRecipe(ExtendOpc, Trunc, PhiTy);
9274+
Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
9275+
Extnd->insertAfter(Trunc);
9276+
if (PhiR->getOperand(1) == NewExitingVPV)
9277+
PhiR->setOperand(1, Extnd->getVPSingleValue());
9278+
9279+
// Update ComputeReductionResult with the truncated exiting value and
9280+
// extend its result.
9281+
FinalReductionResult->setOperand(1, Trunc);
9282+
FinalReductionResult =
9283+
Builder.createScalarCast(ExtendOpc, FinalReductionResult, PhiTy, {});
9284+
}
9285+
92769286
// Update all users outside the vector region.
92779287
OrigExitingVPV->replaceUsesWithIf(
92789288
FinalReductionResult, [FinalReductionResult](VPUser &User, unsigned) {

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp‎

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -668,7 +668,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
668668
assert(!RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK) &&
669669
"should be handled by ComputeFindLastIVResult");
670670

671-
Type *ResultTy = State.TypeAnalysis.inferScalarType(this);
672671
// The recipe's operands are the reduction phi, followed by one operand for
673672
// each part of the reduction.
674673
unsigned UF = getNumOperands() - 1;
@@ -680,15 +679,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
680679
if (hasFastMathFlags())
681680
Builder.setFastMathFlags(getFastMathFlags());
682681

683-
// If the vector reduction can be performed in a smaller type, we truncate
684-
// then extend the loop exit value to enable InstCombine to evaluate the
685-
// entire expression in the smaller type.
686-
// TODO: Handle this in truncateToMinBW.
687-
if (State.VF.isVector() && ResultTy != RdxDesc.getRecurrenceType()) {
688-
Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), State.VF);
689-
for (unsigned Part = 0; Part < UF; ++Part)
690-
RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
691-
}
692682
// Reduce all of the unrolled parts into a single vector.
693683
Value *ReducedPartRdx = RdxParts[0];
694684
if (PhiR->isOrdered()) {
@@ -713,13 +703,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
713703
// All ops in the reduction inherit fast-math-flags from the recurrence
714704
// descriptor.
715705
ReducedPartRdx = createSimpleReduction(Builder, ReducedPartRdx, RK);
716-
717-
// If the reduction can be performed in a smaller type, we need to extend
718-
// the reduction to the wider type before we branch to the original loop.
719-
if (ResultTy != RdxDesc.getRecurrenceType())
720-
ReducedPartRdx = RdxDesc.isSigned()
721-
? Builder.CreateSExt(ReducedPartRdx, ResultTy)
722-
: Builder.CreateZExt(ReducedPartRdx, ResultTy);
723706
}
724707

725708
return ReducedPartRdx;
@@ -1070,6 +1053,7 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
10701053
void VPInstructionWithType::execute(VPTransformState &State) {
10711054
State.setDebugLocFrom(getDebugLoc());
10721055
switch (getOpcode()) {
1056+
case Instruction::SExt:
10731057
case Instruction::ZExt:
10741058
case Instruction::Trunc: {
10751059
Value *Op = State.get(getOperand(0), VPLane(0));

‎llvm/test/Transforms/LoopVectorize/X86/cost-model.ll‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1167,8 +1167,7 @@ define i32 @narrowed_reduction(ptr %a, i1 %cmp) #0 {
11671167
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
11681168
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
11691169
; CHECK: middle.block:
1170-
; CHECK-NEXT: [[TMP10:%.*]] = trunc <16 x i32> [[TMP7]] to <16 x i1>
1171-
; CHECK-NEXT: [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP10]])
1170+
; CHECK-NEXT: [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]])
11721171
; CHECK-NEXT: [[TMP21:%.*]] = zext i1 [[TMP20]] to i32
11731172
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[VEC_EPILOG_PH]]
11741173
; CHECK: scalar.ph:

‎llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -208,8 +208,7 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
208208
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
209209
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
210210
; CHECK: middle.block:
211-
; CHECK-NEXT: [[TMP9:%.*]] = trunc <4 x i32> [[TMP7]] to <4 x i16>
212-
; CHECK-NEXT: [[TMP10:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP9]])
211+
; CHECK-NEXT: [[TMP10:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP6]])
213212
; CHECK-NEXT: [[TMP11:%.*]] = zext i16 [[TMP10]] to i32
214213
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
215214
; CHECK: vec.epilog.iter.check:
@@ -234,8 +233,7 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
234233
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT4]], 256
235234
; CHECK-NEXT: br i1 [[TMP21]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
236235
; CHECK: vec.epilog.middle.block:
237-
; CHECK-NEXT: [[TMP22:%.*]] = trunc <4 x i32> [[TMP20]] to <4 x i16>
238-
; CHECK-NEXT: [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP22]])
236+
; CHECK-NEXT: [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP19]])
239237
; CHECK-NEXT: [[TMP24:%.*]] = zext i16 [[TMP23]] to i32
240238
; CHECK-NEXT: br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
241239
; CHECK: vec.epilog.scalar.ph:

‎llvm/test/Transforms/LoopVectorize/reduction-small-size.ll‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,7 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n) {
2525
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
2626
; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
2727
; CHECK: middle.block:
28-
; CHECK-NEXT: [[TMP6:%.*]] = trunc <4 x i32> [[TMP4]] to <4 x i8>
29-
; CHECK-NEXT: [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP6]])
28+
; CHECK-NEXT: [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP3]])
3029
; CHECK-NEXT: [[TMP8:%.*]] = zext i8 [[TMP7]] to i32
3130
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
3231
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -104,8 +103,7 @@ define i8 @PR34687_no_undef(i1 %c, i32 %x, i32 %n) {
104103
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
105104
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
106105
; CHECK: middle.block:
107-
; CHECK-NEXT: [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i8>
108-
; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP8]])
106+
; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP5]])
109107
; CHECK-NEXT: [[TMP10:%.*]] = zext i8 [[TMP9]] to i32
110108
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
111109
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -183,8 +181,7 @@ define i32 @PR35734(i32 %x, i32 %y) {
183181
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
184182
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
185183
; CHECK: middle.block:
186-
; CHECK-NEXT: [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i1>
187-
; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> [[TMP8]])
184+
; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> [[TMP5]])
188185
; CHECK-NEXT: [[TMP10:%.*]] = sext i1 [[TMP9]] to i32
189186
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
190187
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]

‎llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,7 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
4343
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
4444
; CHECK-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4545
; CHECK: middle.block:
46-
; CHECK-NEXT: [[TMP37:%.*]] = trunc <vscale x 8 x i32> [[TMP34]] to <vscale x 8 x i8>
47-
; CHECK-NEXT: [[TMP38:%.*]] = trunc <vscale x 8 x i32> [[TMP36]] to <vscale x 8 x i8>
48-
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 8 x i8> [[TMP38]], [[TMP37]]
46+
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 8 x i8> [[TMP35]], [[TMP33]]
4947
; CHECK-NEXT: [[TMP39:%.*]] = call i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8> [[BIN_RDX]])
5048
; CHECK-NEXT: [[TMP40:%.*]] = zext i8 [[TMP39]] to i32
5149
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 256, [[N_VEC]]

0 commit comments

Comments
 (0)