Skip to content

Commit 76eb1d8

Browse files
fhahnfhahn
committed
[VPlan] Truncate/Extend ComputeReductionResult at construction (NFC).
Instead of looking up the narrower reduction type via getRecurrenceType we can generate the needed extend directly at constructiond re-use the truncated value from the loop.
1 parent 4a6d31f commit 76eb1d8

File tree

6 files changed

+37
-55
lines changed

6 files changed

+37
-55
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -7226,7 +7226,10 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
72267226
// Get the VPInstruction computing the reduction result in the middle block.
72277227
// The first operand may not be from the middle block if it is not connected
72287228
// to the scalar preheader. In that case, there's nothing to fix.
7229-
auto *EpiRedResult = dyn_cast<VPInstruction>(EpiResumePhiR->getOperand(0));
7229+
VPValue *Incoming = EpiResumePhiR->getOperand(0);
7230+
match(Incoming, VPlanPatternMatch::m_ZExtOrSExt(
7231+
VPlanPatternMatch::m_VPValue(Incoming)));
7232+
auto *EpiRedResult = dyn_cast<VPInstruction>(Incoming);
72307233
if (!EpiRedResult ||
72317234
(EpiRedResult->getOpcode() != VPInstruction::ComputeAnyOfResult &&
72327235
EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult &&
@@ -9201,28 +9204,6 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
92019204
PhiR->setOperand(1, NewExitingVPV);
92029205
}
92039206

9204-
// If the vector reduction can be performed in a smaller type, we truncate
9205-
// then extend the loop exit value to enable InstCombine to evaluate the
9206-
// entire expression in the smaller type.
9207-
if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
9208-
!RecurrenceDescriptor::isAnyOfRecurrenceKind(
9209-
RdxDesc.getRecurrenceKind())) {
9210-
assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
9211-
Type *RdxTy = RdxDesc.getRecurrenceType();
9212-
auto *Trunc =
9213-
new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
9214-
auto *Extnd =
9215-
RdxDesc.isSigned()
9216-
? new VPWidenCastRecipe(Instruction::SExt, Trunc, PhiTy)
9217-
: new VPWidenCastRecipe(Instruction::ZExt, Trunc, PhiTy);
9218-
9219-
Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
9220-
Extnd->insertAfter(Trunc);
9221-
if (PhiR->getOperand(1) == NewExitingVPV)
9222-
PhiR->setOperand(1, Extnd->getVPSingleValue());
9223-
NewExitingVPV = Extnd;
9224-
}
9225-
92269207
// We want code in the middle block to appear to execute on the location of
92279208
// the scalar loop's latch terminator because: (a) it is all compiler
92289209
// generated, (b) these instructions are always executed after evaluating
@@ -9260,6 +9241,31 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
92609241
Builder.createNaryOp(VPInstruction::ComputeReductionResult,
92619242
{PhiR, NewExitingVPV}, Flags, ExitDL);
92629243
}
9244+
// If the vector reduction can be performed in a smaller type, we truncate
9245+
// then extend the loop exit value to enable InstCombine to evaluate the
9246+
// entire expression in the smaller type.
9247+
if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
9248+
!RecurrenceDescriptor::isAnyOfRecurrenceKind(
9249+
RdxDesc.getRecurrenceKind())) {
9250+
assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
9251+
Type *RdxTy = RdxDesc.getRecurrenceType();
9252+
auto *Trunc =
9253+
new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
9254+
Instruction::CastOps ExtendOpc =
9255+
RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
9256+
auto *Extnd = new VPWidenCastRecipe(ExtendOpc, Trunc, PhiTy);
9257+
Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
9258+
Extnd->insertAfter(Trunc);
9259+
if (PhiR->getOperand(1) == NewExitingVPV)
9260+
PhiR->setOperand(1, Extnd->getVPSingleValue());
9261+
9262+
// Update ComputeReductionResult with the truncated exiting value and
9263+
// extend its result.
9264+
FinalReductionResult->setOperand(1, Trunc);
9265+
FinalReductionResult =
9266+
Builder.createScalarCast(ExtendOpc, FinalReductionResult, PhiTy, {});
9267+
}
9268+
92639269
// Update all users outside the vector region.
92649270
OrigExitingVPV->replaceUsesWithIf(
92659271
FinalReductionResult, [FinalReductionResult](VPUser &User, unsigned) {

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp‎

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -668,7 +668,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
668668
assert(!RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK) &&
669669
"should be handled by ComputeFindLastIVResult");
670670

671-
Type *ResultTy = State.TypeAnalysis.inferScalarType(this);
672671
// The recipe's operands are the reduction phi, followed by one operand for
673672
// each part of the reduction.
674673
unsigned UF = getNumOperands() - 1;
@@ -680,15 +679,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
680679
if (hasFastMathFlags())
681680
Builder.setFastMathFlags(getFastMathFlags());
682681

683-
// If the vector reduction can be performed in a smaller type, we truncate
684-
// then extend the loop exit value to enable InstCombine to evaluate the
685-
// entire expression in the smaller type.
686-
// TODO: Handle this in truncateToMinBW.
687-
if (State.VF.isVector() && ResultTy != RdxDesc.getRecurrenceType()) {
688-
Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), State.VF);
689-
for (unsigned Part = 0; Part < UF; ++Part)
690-
RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
691-
}
692682
// Reduce all of the unrolled parts into a single vector.
693683
Value *ReducedPartRdx = RdxParts[0];
694684
if (PhiR->isOrdered()) {
@@ -713,13 +703,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
713703
// All ops in the reduction inherit fast-math-flags from the recurrence
714704
// descriptor.
715705
ReducedPartRdx = createSimpleReduction(Builder, ReducedPartRdx, RK);
716-
717-
// If the reduction can be performed in a smaller type, we need to extend
718-
// the reduction to the wider type before we branch to the original loop.
719-
if (ResultTy != RdxDesc.getRecurrenceType())
720-
ReducedPartRdx = RdxDesc.isSigned()
721-
? Builder.CreateSExt(ReducedPartRdx, ResultTy)
722-
: Builder.CreateZExt(ReducedPartRdx, ResultTy);
723706
}
724707

725708
return ReducedPartRdx;
@@ -1070,6 +1053,7 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
10701053
void VPInstructionWithType::execute(VPTransformState &State) {
10711054
State.setDebugLocFrom(getDebugLoc());
10721055
switch (getOpcode()) {
1056+
case Instruction::SExt:
10731057
case Instruction::ZExt:
10741058
case Instruction::Trunc: {
10751059
Value *Op = State.get(getOperand(0), VPLane(0));

‎llvm/test/Transforms/LoopVectorize/X86/cost-model.ll‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1167,8 +1167,7 @@ define i32 @narrowed_reduction(ptr %a, i1 %cmp) #0 {
11671167
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
11681168
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
11691169
; CHECK: middle.block:
1170-
; CHECK-NEXT: [[TMP10:%.*]] = trunc <16 x i32> [[TMP7]] to <16 x i1>
1171-
; CHECK-NEXT: [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP10]])
1170+
; CHECK-NEXT: [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]])
11721171
; CHECK-NEXT: [[TMP21:%.*]] = zext i1 [[TMP20]] to i32
11731172
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[VEC_EPILOG_PH]]
11741173
; CHECK: scalar.ph:

‎llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -208,8 +208,7 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
208208
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
209209
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
210210
; CHECK: middle.block:
211-
; CHECK-NEXT: [[TMP9:%.*]] = trunc <4 x i32> [[TMP7]] to <4 x i16>
212-
; CHECK-NEXT: [[TMP10:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP9]])
211+
; CHECK-NEXT: [[TMP10:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP6]])
213212
; CHECK-NEXT: [[TMP11:%.*]] = zext i16 [[TMP10]] to i32
214213
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
215214
; CHECK: vec.epilog.iter.check:
@@ -234,8 +233,7 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
234233
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT4]], 256
235234
; CHECK-NEXT: br i1 [[TMP21]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
236235
; CHECK: vec.epilog.middle.block:
237-
; CHECK-NEXT: [[TMP22:%.*]] = trunc <4 x i32> [[TMP20]] to <4 x i16>
238-
; CHECK-NEXT: [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP22]])
236+
; CHECK-NEXT: [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP19]])
239237
; CHECK-NEXT: [[TMP24:%.*]] = zext i16 [[TMP23]] to i32
240238
; CHECK-NEXT: br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
241239
; CHECK: vec.epilog.scalar.ph:

‎llvm/test/Transforms/LoopVectorize/reduction-small-size.ll‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,7 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n) {
2525
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
2626
; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
2727
; CHECK: middle.block:
28-
; CHECK-NEXT: [[TMP6:%.*]] = trunc <4 x i32> [[TMP4]] to <4 x i8>
29-
; CHECK-NEXT: [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP6]])
28+
; CHECK-NEXT: [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP3]])
3029
; CHECK-NEXT: [[TMP8:%.*]] = zext i8 [[TMP7]] to i32
3130
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
3231
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -104,8 +103,7 @@ define i8 @PR34687_no_undef(i1 %c, i32 %x, i32 %n) {
104103
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
105104
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
106105
; CHECK: middle.block:
107-
; CHECK-NEXT: [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i8>
108-
; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP8]])
106+
; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP5]])
109107
; CHECK-NEXT: [[TMP10:%.*]] = zext i8 [[TMP9]] to i32
110108
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
111109
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -183,8 +181,7 @@ define i32 @PR35734(i32 %x, i32 %y) {
183181
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
184182
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
185183
; CHECK: middle.block:
186-
; CHECK-NEXT: [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i1>
187-
; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> [[TMP8]])
184+
; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> [[TMP5]])
188185
; CHECK-NEXT: [[TMP10:%.*]] = sext i1 [[TMP9]] to i32
189186
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
190187
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]

‎llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,7 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
4343
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
4444
; CHECK-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4545
; CHECK: middle.block:
46-
; CHECK-NEXT: [[TMP37:%.*]] = trunc <vscale x 8 x i32> [[TMP34]] to <vscale x 8 x i8>
47-
; CHECK-NEXT: [[TMP38:%.*]] = trunc <vscale x 8 x i32> [[TMP36]] to <vscale x 8 x i8>
48-
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 8 x i8> [[TMP38]], [[TMP37]]
46+
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 8 x i8> [[TMP35]], [[TMP33]]
4947
; CHECK-NEXT: [[TMP39:%.*]] = call i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8> [[BIN_RDX]])
5048
; CHECK-NEXT: [[TMP40:%.*]] = zext i8 [[TMP39]] to i32
5149
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 256, [[N_VEC]]

0 commit comments

Comments
 (0)