Skip to content

Commit e73df41

Browse files
SamTebbs33SamTebbs33
committed
[LV] Add ExtNegatedMulAccReduction expression type
This PR adds the ExtNegatedMulAccReduction expression type for VPExpressionRecipe so that extend-multiply-accumulate reductions with a negated multiply can be bundled. Stacked PRs: 1. llvm#156976 2. -> This 3. llvm#147302
1 parent f1b4a3b commit e73df41

File tree

4 files changed

+192
-17
lines changed

4 files changed

+192
-17
lines changed

‎llvm/lib/Transforms/Vectorize/VPlan.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2997,6 +2997,12 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
29972997
/// vector operands, performing a reduction.add on the result, and adding
29982998
/// the scalar result to a chain.
29992999
MulAccReduction,
3000+
/// Represent an inloop multiply-accumulate reduction, multiplying the
3001+
/// extended vector operands, negating the multiplication, performing a
3002+
/// reduction.add
3003+
/// on the result, and adding
3004+
/// the scalar result to a chain.
3005+
ExtNegatedMulAccReduction,
30003006
};
30013007

30023008
/// Type of the expression.
@@ -3020,6 +3026,11 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
30203026
VPWidenRecipe *Mul, VPReductionRecipe *Red)
30213027
: VPExpressionRecipe(ExpressionTypes::ExtMulAccReduction,
30223028
{Ext0, Ext1, Mul, Red}) {}
3029+
VPExpressionRecipe(VPWidenCastRecipe *Ext0, VPWidenCastRecipe *Ext1,
3030+
VPWidenRecipe *Mul, VPWidenRecipe *Sub,
3031+
VPReductionRecipe *Red)
3032+
: VPExpressionRecipe(ExpressionTypes::ExtNegatedMulAccReduction,
3033+
{Ext0, Ext1, Mul, Sub, Red}) {}
30233034

30243035
~VPExpressionRecipe() override {
30253036
for (auto *R : reverse(ExpressionRecipes))

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2839,12 +2839,17 @@ InstructionCost VPExpressionRecipe::computeCost(ElementCount VF,
28392839
return Ctx.TTI.getMulAccReductionCost(false, Opcode, RedTy, SrcVecTy,
28402840
Ctx.CostKind);
28412841

2842-
case ExpressionTypes::ExtMulAccReduction:
2842+
case ExpressionTypes::ExtNegatedMulAccReduction:
2843+
case ExpressionTypes::ExtMulAccReduction: {
2844+
if (ExpressionType == ExpressionTypes::ExtNegatedMulAccReduction &&
2845+
Opcode == Instruction::Add)
2846+
Opcode = Instruction::Sub;
28432847
return Ctx.TTI.getMulAccReductionCost(
28442848
cast<VPWidenCastRecipe>(ExpressionRecipes.front())->getOpcode() ==
28452849
Instruction::ZExt,
28462850
Opcode, RedTy, SrcVecTy, Ctx.CostKind);
28472851
}
2852+
}
28482853
llvm_unreachable("Unknown VPExpressionRecipe::ExpressionTypes enum");
28492854
}
28502855

@@ -2890,6 +2895,30 @@ void VPExpressionRecipe::print(raw_ostream &O, const Twine &Indent,
28902895
O << ")";
28912896
break;
28922897
}
2898+
case ExpressionTypes::ExtNegatedMulAccReduction: {
2899+
getOperand(getNumOperands() - 1)->printAsOperand(O, SlotTracker);
2900+
O << " + reduce."
2901+
<< Instruction::getOpcodeName(
2902+
RecurrenceDescriptor::getOpcode(Red->getRecurrenceKind()))
2903+
<< " (sub (0, mul";
2904+
auto *Mul = cast<VPWidenRecipe>(ExpressionRecipes[2]);
2905+
Mul->printFlags(O);
2906+
O << "(";
2907+
getOperand(0)->printAsOperand(O, SlotTracker);
2908+
auto *Ext0 = cast<VPWidenCastRecipe>(ExpressionRecipes[0]);
2909+
O << " " << Instruction::getOpcodeName(Ext0->getOpcode()) << " to "
2910+
<< *Ext0->getResultType() << "), (";
2911+
getOperand(1)->printAsOperand(O, SlotTracker);
2912+
auto *Ext1 = cast<VPWidenCastRecipe>(ExpressionRecipes[1]);
2913+
O << " " << Instruction::getOpcodeName(Ext1->getOpcode()) << " to "
2914+
<< *Ext1->getResultType() << ")";
2915+
if (Red->isConditional()) {
2916+
O << ", ";
2917+
Red->getCondOp()->printAsOperand(O, SlotTracker);
2918+
}
2919+
O << "))";
2920+
break;
2921+
}
28932922
case ExpressionTypes::MulAccReduction:
28942923
case ExpressionTypes::ExtMulAccReduction: {
28952924
getOperand(getNumOperands() - 1)->printAsOperand(O, SlotTracker);

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3543,14 +3543,22 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
35433543
};
35443544

35453545
VPValue *VecOp = Red->getVecOp();
3546+
VPValue *Mul = nullptr;
3547+
VPValue *Sub = nullptr;
35463548
VPValue *A, *B;
3549+
// Sub reductions could have a sub between the add reduction and vec op.
3550+
if (match(VecOp,
3551+
m_Binary<Instruction::Sub>(m_SpecificInt(0), m_VPValue(Mul))))
3552+
Sub = VecOp;
3553+
else
3554+
Mul = VecOp;
35473555
// Try to match reduce.add(mul(...)).
3548-
if (match(VecOp, m_Mul(m_VPValue(A), m_VPValue(B)))) {
3556+
if (match(Mul, m_Mul(m_VPValue(A), m_VPValue(B)))) {
35493557
auto *RecipeA =
35503558
dyn_cast_if_present<VPWidenCastRecipe>(A->getDefiningRecipe());
35513559
auto *RecipeB =
35523560
dyn_cast_if_present<VPWidenCastRecipe>(B->getDefiningRecipe());
3553-
auto *Mul = cast<VPWidenRecipe>(VecOp->getDefiningRecipe());
3561+
auto *MulR = cast<VPWidenRecipe>(Mul->getDefiningRecipe());
35543562

35553563
// Match reduce.add(mul(ext, ext)).
35563564
if (RecipeA && RecipeB &&
@@ -3559,29 +3567,35 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
35593567
match(RecipeB, m_ZExtOrSExt(m_VPValue())) &&
35603568
IsMulAccValidAndClampRange(RecipeA->getOpcode() ==
35613569
Instruction::CastOps::ZExt,
3562-
Mul, RecipeA, RecipeB, nullptr)) {
3563-
return new VPExpressionRecipe(RecipeA, RecipeB, Mul, Red);
3570+
MulR, RecipeA, RecipeB, nullptr)) {
3571+
if (Sub)
3572+
return new VPExpressionRecipe(
3573+
RecipeA, RecipeB, MulR,
3574+
cast<VPWidenRecipe>(Sub->getDefiningRecipe()), Red);
3575+
return new VPExpressionRecipe(RecipeA, RecipeB, MulR, Red);
35643576
}
35653577
// Match reduce.add(mul).
3566-
if (IsMulAccValidAndClampRange(true, Mul, nullptr, nullptr, nullptr))
3567-
return new VPExpressionRecipe(Mul, Red);
3578+
// TODO: Add an expression type for this variant with a negated mul
3579+
if (!Sub &&
3580+
IsMulAccValidAndClampRange(true, MulR, nullptr, nullptr, nullptr))
3581+
return new VPExpressionRecipe(MulR, Red);
35683582
}
35693583
// Match reduce.add(ext(mul(ext(A), ext(B)))).
35703584
// All extend recipes must have same opcode or A == B
35713585
// which can be transform to reduce.add(zext(mul(sext(A), sext(B)))).
3572-
if (match(VecOp, m_ZExtOrSExt(m_Mul(m_ZExtOrSExt(m_VPValue()),
3573-
m_ZExtOrSExt(m_VPValue()))))) {
3586+
if (!Sub && match(Mul, m_ZExtOrSExt(m_Mul(m_ZExtOrSExt(m_VPValue()),
3587+
m_ZExtOrSExt(m_VPValue()))))) {
35743588
auto *Ext = cast<VPWidenCastRecipe>(VecOp->getDefiningRecipe());
3575-
auto *Mul = cast<VPWidenRecipe>(Ext->getOperand(0)->getDefiningRecipe());
3589+
auto *MulR = cast<VPWidenRecipe>(Ext->getOperand(0)->getDefiningRecipe());
35763590
auto *Ext0 =
3577-
cast<VPWidenCastRecipe>(Mul->getOperand(0)->getDefiningRecipe());
3591+
cast<VPWidenCastRecipe>(MulR->getOperand(0)->getDefiningRecipe());
35783592
auto *Ext1 =
3579-
cast<VPWidenCastRecipe>(Mul->getOperand(1)->getDefiningRecipe());
3593+
cast<VPWidenCastRecipe>(MulR->getOperand(1)->getDefiningRecipe());
35803594
if ((Ext->getOpcode() == Ext0->getOpcode() || Ext0 == Ext1) &&
35813595
Ext0->getOpcode() == Ext1->getOpcode() &&
35823596
IsMulAccValidAndClampRange(Ext0->getOpcode() ==
35833597
Instruction::CastOps::ZExt,
3584-
Mul, Ext0, Ext1, Ext)) {
3598+
MulR, Ext0, Ext1, Ext)) {
35853599
auto *NewExt0 = new VPWidenCastRecipe(
35863600
Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), *Ext0,
35873601
*Ext0, Ext0->getDebugLoc());
@@ -3594,10 +3608,10 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
35943608
Ext1->getDebugLoc());
35953609
NewExt1->insertBefore(Ext1);
35963610
}
3597-
Mul->setOperand(0, NewExt0);
3598-
Mul->setOperand(1, NewExt1);
3599-
Red->setOperand(1, Mul);
3600-
return new VPExpressionRecipe(NewExt0, NewExt1, Mul, Red);
3611+
MulR->setOperand(0, NewExt0);
3612+
MulR->setOperand(1, NewExt1);
3613+
Red->setOperand(1, MulR);
3614+
return new VPExpressionRecipe(NewExt0, NewExt1, MulR, Red);
36013615
}
36023616
}
36033617
return nullptr;

‎llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -580,6 +580,127 @@ exit:
580580
ret i32 %add
581581
}
582582

583+
define i32 @print_mulacc_negated(ptr %a, ptr %b) {
584+
; CHECK-LABEL: 'print_mulacc_negated'
585+
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
586+
; CHECK-NEXT: Live-in vp<%0> = VF
587+
; CHECK-NEXT: Live-in vp<%1> = VF * UF
588+
; CHECK-NEXT: Live-in vp<%2> = vector-trip-count
589+
; CHECK-NEXT: Live-in ir<1024> = original trip-count
590+
; CHECK-EMPTY:
591+
; CHECK-NEXT: ir-bb<entry>:
592+
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
593+
; CHECK-EMPTY:
594+
; CHECK-NEXT: vector.ph:
595+
; CHECK-NEXT: EMIT vp<%3> = reduction-start-vector ir<0>, ir<0>, ir<1>
596+
; CHECK-NEXT: Successor(s): vector loop
597+
; CHECK-EMPTY:
598+
; CHECK-NEXT: <x1> vector loop: {
599+
; CHECK-NEXT: vector.body:
600+
; CHECK-NEXT: EMIT vp<%4> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
601+
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi vp<%3>, vp<%8>
602+
; CHECK-NEXT: vp<%5> = SCALAR-STEPS vp<%4>, ir<1>, vp<%0>
603+
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<%5>
604+
; CHECK-NEXT: vp<%6> = vector-pointer ir<%gep.a>
605+
; CHECK-NEXT: WIDEN ir<%load.a> = load vp<%6>
606+
; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<%5>
607+
; CHECK-NEXT: vp<%7> = vector-pointer ir<%gep.b>
608+
; CHECK-NEXT: WIDEN ir<%load.b> = load vp<%7>
609+
; CHECK-NEXT: EXPRESSION vp<%8> = ir<%accum> + reduce.add (sub (0, mul (ir<%load.b> zext to i32), (ir<%load.a> zext to i32)))
610+
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%4>, vp<%1>
611+
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<%2>
612+
; CHECK-NEXT: No successors
613+
; CHECK-NEXT: }
614+
; CHECK-NEXT: Successor(s): middle.block
615+
; CHECK-EMPTY:
616+
; CHECK-NEXT: middle.block:
617+
; CHECK-NEXT: EMIT vp<%10> = compute-reduction-result ir<%accum>, vp<%8>
618+
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<1024>, vp<%2>
619+
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
620+
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
621+
; CHECK-EMPTY:
622+
; CHECK-NEXT: ir-bb<exit>:
623+
; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %loop ] (extra operand: vp<%10> from middle.block)
624+
; CHECK-NEXT: No successors
625+
; CHECK-EMPTY:
626+
; CHECK-NEXT: scalar.ph:
627+
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%2>, middle.block ], [ ir<0>, ir-bb<entry> ]
628+
; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<%10>, middle.block ], [ ir<0>, ir-bb<entry> ]
629+
; CHECK-NEXT: Successor(s): ir-bb<loop>
630+
; CHECK-EMPTY:
631+
; CHECK-NEXT: ir-bb<loop>:
632+
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
633+
; CHECK-NEXT: IR %accum = phi i32 [ 0, %entry ], [ %add, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
634+
; CHECK-NEXT: IR %gep.a = getelementptr i8, ptr %a, i64 %iv
635+
; CHECK-NEXT: IR %load.a = load i8, ptr %gep.a, align 1
636+
; CHECK-NEXT: IR %ext.a = zext i8 %load.a to i32
637+
; CHECK-NEXT: IR %gep.b = getelementptr i8, ptr %b, i64 %iv
638+
; CHECK-NEXT: IR %load.b = load i8, ptr %gep.b, align 1
639+
; CHECK-NEXT: IR %ext.b = zext i8 %load.b to i32
640+
; CHECK-NEXT: IR %mul = mul i32 %ext.b, %ext.a
641+
; CHECK-NEXT: IR %sub = sub i32 0, %mul
642+
; CHECK-NEXT: IR %add = add i32 %accum, %sub
643+
; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
644+
; CHECK-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, 1024
645+
; CHECK-NEXT: No successors
646+
; CHECK-NEXT: }
647+
; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
648+
; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
649+
; CHECK-NEXT: Live-in ir<1024> = original trip-count
650+
; CHECK-EMPTY:
651+
; CHECK-NEXT: ir-bb<entry>:
652+
; CHECK-NEXT: Successor(s): vector.ph
653+
; CHECK-EMPTY:
654+
; CHECK-NEXT: vector.ph:
655+
; CHECK-NEXT: Successor(s): vector.body
656+
; CHECK-EMPTY:
657+
; CHECK-NEXT: vector.body:
658+
; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
659+
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi ir<0>, ir<%add>
660+
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<%index>
661+
; CHECK-NEXT: WIDEN ir<%load.a> = load ir<%gep.a>
662+
; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<%index>
663+
; CHECK-NEXT: WIDEN ir<%load.b> = load ir<%gep.b>
664+
; CHECK-NEXT: WIDEN-CAST ir<%ext.b> = zext ir<%load.b> to i32
665+
; CHECK-NEXT: WIDEN-CAST ir<%ext.a> = zext ir<%load.a> to i32
666+
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%ext.b>, ir<%ext.a>
667+
; CHECK-NEXT: WIDEN ir<%sub> = sub ir<0>, ir<%mul>
668+
; CHECK-NEXT: REDUCE ir<%add> = ir<%accum> + reduce.add (ir<%sub>)
669+
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
670+
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, ir<1024>
671+
; CHECK-NEXT: Successor(s): middle.block, vector.body
672+
; CHECK-EMPTY:
673+
; CHECK-NEXT: middle.block:
674+
; CHECK-NEXT: EMIT vp<[[RED_RESULT:%.+]]> = compute-reduction-result ir<%accum>, ir<%add>
675+
; CHECK-NEXT: Successor(s): ir-bb<exit>
676+
; CHECK-EMPTY:
677+
; CHECK-NEXT: ir-bb<exit>:
678+
; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %loop ] (extra operand: vp<[[RED_RESULT]]> from middle.block)
679+
; CHECK-NEXT: No successors
680+
; CHECK-NEXT: }
681+
entry:
682+
br label %loop
683+
684+
loop:
685+
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
686+
%accum = phi i32 [ 0, %entry ], [ %add, %loop ]
687+
%gep.a = getelementptr i8, ptr %a, i64 %iv
688+
%load.a = load i8, ptr %gep.a, align 1
689+
%ext.a = zext i8 %load.a to i32
690+
%gep.b = getelementptr i8, ptr %b, i64 %iv
691+
%load.b = load i8, ptr %gep.b, align 1
692+
%ext.b = zext i8 %load.b to i32
693+
%mul = mul i32 %ext.b, %ext.a
694+
%sub = sub i32 0, %mul
695+
%add = add i32 %accum, %sub
696+
%iv.next = add i64 %iv, 1
697+
%exitcond.not = icmp eq i64 %iv.next, 1024
698+
br i1 %exitcond.not, label %exit, label %loop
699+
700+
exit:
701+
ret i32 %add
702+
}
703+
583704
define i64 @print_mulacc_sub_extended(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) {
584705
; CHECK-LABEL: 'print_mulacc_sub_extended'
585706
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {

0 commit comments

Comments
 (0)