Skip to content

Commit fbfad8f

Browse files
ElvisWang123ElvisWang123
committed
Refactors
Using lamda function to early return when pattern matched. Leave some assertions.
1 parent e537818 commit fbfad8f

File tree

6 files changed

+113
-216
lines changed

6 files changed

+113
-216
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 63 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -7397,7 +7397,7 @@ static bool planContainsAdditionalSimplifications(VPlan &Plan,
73977397
// VPExtendedReductionRecipe contains a folded extend instruction.
73987398
if (auto *ExtendedRed = dyn_cast<VPExtendedReductionRecipe>(&R))
73997399
SeenInstrs.insert(ExtendedRed->getExtInstr());
7400-
// VPMulAccRecupe constians a mul and otional extend instructions.
7400+
// VPMulAccRecipe constians a mul and otional extend instructions.
74017401
else if (auto *MulAcc = dyn_cast<VPMulAccRecipe>(&R)) {
74027402
SeenInstrs.insert(MulAcc->getMulInstr());
74037403
if (MulAcc->isExtended()) {
@@ -9390,77 +9390,82 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
93909390
if (CM.blockNeedsPredicationForAnyReason(BB))
93919391
CondOp = RecipeBuilder.getBlockInMask(BB);
93929392

9393-
VPValue *A, *B;
9394-
VPSingleDefRecipe *RedRecipe;
9395-
// reduce.add(mul(ext, ext)) can folded into VPMulAccRecipe
9396-
if (RdxDesc.getOpcode() == Instruction::Add &&
9397-
match(VecOp, m_Mul(m_VPValue(A), m_VPValue(B)))) {
9398-
VPRecipeBase *RecipeA = A->getDefiningRecipe();
9399-
VPRecipeBase *RecipeB = B->getDefiningRecipe();
9400-
if (RecipeA && RecipeB && match(RecipeA, m_ZExtOrSExt(m_VPValue())) &&
9401-
match(RecipeB, m_ZExtOrSExt(m_VPValue())) &&
9402-
cast<VPWidenCastRecipe>(RecipeA)->getOpcode() ==
9403-
cast<VPWidenCastRecipe>(RecipeB)->getOpcode() &&
9404-
!A->hasMoreThanOneUniqueUser() && !B->hasMoreThanOneUniqueUser()) {
9405-
RedRecipe = new VPMulAccRecipe(
9406-
RdxDesc, CurrentLinkI, PreviousLink, CondOp,
9407-
CM.useOrderedReductions(RdxDesc),
9408-
cast<VPWidenRecipe>(VecOp->getDefiningRecipe()),
9409-
cast<VPWidenCastRecipe>(RecipeA),
9410-
cast<VPWidenCastRecipe>(RecipeB));
9411-
} else {
9412-
RedRecipe = new VPMulAccRecipe(
9413-
RdxDesc, CurrentLinkI, PreviousLink, CondOp,
9414-
CM.useOrderedReductions(RdxDesc),
9415-
cast<VPWidenRecipe>(VecOp->getDefiningRecipe()));
9416-
}
9417-
} else if (RdxDesc.getOpcode() == Instruction::Add &&
9418-
match(VecOp,
9419-
m_ZExtOrSExt(m_Mul(m_ZExtOrSExt(m_VPValue(A)),
9420-
m_ZExtOrSExt(m_VPValue(B)))))) {
9421-
VPWidenCastRecipe *Ext =
9422-
dyn_cast<VPWidenCastRecipe>(VecOp->getDefiningRecipe());
9423-
VPWidenRecipe *Mul =
9424-
dyn_cast<VPWidenRecipe>(Ext->getOperand(0)->getDefiningRecipe());
9425-
if (Mul && match(Mul, m_Mul(m_ZExtOrSExt(m_VPValue()),
9426-
m_ZExtOrSExt(m_VPValue())))) {
9393+
auto TryToMatchMulAcc = [&]() -> VPSingleDefRecipe * {
9394+
VPValue *A, *B;
9395+
if (RdxDesc.getOpcode() != Instruction::Add)
9396+
return nullptr;
9397+
// reduce.add(mul(ext, ext)) can folded into VPMulAccRecipe
9398+
if (match(VecOp, m_Mul(m_VPValue(A), m_VPValue(B))) &&
9399+
!VecOp->hasMoreThanOneUniqueUser()) {
9400+
VPRecipeBase *RecipeA = A->getDefiningRecipe();
9401+
VPRecipeBase *RecipeB = B->getDefiningRecipe();
9402+
if (RecipeA && RecipeB && match(RecipeA, m_ZExtOrSExt(m_VPValue())) &&
9403+
match(RecipeB, m_ZExtOrSExt(m_VPValue())) &&
9404+
cast<VPWidenCastRecipe>(RecipeA)->getOpcode() ==
9405+
cast<VPWidenCastRecipe>(RecipeB)->getOpcode() &&
9406+
!A->hasMoreThanOneUniqueUser() &&
9407+
!B->hasMoreThanOneUniqueUser()) {
9408+
return new VPMulAccRecipe(
9409+
RdxDesc, CurrentLinkI, PreviousLink, CondOp,
9410+
CM.useOrderedReductions(RdxDesc),
9411+
cast<VPWidenRecipe>(VecOp->getDefiningRecipe()),
9412+
cast<VPWidenCastRecipe>(RecipeA),
9413+
cast<VPWidenCastRecipe>(RecipeB));
9414+
} else {
9415+
// Matched reduce.add(mul(...))
9416+
return new VPMulAccRecipe(
9417+
RdxDesc, CurrentLinkI, PreviousLink, CondOp,
9418+
CM.useOrderedReductions(RdxDesc),
9419+
cast<VPWidenRecipe>(VecOp->getDefiningRecipe()));
9420+
}
9421+
// Matched reduce.add(ext(mul(ext, ext)))
9422+
// Note that 3 extend instructions must have same opcode.
9423+
} else if (match(VecOp,
9424+
m_ZExtOrSExt(m_Mul(m_ZExtOrSExt(m_VPValue()),
9425+
m_ZExtOrSExt(m_VPValue())))) &&
9426+
!VecOp->hasMoreThanOneUniqueUser()) {
9427+
VPWidenCastRecipe *Ext =
9428+
dyn_cast<VPWidenCastRecipe>(VecOp->getDefiningRecipe());
94279429
VPWidenRecipe *Mul =
9428-
cast<VPWidenRecipe>(Ext->getOperand(0)->getDefiningRecipe());
9430+
dyn_cast<VPWidenRecipe>(Ext->getOperand(0)->getDefiningRecipe());
94299431
VPWidenCastRecipe *Ext0 =
94309432
cast<VPWidenCastRecipe>(Mul->getOperand(0)->getDefiningRecipe());
94319433
VPWidenCastRecipe *Ext1 =
94329434
cast<VPWidenCastRecipe>(Mul->getOperand(1)->getDefiningRecipe());
94339435
if (Ext->getOpcode() == Ext0->getOpcode() &&
9434-
Ext0->getOpcode() == Ext1->getOpcode()) {
9435-
RedRecipe = new VPMulAccRecipe(
9436+
Ext0->getOpcode() == Ext1->getOpcode() &&
9437+
!Mul->hasMoreThanOneUniqueUser() &&
9438+
!Ext0->hasMoreThanOneUniqueUser() &&
9439+
!Ext1->hasMoreThanOneUniqueUser()) {
9440+
return new VPMulAccRecipe(
94369441
RdxDesc, CurrentLinkI, PreviousLink, CondOp,
94379442
CM.useOrderedReductions(RdxDesc),
94389443
cast<VPWidenCastRecipe>(VecOp->getDefiningRecipe()), Mul,
94399444
cast<VPWidenCastRecipe>(Ext0), cast<VPWidenCastRecipe>(Ext1));
9440-
} else
9441-
RedRecipe = new VPExtendedReductionRecipe(
9442-
RdxDesc, CurrentLinkI,
9443-
cast<CastInst>(
9444-
cast<VPWidenCastRecipe>(VecOp)->getUnderlyingInstr()),
9445-
PreviousLink, cast<VPWidenCastRecipe>(VecOp)->getOperand(0),
9446-
CondOp, CM.useOrderedReductions(RdxDesc),
9447-
cast<VPWidenCastRecipe>(VecOp)->getResultType());
9445+
}
94489446
}
9449-
}
9450-
// VPWidenCastRecipes can folded into VPReductionRecipe
9451-
else if (match(VecOp, m_ZExtOrSExt(m_VPValue(A))) &&
9452-
!VecOp->hasMoreThanOneUniqueUser()) {
9453-
RedRecipe = new VPExtendedReductionRecipe(
9454-
RdxDesc, CurrentLinkI,
9455-
cast<CastInst>(
9456-
cast<VPWidenCastRecipe>(VecOp)->getUnderlyingInstr()),
9457-
PreviousLink, A, CondOp, CM.useOrderedReductions(RdxDesc),
9458-
cast<VPWidenCastRecipe>(VecOp)->getResultType());
9459-
} else {
9447+
return nullptr;
9448+
};
9449+
auto TryToMatchExtendedReduction = [&]() -> VPSingleDefRecipe * {
9450+
VPValue *A;
9451+
if (match(VecOp, m_ZExtOrSExt(m_VPValue(A))) &&
9452+
!VecOp->hasMoreThanOneUniqueUser()) {
9453+
return new VPExtendedReductionRecipe(
9454+
RdxDesc, CurrentLinkI, PreviousLink,
9455+
cast<VPWidenCastRecipe>(VecOp), CondOp,
9456+
CM.useOrderedReductions(RdxDesc));
9457+
}
9458+
return nullptr;
9459+
};
9460+
VPSingleDefRecipe *RedRecipe;
9461+
if (auto *MulAcc = TryToMatchMulAcc())
9462+
RedRecipe = MulAcc;
9463+
else if (auto *ExtendedRed = TryToMatchExtendedReduction())
9464+
RedRecipe = ExtendedRed;
9465+
else
94609466
RedRecipe =
94619467
new VPReductionRecipe(RdxDesc, CurrentLinkI, PreviousLink, VecOp,
94629468
CondOp, CM.useOrderedReductions(RdxDesc));
9463-
}
94649469
// Append the recipe to the end of the VPBasicBlock because we need to
94659470
// ensure that it comes after all of it's inputs, including CondOp.
94669471
// Note that this transformation may leave over dead recipes (including

‎llvm/lib/Transforms/Vectorize/VPlan.h‎

Lines changed: 32 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2670,18 +2670,19 @@ class VPExtendedReductionRecipe : public VPSingleDefRecipe {
26702670
bool IsConditional = false;
26712671
/// Type after extend.
26722672
Type *ResultTy;
2673+
/// Opcode for the extend instruction.
26732674
Instruction::CastOps ExtOp;
2674-
CastInst *CastInstr;
2675+
CastInst *ExtInstr;
26752676
bool IsZExt;
26762677

26772678
protected:
26782679
VPExtendedReductionRecipe(const unsigned char SC,
26792680
const RecurrenceDescriptor &R, Instruction *RedI,
2680-
Instruction::CastOps ExtOp, CastInst *CastI,
2681+
Instruction::CastOps ExtOp, CastInst *ExtI,
26812682
ArrayRef<VPValue *> Operands, VPValue *CondOp,
26822683
bool IsOrdered, Type *ResultTy)
26832684
: VPSingleDefRecipe(SC, Operands, RedI), RdxDesc(R), IsOrdered(IsOrdered),
2684-
ResultTy(ResultTy), ExtOp(ExtOp), CastInstr(CastI) {
2685+
ResultTy(ResultTy), ExtOp(ExtOp), ExtInstr(ExtI) {
26852686
if (CondOp) {
26862687
IsConditional = true;
26872688
addOperand(CondOp);
@@ -2691,20 +2692,13 @@ class VPExtendedReductionRecipe : public VPSingleDefRecipe {
26912692

26922693
public:
26932694
VPExtendedReductionRecipe(const RecurrenceDescriptor &R, Instruction *RedI,
2694-
CastInst *CastI, VPValue *ChainOp, VPValue *VecOp,
2695-
VPValue *CondOp, bool IsOrdered, Type *ResultTy)
2696-
: VPExtendedReductionRecipe(VPDef::VPExtendedReductionSC, R, RedI,
2697-
CastI->getOpcode(), CastI,
2698-
ArrayRef<VPValue *>({ChainOp, VecOp}), CondOp,
2699-
IsOrdered, ResultTy) {}
2700-
2701-
VPExtendedReductionRecipe(VPReductionRecipe *Red, VPWidenCastRecipe *Ext)
2695+
VPValue *ChainOp, VPWidenCastRecipe *Ext,
2696+
VPValue *CondOp, bool IsOrdered)
27022697
: VPExtendedReductionRecipe(
2703-
VPDef::VPExtendedReductionSC, Red->getRecurrenceDescriptor(),
2704-
Red->getUnderlyingInstr(), Ext->getOpcode(),
2698+
VPDef::VPExtendedReductionSC, R, RedI, Ext->getOpcode(),
27052699
cast<CastInst>(Ext->getUnderlyingInstr()),
2706-
ArrayRef<VPValue *>({Red->getChainOp(), Ext->getOperand(0)}),
2707-
Red->getCondOp(), Red->isOrdered(), Ext->getResultType()) {}
2700+
ArrayRef<VPValue *>({ChainOp, Ext->getOperand(0)}), CondOp,
2701+
IsOrdered, Ext->getResultType()) {}
27082702

27092703
~VPExtendedReductionRecipe() override = default;
27102704

@@ -2721,7 +2715,6 @@ class VPExtendedReductionRecipe : public VPSingleDefRecipe {
27212715
return R && classof(R);
27222716
}
27232717

2724-
/// Generate the reduction in the loop
27252718
void execute(VPTransformState &State) override {
27262719
llvm_unreachable("VPExtendedReductionRecipe should be transform to "
27272720
"VPExtendedRecipe + VPReductionRecipe before execution.");
@@ -2753,9 +2746,12 @@ class VPExtendedReductionRecipe : public VPSingleDefRecipe {
27532746
VPValue *getCondOp() const {
27542747
return isConditional() ? getOperand(getNumOperands() - 1) : nullptr;
27552748
}
2749+
/// The Type after extended.
27562750
Type *getResultType() const { return ResultTy; };
2751+
/// The Opcode of extend instruction.
27572752
Instruction::CastOps getExtOpcode() const { return ExtOp; };
2758-
CastInst *getExtInstr() const { return CastInstr; };
2753+
/// The CastInst of the extend instruction.
2754+
CastInst *getExtInstr() const { return ExtInstr; };
27592755
};
27602756

27612757
/// A recipe to represent inloop MulAccreduction operations, performing a
@@ -2771,16 +2767,17 @@ class VPMulAccRecipe : public VPSingleDefRecipe {
27712767
bool IsConditional = false;
27722768
/// Type after extend.
27732769
Type *ResultType;
2774-
/// reduce.add(ext((mul(Ext(), Ext())))
2770+
// Note that all extend instruction must have the same opcode in MulAcc.
27752771
Instruction::CastOps ExtOp;
27762772

2773+
/// reduce.add(ext(mul(ext0(), ext1())))
27772774
Instruction *MulInstr;
27782775
CastInst *ExtInstr = nullptr;
2779-
CastInst *Ext0Instr;
2780-
CastInst *Ext1Instr;
2776+
CastInst *Ext0Instr = nullptr;
2777+
CastInst *Ext1Instr = nullptr;
27812778

2779+
/// Is this MulAcc recipe contains extend recipes?
27822780
bool IsExtended;
2783-
bool IsOuterExtended = false;
27842781

27852782
protected:
27862783
VPMulAccRecipe(const unsigned char SC, const RecurrenceDescriptor &R,
@@ -2794,6 +2791,7 @@ class VPMulAccRecipe : public VPSingleDefRecipe {
27942791
ExtInstr(cast_if_present<CastInst>(ExtInstr)),
27952792
Ext0Instr(cast<CastInst>(Ext0Instr)),
27962793
Ext1Instr(cast<CastInst>(Ext1Instr)) {
2794+
assert(MulInstr->getOpcode() == Instruction::Mul);
27972795
if (CondOp) {
27982796
IsConditional = true;
27992797
addOperand(CondOp);
@@ -2806,6 +2804,7 @@ class VPMulAccRecipe : public VPSingleDefRecipe {
28062804
ArrayRef<VPValue *> Operands, VPValue *CondOp, bool IsOrdered)
28072805
: VPSingleDefRecipe(SC, Operands, RedI), RdxDesc(R), IsOrdered(IsOrdered),
28082806
MulInstr(MulInstr) {
2807+
assert(MulInstr->getOpcode() == Instruction::Mul);
28092808
if (CondOp) {
28102809
IsConditional = true;
28112810
addOperand(CondOp);
@@ -2857,13 +2856,12 @@ class VPMulAccRecipe : public VPSingleDefRecipe {
28572856
return R && classof(R);
28582857
}
28592858

2860-
/// Generate the reduction in the loop
28612859
void execute(VPTransformState &State) override {
28622860
llvm_unreachable("VPMulAccRecipe should transform to VPWidenCastRecipe + "
28632861
"VPWidenRecipe + VPReductionRecipe before execution");
28642862
}
28652863

2866-
/// Return the cost of VPExtendedReductionRecipe.
2864+
/// Return the cost of VPMulAccRecipe.
28672865
InstructionCost computeCost(ElementCount VF,
28682866
VPCostContext &Ctx) const override;
28692867

@@ -2890,13 +2888,24 @@ class VPMulAccRecipe : public VPSingleDefRecipe {
28902888
VPValue *getCondOp() const {
28912889
return isConditional() ? getOperand(getNumOperands() - 1) : nullptr;
28922890
}
2891+
/// Return the type after inner extended, which must equal to the type of mul
2892+
/// instruction. If the ResultType != recurrenceType, than it must have a
2893+
/// extend recipe after mul recipe.
28932894
Type *getResultType() const { return ResultType; };
2895+
/// The opcode of the extend instructions.
28942896
Instruction::CastOps getExtOpcode() const { return ExtOp; };
2897+
/// The underlying instruction for VPWidenRecipe.
28952898
Instruction *getMulInstr() const { return MulInstr; };
2899+
/// The underlying Instruction for outer VPWidenCastRecipe.
28962900
CastInst *getExtInstr() const { return ExtInstr; };
2901+
/// The underlying Instruction for inner VPWidenCastRecipe.
28972902
CastInst *getExt0Instr() const { return Ext0Instr; };
2903+
/// The underlying Instruction for inner VPWidenCastRecipe.
28982904
CastInst *getExt1Instr() const { return Ext1Instr; };
2905+
/// Return if this MulAcc recipe contains extend instructions.
28992906
bool isExtended() const { return IsExtended; };
2907+
/// Return if the operands of mul instruction come from same extend.
2908+
bool isSameExtend() const { return Ext0Instr == Ext1Instr; };
29002909
};
29012910

29022911
/// VPReplicateRecipe replicates a given instruction producing multiple scalar

0 commit comments

Comments
 (0)