Skip to content

Commit 9c2d84d

Browse files
LiqinWengLiqinWeng
committed
[LV][VPlan] Extract the implementation of transform Recipe to EVLRecipe into a small function
1 parent 86fa35c commit 9c2d84d

File tree

1 file changed

+89
-97
lines changed

1 file changed

+89
-97
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 89 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1443,13 +1443,93 @@ void VPlanTransforms::addActiveLaneMask(
14431443
HeaderMask->replaceAllUsesWith(LaneMask);
14441444
}
14451445

1446+
/// Try to convert \p CurRecipe to a corresponding EVL-based recipe. Returns
1447+
/// nullptr if no EVL-based recipe could be created.
1448+
/// \p HeaderMask Header Mask.
1449+
/// \p CurRecipe Recipe to be transform.
1450+
/// \p TypeInfo VPlan-based type analysis.
1451+
/// \p AllOneMask The vector mask parameter of vector-predication intrinsics.
1452+
/// \p EVL The explicit vector length parameter of vector-predication
1453+
/// intrinsics.
1454+
static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
1455+
VPRecipeBase &CurRecipe,
1456+
VPTypeAnalysis &TypeInfo,
1457+
VPValue &AllOneMask, VPValue &EVL) {
1458+
using namespace llvm::VPlanPatternMatch;
1459+
auto GetNewMask = [&](VPValue *OrigMask) -> VPValue * {
1460+
assert(OrigMask && "Unmasked recipe when folding tail");
1461+
return HeaderMask == OrigMask ? nullptr : OrigMask;
1462+
};
1463+
1464+
return TypeSwitch<VPRecipeBase *, VPRecipeBase *>(&CurRecipe)
1465+
.Case<VPWidenLoadRecipe>([&](VPWidenLoadRecipe *L) {
1466+
VPValue *NewMask = GetNewMask(L->getMask());
1467+
return new VPWidenLoadEVLRecipe(*L, EVL, NewMask);
1468+
})
1469+
.Case<VPWidenStoreRecipe>([&](VPWidenStoreRecipe *S) {
1470+
VPValue *NewMask = GetNewMask(S->getMask());
1471+
return new VPWidenStoreEVLRecipe(*S, EVL, NewMask);
1472+
})
1473+
.Case<VPWidenRecipe>([&](VPWidenRecipe *W) -> VPRecipeBase * {
1474+
unsigned Opcode = W->getOpcode();
1475+
if (!Instruction::isBinaryOp(Opcode) && !Instruction::isUnaryOp(Opcode))
1476+
return nullptr;
1477+
return new VPWidenEVLRecipe(*W, EVL);
1478+
})
1479+
.Case<VPReductionRecipe>([&](VPReductionRecipe *Red) {
1480+
VPValue *NewMask = GetNewMask(Red->getCondOp());
1481+
return new VPReductionEVLRecipe(*Red, EVL, NewMask);
1482+
})
1483+
.Case<VPWidenIntrinsicRecipe, VPWidenCastRecipe>(
1484+
[&](auto *CR) -> VPRecipeBase * {
1485+
Intrinsic::ID VPID;
1486+
if (auto *CallR = dyn_cast<VPWidenIntrinsicRecipe>(CR))
1487+
VPID =
1488+
VPIntrinsic::getForIntrinsic(CallR->getVectorIntrinsicID());
1489+
else if (auto *CastR = dyn_cast<VPWidenCastRecipe>(CR))
1490+
VPID = VPIntrinsic::getForOpcode(CastR->getOpcode());
1491+
assert(VPID != Intrinsic::not_intrinsic && "Expected VP intrinsic");
1492+
assert(VPIntrinsic::getMaskParamPos(VPID) &&
1493+
VPIntrinsic::getVectorLengthParamPos(VPID) &&
1494+
"Expected VP intrinsic");
1495+
1496+
SmallVector<VPValue *> Ops(CR->operands());
1497+
Ops.push_back(&AllOneMask);
1498+
Ops.push_back(&EVL);
1499+
return new VPWidenIntrinsicRecipe(
1500+
VPID, Ops, TypeInfo.inferScalarType(CR), CR->getDebugLoc());
1501+
})
1502+
.Case<VPWidenSelectRecipe>([&](VPWidenSelectRecipe *Sel) {
1503+
SmallVector<VPValue *> Ops(Sel->operands());
1504+
Ops.push_back(&EVL);
1505+
return new VPWidenIntrinsicRecipe(Intrinsic::vp_select, Ops,
1506+
TypeInfo.inferScalarType(Sel),
1507+
Sel->getDebugLoc());
1508+
})
1509+
.Case<VPInstruction>([&](VPInstruction *VPI) -> VPRecipeBase * {
1510+
VPValue *LHS, *RHS;
1511+
// Transform select with a header mask condition
1512+
// select(header_mask, LHS, RHS)
1513+
// into vector predication merge.
1514+
// vp.merge(all-true, LHS, RHS, EVL)
1515+
if (!match(VPI, m_Select(m_Specific(HeaderMask), m_VPValue(LHS),
1516+
m_VPValue(RHS))))
1517+
return nullptr;
1518+
// Use all true as the condition because this transformation is
1519+
// limited to selects whose condition is a header mask.
1520+
return new VPWidenIntrinsicRecipe(
1521+
Intrinsic::vp_merge, {&AllOneMask, LHS, RHS, &EVL},
1522+
TypeInfo.inferScalarType(LHS), VPI->getDebugLoc());
1523+
})
1524+
.Default([&](VPRecipeBase *R) { return nullptr; });
1525+
}
1526+
14461527
/// Replace recipes with their EVL variants.
14471528
static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
1448-
using namespace llvm::VPlanPatternMatch;
14491529
Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
14501530
VPTypeAnalysis TypeInfo(CanonicalIVType);
14511531
LLVMContext &Ctx = CanonicalIVType->getContext();
1452-
SmallVector<VPValue *> HeaderMasks = collectAllHeaderMasks(Plan);
1532+
VPValue *AllOneMask = Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
14531533

14541534
for (VPUser *U : Plan.getVF().users()) {
14551535
if (auto *R = dyn_cast<VPReverseVectorPointerRecipe>(U))
@@ -1461,110 +1541,22 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
14611541
for (VPValue *HeaderMask : collectAllHeaderMasks(Plan)) {
14621542
for (VPUser *U : collectUsersRecursively(HeaderMask)) {
14631543
auto *CurRecipe = cast<VPRecipeBase>(U);
1464-
auto GetNewMask = [&](VPValue *OrigMask) -> VPValue * {
1465-
assert(OrigMask && "Unmasked recipe when folding tail");
1466-
return HeaderMask == OrigMask ? nullptr : OrigMask;
1467-
};
1468-
1469-
VPRecipeBase *NewRecipe =
1470-
TypeSwitch<VPRecipeBase *, VPRecipeBase *>(CurRecipe)
1471-
.Case<VPWidenLoadRecipe>([&](VPWidenLoadRecipe *L) {
1472-
VPValue *NewMask = GetNewMask(L->getMask());
1473-
return new VPWidenLoadEVLRecipe(*L, EVL, NewMask);
1474-
})
1475-
.Case<VPWidenStoreRecipe>([&](VPWidenStoreRecipe *S) {
1476-
VPValue *NewMask = GetNewMask(S->getMask());
1477-
return new VPWidenStoreEVLRecipe(*S, EVL, NewMask);
1478-
})
1479-
.Case<VPWidenRecipe>([&](VPWidenRecipe *W) -> VPRecipeBase * {
1480-
unsigned Opcode = W->getOpcode();
1481-
if (!Instruction::isBinaryOp(Opcode) &&
1482-
!Instruction::isUnaryOp(Opcode))
1483-
return nullptr;
1484-
return new VPWidenEVLRecipe(*W, EVL);
1485-
})
1486-
.Case<VPReductionRecipe>([&](VPReductionRecipe *Red) {
1487-
VPValue *NewMask = GetNewMask(Red->getCondOp());
1488-
return new VPReductionEVLRecipe(*Red, EVL, NewMask);
1489-
})
1490-
.Case<VPWidenIntrinsicRecipe>(
1491-
[&](VPWidenIntrinsicRecipe *CallR) -> VPRecipeBase * {
1492-
Intrinsic::ID VPID = VPIntrinsic::getForIntrinsic(
1493-
CallR->getVectorIntrinsicID());
1494-
assert(VPID != Intrinsic::not_intrinsic &&
1495-
"Expected vp.casts Instrinsic");
1496-
assert(VPIntrinsic::getMaskParamPos(VPID) &&
1497-
VPIntrinsic::getVectorLengthParamPos(VPID) &&
1498-
"Expected VP intrinsic");
1499-
1500-
SmallVector<VPValue *> Ops(CallR->operands());
1501-
VPValue *Mask =
1502-
Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
1503-
Ops.push_back(Mask);
1504-
Ops.push_back(&EVL);
1505-
return new VPWidenIntrinsicRecipe(
1506-
VPID, Ops, TypeInfo.inferScalarType(CallR),
1507-
CallR->getDebugLoc());
1508-
})
1509-
.Case<VPWidenCastRecipe>(
1510-
[&](VPWidenCastRecipe *CastR) -> VPRecipeBase * {
1511-
Intrinsic::ID VPID =
1512-
VPIntrinsic::getForOpcode(CastR->getOpcode());
1513-
assert(VPID != Intrinsic::not_intrinsic &&
1514-
"Expected vp.casts Instrinsic");
1515-
1516-
SmallVector<VPValue *> Ops(CastR->operands());
1517-
assert(VPIntrinsic::getMaskParamPos(VPID) &&
1518-
VPIntrinsic::getVectorLengthParamPos(VPID) &&
1519-
"Expected VP intrinsic");
1520-
VPValue *Mask =
1521-
Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
1522-
Ops.push_back(Mask);
1523-
Ops.push_back(&EVL);
1524-
return new VPWidenIntrinsicRecipe(
1525-
VPID, Ops, TypeInfo.inferScalarType(CastR),
1526-
CastR->getDebugLoc());
1527-
})
1528-
.Case<VPWidenSelectRecipe>([&](VPWidenSelectRecipe *Sel) {
1529-
SmallVector<VPValue *> Ops(Sel->operands());
1530-
Ops.push_back(&EVL);
1531-
return new VPWidenIntrinsicRecipe(Intrinsic::vp_select, Ops,
1532-
TypeInfo.inferScalarType(Sel),
1533-
Sel->getDebugLoc());
1534-
})
1535-
.Case<VPInstruction>([&](VPInstruction *VPI) -> VPRecipeBase * {
1536-
VPValue *LHS, *RHS;
1537-
// Transform select with a header mask condition
1538-
// select(header_mask, LHS, RHS)
1539-
// into vector predication merge.
1540-
// vp.merge(all-true, LHS, RHS, EVL)
1541-
if (!match(VPI, m_Select(m_Specific(HeaderMask), m_VPValue(LHS),
1542-
m_VPValue(RHS))))
1543-
return nullptr;
1544-
// Use all true as the condition because this transformation is
1545-
// limited to selects whose condition is a header mask.
1546-
VPValue *AllTrue =
1547-
Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
1548-
return new VPWidenIntrinsicRecipe(
1549-
Intrinsic::vp_merge, {AllTrue, LHS, RHS, &EVL},
1550-
TypeInfo.inferScalarType(LHS), VPI->getDebugLoc());
1551-
})
1552-
.Default([&](VPRecipeBase *R) { return nullptr; });
1553-
1554-
if (!NewRecipe)
1544+
VPRecipeBase *EVLRecipe =
1545+
createEVLRecipe(HeaderMask, *CurRecipe, TypeInfo, *AllOneMask, EVL);
1546+
if (!EVLRecipe)
15551547
continue;
15561548

1557-
[[maybe_unused]] unsigned NumDefVal = NewRecipe->getNumDefinedValues();
1549+
[[maybe_unused]] unsigned NumDefVal = EVLRecipe->getNumDefinedValues();
15581550
assert(NumDefVal == CurRecipe->getNumDefinedValues() &&
15591551
"New recipe must define the same number of values as the "
15601552
"original.");
15611553
assert(
15621554
NumDefVal <= 1 &&
15631555
"Only supports recipes with a single definition or without users.");
1564-
NewRecipe->insertBefore(CurRecipe);
1565-
if (isa<VPSingleDefRecipe, VPWidenLoadEVLRecipe>(NewRecipe)) {
1556+
EVLRecipe->insertBefore(CurRecipe);
1557+
if (isa<VPSingleDefRecipe, VPWidenLoadEVLRecipe>(EVLRecipe)) {
15661558
VPValue *CurVPV = CurRecipe->getVPSingleValue();
1567-
CurVPV->replaceAllUsesWith(NewRecipe->getVPSingleValue());
1559+
CurVPV->replaceAllUsesWith(EVLRecipe->getVPSingleValue());
15681560
}
15691561
// Defer erasing recipes till the end so that we don't invalidate the
15701562
// VPTypeAnalysis cache.

0 commit comments

Comments
 (0)