Skip to content

Commit 8a51471

Browse files
LiqinWengLiqinWeng
authored
[LV][VPlan] Extract the implementation of transform Recipe to EVLRecipe into a small function. NFC (#119510)
1 parent b842185 commit 8a51471

File tree

1 file changed

+89
-97
lines changed

1 file changed

+89
-97
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 89 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1449,13 +1449,93 @@ void VPlanTransforms::addActiveLaneMask(
14491449
HeaderMask->replaceAllUsesWith(LaneMask);
14501450
}
14511451

1452+
/// Try to convert \p CurRecipe to a corresponding EVL-based recipe. Returns
1453+
/// nullptr if no EVL-based recipe could be created.
1454+
/// \p HeaderMask Header Mask.
1455+
/// \p CurRecipe Recipe to be transform.
1456+
/// \p TypeInfo VPlan-based type analysis.
1457+
/// \p AllOneMask The vector mask parameter of vector-predication intrinsics.
1458+
/// \p EVL The explicit vector length parameter of vector-predication
1459+
/// intrinsics.
1460+
static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
1461+
VPRecipeBase &CurRecipe,
1462+
VPTypeAnalysis &TypeInfo,
1463+
VPValue &AllOneMask, VPValue &EVL) {
1464+
using namespace llvm::VPlanPatternMatch;
1465+
auto GetNewMask = [&](VPValue *OrigMask) -> VPValue * {
1466+
assert(OrigMask && "Unmasked recipe when folding tail");
1467+
return HeaderMask == OrigMask ? nullptr : OrigMask;
1468+
};
1469+
1470+
return TypeSwitch<VPRecipeBase *, VPRecipeBase *>(&CurRecipe)
1471+
.Case<VPWidenLoadRecipe>([&](VPWidenLoadRecipe *L) {
1472+
VPValue *NewMask = GetNewMask(L->getMask());
1473+
return new VPWidenLoadEVLRecipe(*L, EVL, NewMask);
1474+
})
1475+
.Case<VPWidenStoreRecipe>([&](VPWidenStoreRecipe *S) {
1476+
VPValue *NewMask = GetNewMask(S->getMask());
1477+
return new VPWidenStoreEVLRecipe(*S, EVL, NewMask);
1478+
})
1479+
.Case<VPWidenRecipe>([&](VPWidenRecipe *W) -> VPRecipeBase * {
1480+
unsigned Opcode = W->getOpcode();
1481+
if (!Instruction::isBinaryOp(Opcode) && !Instruction::isUnaryOp(Opcode))
1482+
return nullptr;
1483+
return new VPWidenEVLRecipe(*W, EVL);
1484+
})
1485+
.Case<VPReductionRecipe>([&](VPReductionRecipe *Red) {
1486+
VPValue *NewMask = GetNewMask(Red->getCondOp());
1487+
return new VPReductionEVLRecipe(*Red, EVL, NewMask);
1488+
})
1489+
.Case<VPWidenIntrinsicRecipe, VPWidenCastRecipe>(
1490+
[&](auto *CR) -> VPRecipeBase * {
1491+
Intrinsic::ID VPID;
1492+
if (auto *CallR = dyn_cast<VPWidenIntrinsicRecipe>(CR))
1493+
VPID =
1494+
VPIntrinsic::getForIntrinsic(CallR->getVectorIntrinsicID());
1495+
else if (auto *CastR = dyn_cast<VPWidenCastRecipe>(CR))
1496+
VPID = VPIntrinsic::getForOpcode(CastR->getOpcode());
1497+
assert(VPID != Intrinsic::not_intrinsic && "Expected VP intrinsic");
1498+
assert(VPIntrinsic::getMaskParamPos(VPID) &&
1499+
VPIntrinsic::getVectorLengthParamPos(VPID) &&
1500+
"Expected VP intrinsic");
1501+
1502+
SmallVector<VPValue *> Ops(CR->operands());
1503+
Ops.push_back(&AllOneMask);
1504+
Ops.push_back(&EVL);
1505+
return new VPWidenIntrinsicRecipe(
1506+
VPID, Ops, TypeInfo.inferScalarType(CR), CR->getDebugLoc());
1507+
})
1508+
.Case<VPWidenSelectRecipe>([&](VPWidenSelectRecipe *Sel) {
1509+
SmallVector<VPValue *> Ops(Sel->operands());
1510+
Ops.push_back(&EVL);
1511+
return new VPWidenIntrinsicRecipe(Intrinsic::vp_select, Ops,
1512+
TypeInfo.inferScalarType(Sel),
1513+
Sel->getDebugLoc());
1514+
})
1515+
.Case<VPInstruction>([&](VPInstruction *VPI) -> VPRecipeBase * {
1516+
VPValue *LHS, *RHS;
1517+
// Transform select with a header mask condition
1518+
// select(header_mask, LHS, RHS)
1519+
// into vector predication merge.
1520+
// vp.merge(all-true, LHS, RHS, EVL)
1521+
if (!match(VPI, m_Select(m_Specific(HeaderMask), m_VPValue(LHS),
1522+
m_VPValue(RHS))))
1523+
return nullptr;
1524+
// Use all true as the condition because this transformation is
1525+
// limited to selects whose condition is a header mask.
1526+
return new VPWidenIntrinsicRecipe(
1527+
Intrinsic::vp_merge, {&AllOneMask, LHS, RHS, &EVL},
1528+
TypeInfo.inferScalarType(LHS), VPI->getDebugLoc());
1529+
})
1530+
.Default([&](VPRecipeBase *R) { return nullptr; });
1531+
}
1532+
14521533
/// Replace recipes with their EVL variants.
14531534
static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
1454-
using namespace llvm::VPlanPatternMatch;
14551535
Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
14561536
VPTypeAnalysis TypeInfo(CanonicalIVType);
14571537
LLVMContext &Ctx = CanonicalIVType->getContext();
1458-
SmallVector<VPValue *> HeaderMasks = collectAllHeaderMasks(Plan);
1538+
VPValue *AllOneMask = Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
14591539

14601540
for (VPUser *U : Plan.getVF().users()) {
14611541
if (auto *R = dyn_cast<VPReverseVectorPointerRecipe>(U))
@@ -1467,110 +1547,22 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
14671547
for (VPValue *HeaderMask : collectAllHeaderMasks(Plan)) {
14681548
for (VPUser *U : collectUsersRecursively(HeaderMask)) {
14691549
auto *CurRecipe = cast<VPRecipeBase>(U);
1470-
auto GetNewMask = [&](VPValue *OrigMask) -> VPValue * {
1471-
assert(OrigMask && "Unmasked recipe when folding tail");
1472-
return HeaderMask == OrigMask ? nullptr : OrigMask;
1473-
};
1474-
1475-
VPRecipeBase *NewRecipe =
1476-
TypeSwitch<VPRecipeBase *, VPRecipeBase *>(CurRecipe)
1477-
.Case<VPWidenLoadRecipe>([&](VPWidenLoadRecipe *L) {
1478-
VPValue *NewMask = GetNewMask(L->getMask());
1479-
return new VPWidenLoadEVLRecipe(*L, EVL, NewMask);
1480-
})
1481-
.Case<VPWidenStoreRecipe>([&](VPWidenStoreRecipe *S) {
1482-
VPValue *NewMask = GetNewMask(S->getMask());
1483-
return new VPWidenStoreEVLRecipe(*S, EVL, NewMask);
1484-
})
1485-
.Case<VPWidenRecipe>([&](VPWidenRecipe *W) -> VPRecipeBase * {
1486-
unsigned Opcode = W->getOpcode();
1487-
if (!Instruction::isBinaryOp(Opcode) &&
1488-
!Instruction::isUnaryOp(Opcode))
1489-
return nullptr;
1490-
return new VPWidenEVLRecipe(*W, EVL);
1491-
})
1492-
.Case<VPReductionRecipe>([&](VPReductionRecipe *Red) {
1493-
VPValue *NewMask = GetNewMask(Red->getCondOp());
1494-
return new VPReductionEVLRecipe(*Red, EVL, NewMask);
1495-
})
1496-
.Case<VPWidenIntrinsicRecipe>(
1497-
[&](VPWidenIntrinsicRecipe *CallR) -> VPRecipeBase * {
1498-
Intrinsic::ID VPID = VPIntrinsic::getForIntrinsic(
1499-
CallR->getVectorIntrinsicID());
1500-
assert(VPID != Intrinsic::not_intrinsic &&
1501-
"Expected vp.casts Instrinsic");
1502-
assert(VPIntrinsic::getMaskParamPos(VPID) &&
1503-
VPIntrinsic::getVectorLengthParamPos(VPID) &&
1504-
"Expected VP intrinsic");
1505-
1506-
SmallVector<VPValue *> Ops(CallR->operands());
1507-
VPValue *Mask =
1508-
Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
1509-
Ops.push_back(Mask);
1510-
Ops.push_back(&EVL);
1511-
return new VPWidenIntrinsicRecipe(
1512-
VPID, Ops, TypeInfo.inferScalarType(CallR),
1513-
CallR->getDebugLoc());
1514-
})
1515-
.Case<VPWidenCastRecipe>(
1516-
[&](VPWidenCastRecipe *CastR) -> VPRecipeBase * {
1517-
Intrinsic::ID VPID =
1518-
VPIntrinsic::getForOpcode(CastR->getOpcode());
1519-
assert(VPID != Intrinsic::not_intrinsic &&
1520-
"Expected vp.casts Instrinsic");
1521-
1522-
SmallVector<VPValue *> Ops(CastR->operands());
1523-
assert(VPIntrinsic::getMaskParamPos(VPID) &&
1524-
VPIntrinsic::getVectorLengthParamPos(VPID) &&
1525-
"Expected VP intrinsic");
1526-
VPValue *Mask =
1527-
Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
1528-
Ops.push_back(Mask);
1529-
Ops.push_back(&EVL);
1530-
return new VPWidenIntrinsicRecipe(
1531-
VPID, Ops, TypeInfo.inferScalarType(CastR),
1532-
CastR->getDebugLoc());
1533-
})
1534-
.Case<VPWidenSelectRecipe>([&](VPWidenSelectRecipe *Sel) {
1535-
SmallVector<VPValue *> Ops(Sel->operands());
1536-
Ops.push_back(&EVL);
1537-
return new VPWidenIntrinsicRecipe(Intrinsic::vp_select, Ops,
1538-
TypeInfo.inferScalarType(Sel),
1539-
Sel->getDebugLoc());
1540-
})
1541-
.Case<VPInstruction>([&](VPInstruction *VPI) -> VPRecipeBase * {
1542-
VPValue *LHS, *RHS;
1543-
// Transform select with a header mask condition
1544-
// select(header_mask, LHS, RHS)
1545-
// into vector predication merge.
1546-
// vp.merge(all-true, LHS, RHS, EVL)
1547-
if (!match(VPI, m_Select(m_Specific(HeaderMask), m_VPValue(LHS),
1548-
m_VPValue(RHS))))
1549-
return nullptr;
1550-
// Use all true as the condition because this transformation is
1551-
// limited to selects whose condition is a header mask.
1552-
VPValue *AllTrue =
1553-
Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
1554-
return new VPWidenIntrinsicRecipe(
1555-
Intrinsic::vp_merge, {AllTrue, LHS, RHS, &EVL},
1556-
TypeInfo.inferScalarType(LHS), VPI->getDebugLoc());
1557-
})
1558-
.Default([&](VPRecipeBase *R) { return nullptr; });
1559-
1560-
if (!NewRecipe)
1550+
VPRecipeBase *EVLRecipe =
1551+
createEVLRecipe(HeaderMask, *CurRecipe, TypeInfo, *AllOneMask, EVL);
1552+
if (!EVLRecipe)
15611553
continue;
15621554

1563-
[[maybe_unused]] unsigned NumDefVal = NewRecipe->getNumDefinedValues();
1555+
[[maybe_unused]] unsigned NumDefVal = EVLRecipe->getNumDefinedValues();
15641556
assert(NumDefVal == CurRecipe->getNumDefinedValues() &&
15651557
"New recipe must define the same number of values as the "
15661558
"original.");
15671559
assert(
15681560
NumDefVal <= 1 &&
15691561
"Only supports recipes with a single definition or without users.");
1570-
NewRecipe->insertBefore(CurRecipe);
1571-
if (isa<VPSingleDefRecipe, VPWidenLoadEVLRecipe>(NewRecipe)) {
1562+
EVLRecipe->insertBefore(CurRecipe);
1563+
if (isa<VPSingleDefRecipe, VPWidenLoadEVLRecipe>(EVLRecipe)) {
15721564
VPValue *CurVPV = CurRecipe->getVPSingleValue();
1573-
CurVPV->replaceAllUsesWith(NewRecipe->getVPSingleValue());
1565+
CurVPV->replaceAllUsesWith(EVLRecipe->getVPSingleValue());
15741566
}
15751567
// Defer erasing recipes till the end so that we don't invalidate the
15761568
// VPTypeAnalysis cache.

0 commit comments

Comments
 (0)