@@ -1442,17 +1442,18 @@ void VPlanTransforms::addActiveLaneMask(
14421442 HeaderMask->replaceAllUsesWith (LaneMask);
14431443}
14441444
1445- static VPRecipeBase *createEVLRecipe (VPValue &EVL, VPValue *HeaderMask,
1446- VPValue *AllOneMask,
1447- VPRecipeBase *CurRecipe,
1448- VPTypeAnalysis TypeInfo) {
1445+ // / Create EVLRecipe with Recipe
1446+ static VPRecipeBase *createEVLRecipe (VPValue *HeaderMask,
1447+ VPRecipeBase &CurRecipe,
1448+ VPTypeAnalysis &TypeInfo,
1449+ VPValue &AllOneMask, VPValue &EVL) {
14491450 using namespace llvm ::VPlanPatternMatch;
14501451 auto GetNewMask = [&](VPValue *OrigMask) -> VPValue * {
14511452 assert (OrigMask && " Unmasked recipe when folding tail" );
14521453 return HeaderMask == OrigMask ? nullptr : OrigMask;
14531454 };
14541455
1455- return TypeSwitch<VPRecipeBase *, VPRecipeBase *>(CurRecipe)
1456+ return TypeSwitch<VPRecipeBase *, VPRecipeBase *>(& CurRecipe)
14561457 .Case <VPWidenLoadRecipe>([&](VPWidenLoadRecipe *L) {
14571458 VPValue *NewMask = GetNewMask (L->getMask ());
14581459 return new VPWidenLoadEVLRecipe (*L, EVL, NewMask);
@@ -1473,18 +1474,16 @@ static VPRecipeBase *createEVLRecipe(VPValue &EVL, VPValue *HeaderMask,
14731474 })
14741475 .Case <VPWidenIntrinsicRecipe>(
14751476 [&](VPWidenIntrinsicRecipe *CInst) -> VPRecipeBase * {
1476- auto *CI = cast <CallInst>(CInst->getUnderlyingInstr ());
1477+ auto *CI = dyn_cast <CallInst>(CInst->getUnderlyingInstr ());
14771478 Intrinsic::ID VPID = VPIntrinsic::getForIntrinsic (
14781479 CI->getCalledFunction ()->getIntrinsicID ());
14791480 assert (VPID != Intrinsic::not_intrinsic &&
1480- " Expected VP Instrinsic" );
1481-
1482- SmallVector<VPValue *> Ops (CInst->operands ());
1483- assert (VPIntrinsic::getMaskParamPos (VPID) &&
1481+ VPIntrinsic::getMaskParamPos (VPID) &&
14841482 VPIntrinsic::getVectorLengthParamPos (VPID) &&
14851483 " Expected VP intrinsic" );
14861484
1487- Ops.push_back (AllOneMask);
1485+ SmallVector<VPValue *> Ops (CInst->operands ());
1486+ Ops.push_back (&AllOneMask);
14881487 Ops.push_back (&EVL);
14891488 return new VPWidenIntrinsicRecipe (*CI, VPID, Ops,
14901489 TypeInfo.inferScalarType (CInst),
@@ -1494,13 +1493,12 @@ static VPRecipeBase *createEVLRecipe(VPValue &EVL, VPValue *HeaderMask,
14941493 auto *CI = dyn_cast<CastInst>(CInst->getUnderlyingInstr ());
14951494 Intrinsic::ID VPID = VPIntrinsic::getForOpcode (CI->getOpcode ());
14961495 assert (VPID != Intrinsic::not_intrinsic &&
1497- " Expected vp.casts Instrinsic" );
1496+ VPIntrinsic::getMaskParamPos (VPID) &&
1497+ VPIntrinsic::getVectorLengthParamPos (VPID) &&
1498+ " Expected vp.cast intrinsic" );
14981499
14991500 SmallVector<VPValue *> Ops (CInst->operands ());
1500- assert (VPIntrinsic::getMaskParamPos (VPID) &&
1501- VPIntrinsic::getVectorLengthParamPos (VPID) &&
1502- " Expected VP intrinsic" );
1503- Ops.push_back (AllOneMask);
1501+ Ops.push_back (&AllOneMask);
15041502 Ops.push_back (&EVL);
15051503 return new VPWidenIntrinsicRecipe (
15061504 VPID, Ops, TypeInfo.inferScalarType (CInst), CInst->getDebugLoc ());
@@ -1524,7 +1522,7 @@ static VPRecipeBase *createEVLRecipe(VPValue &EVL, VPValue *HeaderMask,
15241522 // Use all true as the condition because this transformation is
15251523 // limited to selects whose condition is a header mask.
15261524 return new VPWidenIntrinsicRecipe (
1527- Intrinsic::vp_merge, {AllOneMask, LHS, RHS, &EVL},
1525+ Intrinsic::vp_merge, {& AllOneMask, LHS, RHS, &EVL},
15281526 TypeInfo.inferScalarType (LHS), VPI->getDebugLoc ());
15291527 })
15301528 .Default ([&](VPRecipeBase *R) { return nullptr ; });
@@ -1548,7 +1546,7 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
15481546 for (VPUser *U : collectUsersRecursively (HeaderMask)) {
15491547 auto *CurRecipe = cast<VPRecipeBase>(U);
15501548 VPRecipeBase *EVLRecipe =
1551- createEVLRecipe (EVL, HeaderMask, AllOneMask, CurRecipe, TypeInfo );
1549+ createEVLRecipe (HeaderMask, *CurRecipe, TypeInfo, *AllOneMask, EVL );
15521550 if (!EVLRecipe)
15531551 continue ;
15541552
0 commit comments