@@ -3107,28 +3107,22 @@ void VPlanTransforms::materializeBroadcasts(VPlan &Plan) {
31073107// / that feeds a store interleave group at index \p Idx, \p WideMember0 is the
31083108// / recipe feeding the same interleave group at index 0. A VPWidenLoadRecipe can
31093109// / be narrowed to an index-independent load if it feeds all wide ops at all
3110- // / indices (checked by via the operands of the wide recipe at lane0, \p
3111- // / WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if \p V
3112- // / is defined at \p Idx of a load interleave group.
3110+ // / indices (\p OpV must be the operand at index \p OpIdx for both the recipe at
3111+ // / lane 0, \p WideMember0, and \p WideMember). A VPInterleaveRecipe can be
3112+ // / narrowed to a wide load, if \p V is defined at \p Idx of a load interleave
3113+ // / group.
31133114static bool canNarrowLoad (VPWidenRecipe *WideMember0, VPWidenRecipe *WideMember,
3114- VPValue *V , unsigned Idx) {
3115- auto *DefR = V ->getDefiningRecipe ();
3115+ unsigned OpIdx, VPValue *OpV , unsigned Idx) {
3116+ auto *DefR = OpV ->getDefiningRecipe ();
31163117 if (!DefR)
31173118 return false ;
31183119 if (auto *W = dyn_cast<VPWidenLoadRecipe>(DefR))
3119- return !W->getMask () &&
3120- all_of (zip (WideMember0->operands (), WideMember->operands ()),
3121- [V](const auto P) {
3122- // V must be as at the same places in both WideMember0 and
3123- // WideMember.
3124- const auto &[WideMember0Op, WideMemberOp] = P;
3125- return (WideMember0Op == V) == (WideMemberOp == V);
3126- });
3120+ return !W->getMask () && WideMember0->getOperand (OpIdx) == OpV;
31273121
31283122 if (auto *IR = dyn_cast<VPInterleaveRecipe>(DefR))
31293123 return IR->getInterleaveGroup ()->getFactor () ==
31303124 IR->getInterleaveGroup ()->getNumMembers () &&
3131- IR->getVPValue (Idx) == V ;
3125+ IR->getVPValue (Idx) == OpV ;
31323126 return false ;
31333127}
31343128
@@ -3243,9 +3237,11 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
32433237 if (!R || R->getOpcode () != WideMember0->getOpcode () ||
32443238 R->getNumOperands () > 2 )
32453239 return ;
3246- if (any_of (R->operands (), [WideMember0, Idx = I, R](VPValue *V) {
3247- return !canNarrowLoad (WideMember0, R, V, Idx);
3248- }))
3240+ if (any_of (enumerate(R->operands ()),
3241+ [WideMember0, Idx = I, R](const auto &P) {
3242+ const auto &[OpIdx, OpV] = P;
3243+ return !canNarrowLoad (WideMember0, R, OpIdx, OpV, Idx);
3244+ }))
32493245 return ;
32503246 }
32513247 StoreGroups.push_back (InterleaveR);
0 commit comments