@@ -4158,6 +4158,32 @@ static bool canNarrowLoad(VPWidenRecipe *WideMember0, unsigned OpIdx,
41584158 return false ;
41594159}
41604160
4161+ static bool canNarrowOps (ArrayRef<VPValue *> Ops) {
4162+ SmallVector<VPValue *> Ops0;
4163+ auto *WideMember0 = dyn_cast<VPWidenRecipe>(Ops[0 ]);
4164+ if (!WideMember0)
4165+ return false ;
4166+
4167+ for (unsigned Idx = 0 ; Idx != WideMember0->getNumOperands (); ++Idx) {
4168+ SmallVector<VPValue *> Ops0;
4169+ for (const auto &[I, V] : enumerate(Ops)) {
4170+ auto *R = dyn_cast_or_null<VPWidenRecipe>(V->getDefiningRecipe ());
4171+ if (!R || R->getOpcode () != WideMember0->getOpcode () ||
4172+ R->getNumOperands () > 2 )
4173+ return false ;
4174+
4175+ Ops0.push_back (R->getOperand (Idx));
4176+ }
4177+ if (any_of (enumerate(Ops0), [WideMember0, Idx](const auto &P) {
4178+ const auto &[OpIdx, OpV] = P;
4179+ return !canNarrowLoad (WideMember0, Idx, OpV, OpIdx);
4180+ }))
4181+ return false ;
4182+ }
4183+
4184+ return true ;
4185+ }
4186+
41614187// / Returns true if \p IR is a full interleave group with factor and number of
41624188// / members both equal to \p VF. The interleave group must also access the full
41634189// / vector width \p VectorRegWidth.
@@ -4329,22 +4355,9 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
43294355
43304356 // Check if all values feeding InterleaveR are matching wide recipes, which
43314357 // operands that can be narrowed.
4332- auto *WideMember0 = dyn_cast_or_null<VPWidenRecipe>(
4333- InterleaveR->getStoredValues ()[0 ]->getDefiningRecipe ());
4334- if (!WideMember0)
4358+ //
4359+ if (!canNarrowOps (InterleaveR->getStoredValues ()))
43354360 return ;
4336- for (const auto &[I, V] : enumerate(InterleaveR->getStoredValues ())) {
4337- auto *R = dyn_cast_or_null<VPWidenRecipe>(V->getDefiningRecipe ());
4338- if (!R || R->getOpcode () != WideMember0->getOpcode () ||
4339- R->getNumOperands () > 2 )
4340- return ;
4341- if (any_of (enumerate(R->operands ()),
4342- [WideMember0, Idx = I](const auto &P) {
4343- const auto &[OpIdx, OpV] = P;
4344- return !canNarrowLoad (WideMember0, OpIdx, OpV, Idx);
4345- }))
4346- return ;
4347- }
43484361 StoreGroups.push_back (InterleaveR);
43494362 }
43504363
0 commit comments