@@ -4268,6 +4268,32 @@ static bool canNarrowLoad(VPWidenRecipe *WideMember0, unsigned OpIdx,
42684268 return false ;
42694269}
42704270
4271+ static bool canNarrowOps (ArrayRef<VPValue *> Ops) {
4272+ SmallVector<VPValue *> Ops0;
4273+ auto *WideMember0 = dyn_cast<VPWidenRecipe>(Ops[0 ]);
4274+ if (!WideMember0)
4275+ return false ;
4276+
4277+ for (unsigned Idx = 0 ; Idx != WideMember0->getNumOperands (); ++Idx) {
4278+ SmallVector<VPValue *> Ops0;
4279+ for (const auto &[I, V] : enumerate(Ops)) {
4280+ auto *R = dyn_cast_or_null<VPWidenRecipe>(V);
4281+ if (!R || R->getOpcode () != WideMember0->getOpcode () ||
4282+ R->getNumOperands () > 2 )
4283+ return false ;
4284+
4285+ Ops0.push_back (R->getOperand (Idx));
4286+ }
4287+ if (any_of (enumerate(Ops0), [WideMember0, Idx](const auto &P) {
4288+ const auto &[OpIdx, OpV] = P;
4289+ return !canNarrowLoad (WideMember0, Idx, OpV, OpIdx);
4290+ }))
4291+ return false ;
4292+ }
4293+
4294+ return true ;
4295+ }
4296+
42714297// / Returns true if \p IR is a full interleave group with factor and number of
42724298// / members both equal to \p VF. The interleave group must also access the full
42734299// / vector width \p VectorRegWidth.
@@ -4441,22 +4467,8 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
44414467
44424468 // Check if all values feeding InterleaveR are matching wide recipes, which
44434469 // operands that can be narrowed.
4444- auto *WideMember0 =
4445- dyn_cast_or_null<VPWidenRecipe>(InterleaveR->getStoredValues ()[0 ]);
4446- if (!WideMember0)
4470+ if (!canNarrowOps (InterleaveR->getStoredValues ()))
44474471 return ;
4448- for (const auto &[I, V] : enumerate(InterleaveR->getStoredValues ())) {
4449- auto *R = dyn_cast_or_null<VPWidenRecipe>(V);
4450- if (!R || R->getOpcode () != WideMember0->getOpcode () ||
4451- R->getNumOperands () > 2 )
4452- return ;
4453- if (any_of (enumerate(R->operands ()),
4454- [WideMember0, Idx = I](const auto &P) {
4455- const auto &[OpIdx, OpV] = P;
4456- return !canNarrowLoad (WideMember0, OpIdx, OpV, Idx);
4457- }))
4458- return ;
4459- }
44604472 StoreGroups.push_back (InterleaveR);
44614473 }
44624474
0 commit comments