@@ -4142,6 +4142,32 @@ static bool canNarrowLoad(VPWidenRecipe *WideMember0, unsigned OpIdx,
41424142 return false ;
41434143}
41444144
4145+ static bool canNarrowOps (ArrayRef<VPValue *> Ops) {
4146+ SmallVector<VPValue *> Ops0;
4147+ auto *WideMember0 = dyn_cast<VPWidenRecipe>(Ops[0 ]);
4148+ if (!WideMember0)
4149+ return false ;
4150+
4151+ for (unsigned Idx = 0 ; Idx != WideMember0->getNumOperands (); ++Idx) {
4152+ SmallVector<VPValue *> Ops0;
4153+ for (const auto &[I, V] : enumerate(Ops)) {
4154+ auto *R = dyn_cast_or_null<VPWidenRecipe>(V->getDefiningRecipe ());
4155+ if (!R || R->getOpcode () != WideMember0->getOpcode () ||
4156+ R->getNumOperands () > 2 )
4157+ return false ;
4158+
4159+ Ops0.push_back (R->getOperand (Idx));
4160+ }
4161+ if (any_of (enumerate(Ops0), [WideMember0, Idx](const auto &P) {
4162+ const auto &[OpIdx, OpV] = P;
4163+ return !canNarrowLoad (WideMember0, Idx, OpV, OpIdx);
4164+ }))
4165+ return false ;
4166+ }
4167+
4168+ return true ;
4169+ }
4170+
41454171// / Returns true if \p IR is a full interleave group with factor and number of
41464172// / members both equal to \p VF. The interleave group must also access the full
41474173// / vector width \p VectorRegWidth.
@@ -4313,22 +4339,9 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
43134339
43144340 // Check if all values feeding InterleaveR are matching wide recipes, which
43154341 // operands that can be narrowed.
4316- auto *WideMember0 = dyn_cast_or_null<VPWidenRecipe>(
4317- InterleaveR->getStoredValues ()[0 ]->getDefiningRecipe ());
4318- if (!WideMember0)
4342+ //
4343+ if (!canNarrowOps (InterleaveR->getStoredValues ()))
43194344 return ;
4320- for (const auto &[I, V] : enumerate(InterleaveR->getStoredValues ())) {
4321- auto *R = dyn_cast_or_null<VPWidenRecipe>(V->getDefiningRecipe ());
4322- if (!R || R->getOpcode () != WideMember0->getOpcode () ||
4323- R->getNumOperands () > 2 )
4324- return ;
4325- if (any_of (enumerate(R->operands ()),
4326- [WideMember0, Idx = I](const auto &P) {
4327- const auto &[OpIdx, OpV] = P;
4328- return !canNarrowLoad (WideMember0, OpIdx, OpV, Idx);
4329- }))
4330- return ;
4331- }
43324345 StoreGroups.push_back (InterleaveR);
43334346 }
43344347
0 commit comments