@@ -757,6 +757,31 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
757757 if (!PhiR)
758758 continue ;
759759
760+ // Try to narrow wide and replicating recipes to uniform recipes, based on
761+ // VPlan analysis.
762+ // TODO: Apply to all recipes in the future, to replace legacy uniformity
763+ // analysis.
764+ auto Users = collectUsersRecursively (PhiR);
765+ for (VPUser *U : reverse (Users)) {
766+ auto *Def = dyn_cast<VPRecipeWithIRFlags>(U);
767+ auto *RepR = dyn_cast<VPReplicateRecipe>(U);
768+ // Skip recipes that shouldn't be narrowed.
769+ if (!Def || !isa<VPReplicateRecipe, VPWidenRecipe>(Def) ||
770+ Def->getNumUsers () == 0 || !Def->getUnderlyingValue () ||
771+ (RepR && (RepR->isSingleScalar () || RepR->isPredicated ())))
772+ continue ;
773+
774+ // Skip recipes that may have other lanes than their first used.
775+ if (!vputils::isSingleScalar (Def) && !vputils::onlyFirstLaneUsed (Def))
776+ continue ;
777+
778+ auto *Clone = new VPReplicateRecipe (Def->getUnderlyingInstr (),
779+ Def->operands (), /* IsUniform*/ true ,
780+ /* Mask*/ nullptr , /* Flags*/ *Def);
781+ Clone->insertAfter (Def);
782+ Def->replaceAllUsesWith (Clone);
783+ }
784+
760785 // Replace wide pointer inductions which have only their scalars used by
761786 // PtrAdd(IndStart, ScalarIVSteps (0, Step)).
762787 if (auto *PtrIV = dyn_cast<VPWidenPointerInductionRecipe>(&Phi)) {
@@ -1522,11 +1547,8 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
15221547 continue ;
15231548 }
15241549
1525- // Skip recipes that aren't single scalars and don't just have their first
1526- // lane used.
1527- if (!vputils::isSingleScalar (RepOrWidenR) &&
1528- (!vputils::onlyFirstLaneUsed (RepOrWidenR) ||
1529- RepOrWidenR->getNumUsers () == 0 ))
1550+ // Skip recipes that aren't single scalars.
1551+ if (!vputils::isSingleScalar (RepOrWidenR))
15301552 continue ;
15311553
15321554 // Skip recipes for which conversion to single-scalar does introduce
0 commit comments