From ba534e4d4ddb3e83cb0f61a2a933dab571e048ba Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Thu, 31 Jul 2025 12:58:49 +0100 Subject: [PATCH 1/2] [VPlan] Consolidate logic for narrowToSingleScalars (NFCI) The logic for narrowing to single scalar recipes is in two different places: narrowToSingleScalarRecipes and legalizeAndOptimizeInductions. Consolidate them. --- .../Transforms/Vectorize/VPlanTransforms.cpp | 32 +++---------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 38024aa6897fc..1847634cc57a3 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -757,31 +757,6 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) { if (!PhiR) continue; - // Try to narrow wide and replicating recipes to uniform recipes, based on - // VPlan analysis. - // TODO: Apply to all recipes in the future, to replace legacy uniformity - // analysis. - auto Users = collectUsersRecursively(PhiR); - for (VPUser *U : reverse(Users)) { - auto *Def = dyn_cast(U); - auto *RepR = dyn_cast(U); - // Skip recipes that shouldn't be narrowed. - if (!Def || !isa(Def) || - Def->getNumUsers() == 0 || !Def->getUnderlyingValue() || - (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))) - continue; - - // Skip recipes that may have other lanes than their first used. - if (!vputils::isSingleScalar(Def) && !vputils::onlyFirstLaneUsed(Def)) - continue; - - auto *Clone = new VPReplicateRecipe(Def->getUnderlyingInstr(), - Def->operands(), /*IsUniform*/ true, - /*Mask*/ nullptr, /*Flags*/ *Def); - Clone->insertAfter(Def); - Def->replaceAllUsesWith(Clone); - } - // Replace wide pointer inductions which have only their scalars used by // PtrAdd(IndStart, ScalarIVSteps (0, Step)). if (auto *PtrIV = dyn_cast(&Phi)) { @@ -1546,8 +1521,11 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) { continue; } - // Skip recipes that aren't single scalars. - if (!vputils::isSingleScalar(RepOrWidenR)) + // Skip recipes that aren't single scalars and don't just have their first + // lane used. + if ((!vputils::isSingleScalar(RepOrWidenR) && + !vputils::onlyFirstLaneUsed(RepOrWidenR)) || + RepOrWidenR->getNumUsers() == 0) continue; // Skip recipes for which conversion to single-scalar does introduce From 4bed81bee29f878fbf77def829732914699f82f7 Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Wed, 3 Dec 2025 10:41:45 +0000 Subject: [PATCH 2/2] [VPlan] Restrict users-check to onlyFirstLaneUsed --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 1847634cc57a3..d2f9263e32213 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1523,9 +1523,9 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) { // Skip recipes that aren't single scalars and don't just have their first // lane used. - if ((!vputils::isSingleScalar(RepOrWidenR) && - !vputils::onlyFirstLaneUsed(RepOrWidenR)) || - RepOrWidenR->getNumUsers() == 0) + if (!vputils::isSingleScalar(RepOrWidenR) && + (!vputils::onlyFirstLaneUsed(RepOrWidenR) || + RepOrWidenR->getNumUsers() == 0)) continue; // Skip recipes for which conversion to single-scalar does introduce