diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 3e85e6ff514ef..39909a46b3a33 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -189,19 +189,7 @@ static bool sinkScalarOperands(VPlan &Plan) { if (NeedsDuplicating) { if (ScalarVFOnly) continue; - VPSingleDefRecipe *Clone; - if (auto *SinkCandidateRepR = - dyn_cast(SinkCandidate)) { - // TODO: Handle converting to uniform recipes as separate transform, - // then cloning should be sufficient here. - Instruction *I = SinkCandidate->getUnderlyingInstr(); - Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true, - nullptr /*Mask*/, *SinkCandidateRepR); - // TODO: add ".cloned" suffix to name of Clone's VPValue. - } else { - Clone = SinkCandidate->clone(); - } - + VPSingleDefRecipe *Clone = vputils::getSingleScalarClone(SinkCandidate); Clone->insertBefore(SinkCandidate); SinkCandidate->replaceUsesWithIf(Clone, [SinkTo](VPUser &U, unsigned) { return cast(&U)->getParent() != SinkTo; @@ -667,8 +655,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) { if (!vputils::isSingleScalar(Def) && !vputils::onlyFirstLaneUsed(Def)) continue; - auto *Clone = new VPReplicateRecipe(Def->getUnderlyingInstr(), - Def->operands(), /*IsUniform*/ true); + VPSingleDefRecipe *Clone = vputils::getSingleScalarClone(Def); Clone->insertAfter(Def); Def->replaceAllUsesWith(Clone); } @@ -1335,9 +1322,8 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) { auto *RepOrWidenR = cast(&R); if (RepR && isa(RepR->getUnderlyingInstr()) && vputils::isSingleScalar(RepR->getOperand(1))) { - auto *Clone = new VPReplicateRecipe( - RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(), - true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Metadata*/); + auto *Clone = + cast(vputils::getSingleScalarClone(RepOrWidenR)); Clone->insertBefore(RepOrWidenR); unsigned ExtractOpc = vputils::isUniformAcrossVFsAndUFs(RepR->getOperand(1)) @@ -1362,9 +1348,7 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) { })) continue; - auto *Clone = new VPReplicateRecipe(RepOrWidenR->getUnderlyingInstr(), - RepOrWidenR->operands(), - true /*IsSingleScalar*/); + VPSingleDefRecipe *Clone = vputils::getSingleScalarClone(RepOrWidenR); Clone->insertBefore(RepOrWidenR); RepOrWidenR->replaceAllUsesWith(Clone); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp index 06c3d75e5708c..0bc8eb3e0e9ea 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp @@ -257,6 +257,28 @@ vputils::getRecipesForUncountableExit(VPlan &Plan, return UncountableCondition; } +VPSingleDefRecipe *vputils::getSingleScalarClone(VPSingleDefRecipe *R) { + // TODO: add ".cloned" suffix to name of Clone's VPValue. + return TypeSwitch(R) + .Case([](auto *I) { + return new VPReplicateRecipe(I->getUnderlyingInstr(), I->operands(), + /*IsSingleScalar*/ true, + /*Mask*/ nullptr, + /*Metadata*/ *I); + }) + .Case([](auto *I) { + // WidenGEP does not have metadata. + return new VPReplicateRecipe(I->getUnderlyingInstr(), I->operands(), + /*IsSingleScalar*/ true, /*Mask*/ nullptr); + }) + .Case([](auto *I) { return I->clone(); }) + .Default([](auto *I) { + llvm_unreachable("Don't know how to convert to single-scalar"); + return nullptr; + }); +} + bool VPBlockUtils::isHeader(const VPBlockBase *VPB, const VPDominatorTree &VPDT) { auto *VPBB = dyn_cast(VPB); diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h index 840a5b9e12a01..1b6ea3951c534 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h +++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h @@ -122,6 +122,10 @@ std::optional getRecipesForUncountableExit(VPlan &Plan, SmallVectorImpl &Recipes, SmallVectorImpl &GEPs); + +/// Returns a single-scalar version of \p R, creating a fresh single-scalar +/// VPReplicateRecipe or just cloning the recipe. +VPSingleDefRecipe *getSingleScalarClone(VPSingleDefRecipe *R); } // namespace vputils //===----------------------------------------------------------------------===//