From 49e9003b3b53e5483ad8bde021320a0f78c31ed0 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Tue, 1 Apr 2025 19:05:46 +0100 Subject: [PATCH 1/4] [VPlan] Process simplifyRecipes as a worklist. NFCI --- .../Transforms/Vectorize/VPlanTransforms.cpp | 49 ++++++++++++------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 67a55aa67c978..a4eb6927f3f75 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -923,7 +923,7 @@ static void recursivelyDeleteDeadRecipes(VPValue *V) { } /// Try to simplify recipe \p R. -static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { +static VPValue *simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { using namespace llvm::VPlanPatternMatch; // VPScalarIVSteps can only be simplified after unrolling. VPScalarIVSteps for @@ -932,8 +932,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { if (auto *Steps = dyn_cast(&R)) { if (Steps->getParent()->getPlan()->isUnrolled() && Steps->isPart0() && vputils::onlyFirstLaneUsed(Steps)) { - Steps->replaceAllUsesWith(Steps->getOperand(0)); - return; + return Steps->getOperand(0); } } @@ -943,11 +942,11 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { Type *TruncTy = TypeInfo.inferScalarType(Trunc); Type *ATy = TypeInfo.inferScalarType(A); if (TruncTy == ATy) { - Trunc->replaceAllUsesWith(A); + return A; } else { // Don't replace a scalarizing recipe with a widened cast. if (isa(&R)) - return; + return nullptr; if (ATy->getScalarSizeInBits() < TruncTy->getScalarSizeInBits()) { unsigned ExtOpcode = match(R.getOperand(0), m_SExt(m_VPValue())) @@ -960,11 +959,11 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { VPC->setUnderlyingValue(UnderlyingExt); } VPC->insertBefore(&R); - Trunc->replaceAllUsesWith(VPC); + return VPC; } else if (ATy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits()) { auto *VPC = new VPWidenCastRecipe(Instruction::Trunc, A, TruncTy); VPC->insertBefore(&R); - Trunc->replaceAllUsesWith(VPC); + return VPC; } } #ifndef NDEBUG @@ -988,20 +987,17 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { VPValue *X, *Y; if (match(&R, m_c_BinaryOr(m_LogicalAnd(m_VPValue(X), m_VPValue(Y)), - m_LogicalAnd(m_Deferred(X), m_Not(m_Deferred(Y)))))) { - R.getVPSingleValue()->replaceAllUsesWith(X); - R.eraseFromParent(); - return; - } + m_LogicalAnd(m_Deferred(X), m_Not(m_Deferred(Y)))))) + return X; if (match(&R, m_Select(m_VPValue(), m_VPValue(X), m_Deferred(X)))) - return R.getVPSingleValue()->replaceAllUsesWith(X); + return X; if (match(&R, m_c_Mul(m_VPValue(A), m_SpecificInt(1)))) - return R.getVPSingleValue()->replaceAllUsesWith(A); + return A; if (match(&R, m_Not(m_Not(m_VPValue(A))))) - return R.getVPSingleValue()->replaceAllUsesWith(A); + return A; // Remove redundant DerviedIVs, that is 0 + A * 1 -> A and 0 + 0 * x -> 0. if ((match(&R, @@ -1010,16 +1006,31 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { m_DerivedIV(m_SpecificInt(0), m_SpecificInt(0), m_VPValue()))) && TypeInfo.inferScalarType(R.getOperand(1)) == TypeInfo.inferScalarType(R.getVPSingleValue())) - return R.getVPSingleValue()->replaceAllUsesWith(R.getOperand(1)); + return R.getOperand(1); + + return nullptr; } void VPlanTransforms::simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) { ReversePostOrderTraversal> RPOT( Plan.getEntry()); VPTypeAnalysis TypeInfo(&CanonicalIVTy); - for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(RPOT)) { - for (VPRecipeBase &R : make_early_inc_range(*VPBB)) { - simplifyRecipe(R, TypeInfo); + SetVector Worklist; + for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(RPOT)) + for (VPRecipeBase &R : make_early_inc_range(*VPBB)) + Worklist.insert(&R); + + while (!Worklist.empty()) { + VPRecipeBase *R = Worklist.pop_back_val(); + if (VPValue *Result = simplifyRecipe(*R, TypeInfo)) { + R->getVPSingleValue()->replaceAllUsesWith(Result); + R->eraseFromParent(); + if (VPRecipeBase *ResultR = Result->getDefiningRecipe()) + Worklist.insert(ResultR); + for (VPUser *U : Result->users()) + if (auto *UR = dyn_cast(U)) + if (UR != R) + Worklist.insert(UR); } } } From 7731e0a5f14d5b9349a71aef12b926fc001c8c9d Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 2 Apr 2025 12:36:02 +0100 Subject: [PATCH 2/4] Remove erased type from VTypeAnalysis cache, move assertion into all simplifications --- llvm/lib/Transforms/Vectorize/VPlanAnalysis.h | 3 +++ .../Transforms/Vectorize/VPlanTransforms.cpp | 27 ++++++++++--------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h index cc21870bee2e3..ac2a8d997d2e9 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h @@ -63,6 +63,9 @@ class VPTypeAnalysis { /// Return the LLVMContext used by the analysis. LLVMContext &getContext() { return Ctx; } + + /// Remove \p V from the cache. You must call this after a value is erased. + void erase(VPValue *V) { CachedTypes.erase(V); } }; // Collect a VPlan's ephemeral recipes (those used only by an assume). diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index a4eb6927f3f75..2d769c3129167 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -966,18 +966,6 @@ static VPValue *simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { return VPC; } } -#ifndef NDEBUG - // Verify that the cached type info is for both A and its users is still - // accurate by comparing it to freshly computed types. - VPTypeAnalysis TypeInfo2( - R.getParent()->getPlan()->getCanonicalIV()->getScalarType()); - assert(TypeInfo.inferScalarType(A) == TypeInfo2.inferScalarType(A)); - for (VPUser *U : A->users()) { - auto *R = cast(U); - for (VPValue *VPV : R->definedValues()) - assert(TypeInfo.inferScalarType(VPV) == TypeInfo2.inferScalarType(VPV)); - } -#endif } // Simplify (X && Y) || (X && !Y) -> X. @@ -1024,6 +1012,7 @@ void VPlanTransforms::simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) { VPRecipeBase *R = Worklist.pop_back_val(); if (VPValue *Result = simplifyRecipe(*R, TypeInfo)) { R->getVPSingleValue()->replaceAllUsesWith(Result); + TypeInfo.erase(R->getVPSingleValue()); R->eraseFromParent(); if (VPRecipeBase *ResultR = Result->getDefiningRecipe()) Worklist.insert(ResultR); @@ -1031,6 +1020,20 @@ void VPlanTransforms::simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) { if (auto *UR = dyn_cast(U)) if (UR != R) Worklist.insert(UR); + +#ifndef NDEBUG + // Verify that the cached type info is for both Result and its users is + // still accurate by comparing it to freshly computed types. + VPTypeAnalysis TypeInfo2(&CanonicalIVTy); + assert(TypeInfo.inferScalarType(Result) == + TypeInfo2.inferScalarType(Result)); + for (VPUser *U : Result->users()) { + auto *R = cast(U); + for (VPValue *VPV : R->definedValues()) + assert(TypeInfo.inferScalarType(VPV) == + TypeInfo2.inferScalarType(VPV)); + } +#endif } } } From 7093020f0335d71e391ac7500f6a1e8d115e40da Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 2 Apr 2025 13:09:05 +0100 Subject: [PATCH 3/4] Add instructions in reverse order, so it's processed in order. This matches InstCombine --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 2d769c3129167..41644f9bad90a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1005,7 +1005,7 @@ void VPlanTransforms::simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) { VPTypeAnalysis TypeInfo(&CanonicalIVTy); SetVector Worklist; for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(RPOT)) - for (VPRecipeBase &R : make_early_inc_range(*VPBB)) + for (VPRecipeBase &R : reverse(*VPBB)) Worklist.insert(&R); while (!Worklist.empty()) { From e7a77ea3fca7226ac56475c4fd452fed9972a6d3 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 2 Apr 2025 14:21:29 +0100 Subject: [PATCH 4/4] Remove redundant UR != R check --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 41644f9bad90a..a3fd094a218d5 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1018,8 +1018,7 @@ void VPlanTransforms::simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) { Worklist.insert(ResultR); for (VPUser *U : Result->users()) if (auto *UR = dyn_cast(U)) - if (UR != R) - Worklist.insert(UR); + Worklist.insert(UR); #ifndef NDEBUG // Verify that the cached type info is for both Result and its users is