From 07be4a1907ba901d8b5535584c82dc1e8062219e Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Mon, 8 Sep 2025 11:26:47 +0100 Subject: [PATCH 1/4] [LV] Introduce m_OneInt and improve (0|1)-match (NFC) --- .../lib/Transforms/Vectorize/EVLIndVarSimplify.cpp | 2 +- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +- llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h | 8 ++++++++ llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 14 ++++++-------- llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 2 +- llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 5 ++--- 7 files changed, 20 insertions(+), 15 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/EVLIndVarSimplify.cpp b/llvm/lib/Transforms/Vectorize/EVLIndVarSimplify.cpp index 5dd689799b828..0718d94597fcb 100644 --- a/llvm/lib/Transforms/Vectorize/EVLIndVarSimplify.cpp +++ b/llvm/lib/Transforms/Vectorize/EVLIndVarSimplify.cpp @@ -186,7 +186,7 @@ bool EVLIndVarSimplifyImpl::run(Loop &L) { Value *TC = nullptr; auto IntrinsicMatch = m_Intrinsic( m_Value(RemTC), m_SpecificInt(VF), - /*Scalable=*/m_SpecificInt(1)); + /*Scalable=*/m_One()); for (PHINode &PN : BB->phis()) { if (&PN == IndVar) continue; diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 6c96214cea13c..4b00150b56d4b 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -9535,7 +9535,7 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) { auto ResumePhiIter = find_if(MainScalarPH->phis(), [VectorTC](VPRecipeBase &R) { return match(&R, m_VPInstruction(m_Specific(VectorTC), - m_SpecificInt(0))); + m_ZeroInt())); }); VPPhi *ResumePhi = nullptr; if (ResumePhiIter == MainScalarPH->phis().end()) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index 109156c1469c5..a859233f4721f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -149,12 +149,20 @@ struct is_zero_int { bool isValue(const APInt &C) const { return C.isZero(); } }; +struct is_one_int { + bool isValue(const APInt &C) const { return C.isOne(); } +}; + /// Match an integer 0 or a vector with all elements equal to 0. /// For vectors, this includes constants with undefined elements. inline int_pred_ty m_ZeroInt() { return int_pred_ty(); } +/// Match an integer 1 or a vector with all elements equal to 1. +/// For vectors, this includes constants with undefined elements. +inline int_pred_ty m_OneInt() { return int_pred_ty(); } + /// Matching combinators template struct match_combine_or { LTy L; diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index bf51489543098..16fc9c30021e7 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -328,7 +328,7 @@ VPPartialReductionRecipe::computeCost(ElementCount VF, // Pick out opcode, type/ext information and use sub side effects from a widen // recipe. auto HandleWiden = [&](VPWidenRecipe *Widen) { - if (match(Widen, m_Sub(m_SpecificInt(0), m_VPValue(Op)))) { + if (match(Widen, m_Sub(m_ZeroInt(), m_VPValue(Op)))) { Widen = dyn_cast(Op->getDefiningRecipe()); } Opcode = Widen->getOpcode(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 6b122e19c0169..9986f44eda018 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1134,10 +1134,10 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { return Def->replaceAllUsesWith( Builder.createLogicalAnd(X, Builder.createLogicalAnd(Y, Z))); - if (match(Def, m_c_Mul(m_VPValue(A), m_SpecificInt(1)))) + if (match(Def, m_c_Mul(m_VPValue(A), m_OneInt()))) return Def->replaceAllUsesWith(A); - if (match(Def, m_c_Mul(m_VPValue(A), m_SpecificInt(0)))) + if (match(Def, m_c_Mul(m_VPValue(A), m_ZeroInt()))) return Def->replaceAllUsesWith(R.getOperand(0) == A ? R.getOperand(1) : R.getOperand(0)); @@ -1176,16 +1176,14 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { } // Remove redundant DerviedIVs, that is 0 + A * 1 -> A and 0 + 0 * x -> 0. - if ((match(Def, - m_DerivedIV(m_SpecificInt(0), m_VPValue(A), m_SpecificInt(1))) || - match(Def, - m_DerivedIV(m_SpecificInt(0), m_SpecificInt(0), m_VPValue()))) && + if ((match(Def, m_DerivedIV(m_ZeroInt(), m_VPValue(A), m_OneInt())) || + match(Def, m_DerivedIV(m_ZeroInt(), m_ZeroInt(), m_VPValue()))) && TypeInfo.inferScalarType(Def->getOperand(1)) == TypeInfo.inferScalarType(Def)) return Def->replaceAllUsesWith(Def->getOperand(1)); - if (match(Def, m_VPInstruction( - m_VPValue(X), m_SpecificInt(1)))) { + if (match(Def, m_VPInstruction(m_VPValue(X), + m_OneInt()))) { Type *WideStepTy = TypeInfo.inferScalarType(Def); if (TypeInfo.inferScalarType(X) != WideStepTy) X = Builder.createWidenCast(Instruction::Trunc, X, WideStepTy); diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp index 7a63d20825a31..b431aee79b407 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp @@ -238,7 +238,7 @@ void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R, if (Part != 1) continue; VPValue *StartV; - if (match(VPI->getOperand(2), m_SpecificInt(1))) { + if (match(VPI->getOperand(2), m_OneInt())) { StartV = VPI->getOperand(1); } else { auto *C = VPI->clone(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp index c6c1ef3369825..0ff66462f0cb1 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp @@ -65,11 +65,10 @@ bool vputils::isHeaderMask(const VPValue *V, VPlan &Plan) { VPValue *A, *B; using namespace VPlanPatternMatch; - if (match(V, m_ActiveLaneMask(m_VPValue(A), m_VPValue(B), m_SpecificInt(1)))) + if (match(V, m_ActiveLaneMask(m_VPValue(A), m_VPValue(B), m_OneInt()))) return B == Plan.getTripCount() && (match(A, m_ScalarIVSteps(m_Specific(Plan.getCanonicalIV()), - m_SpecificInt(1), - m_Specific(&Plan.getVF()))) || + m_OneInt(), m_Specific(&Plan.getVF()))) || IsWideCanonicalIV(A)); return match(V, m_Binary(m_VPValue(A), m_VPValue(B))) && From 325ea8cc3f046a2fcb0c8f37e300c2b56a8bf7cf Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Tue, 9 Sep 2025 12:08:40 +0100 Subject: [PATCH 2/4] [LV] Preserve m_ZeroInt, address review --- llvm/lib/Transforms/Vectorize/EVLIndVarSimplify.cpp | 2 +- llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h | 6 ++++-- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 6 +++--- llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 2 +- llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 6 +++--- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/EVLIndVarSimplify.cpp b/llvm/lib/Transforms/Vectorize/EVLIndVarSimplify.cpp index 0718d94597fcb..5dd689799b828 100644 --- a/llvm/lib/Transforms/Vectorize/EVLIndVarSimplify.cpp +++ b/llvm/lib/Transforms/Vectorize/EVLIndVarSimplify.cpp @@ -186,7 +186,7 @@ bool EVLIndVarSimplifyImpl::run(Loop &L) { Value *TC = nullptr; auto IntrinsicMatch = m_Intrinsic( m_Value(RemTC), m_SpecificInt(VF), - /*Scalable=*/m_One()); + /*Scalable=*/m_SpecificInt(1)); for (PHINode &PN : BB->phis()) { if (&PN == IndVar) continue; diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index a859233f4721f..2b22c1ab6fe27 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -149,19 +149,21 @@ struct is_zero_int { bool isValue(const APInt &C) const { return C.isZero(); } }; -struct is_one_int { +struct is_one { bool isValue(const APInt &C) const { return C.isOne(); } }; /// Match an integer 0 or a vector with all elements equal to 0. /// For vectors, this includes constants with undefined elements. +/// The reason for not naming this m_Zero() is because PatternMatch::m_Zero() +/// also matches null constants. inline int_pred_ty m_ZeroInt() { return int_pred_ty(); } /// Match an integer 1 or a vector with all elements equal to 1. /// For vectors, this includes constants with undefined elements. -inline int_pred_ty m_OneInt() { return int_pred_ty(); } +inline int_pred_ty m_One() { return int_pred_ty(); } /// Matching combinators template struct match_combine_or { diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 9986f44eda018..075395e1c8eca 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1134,7 +1134,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { return Def->replaceAllUsesWith( Builder.createLogicalAnd(X, Builder.createLogicalAnd(Y, Z))); - if (match(Def, m_c_Mul(m_VPValue(A), m_OneInt()))) + if (match(Def, m_c_Mul(m_VPValue(A), m_One()))) return Def->replaceAllUsesWith(A); if (match(Def, m_c_Mul(m_VPValue(A), m_ZeroInt()))) @@ -1176,14 +1176,14 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { } // Remove redundant DerviedIVs, that is 0 + A * 1 -> A and 0 + 0 * x -> 0. - if ((match(Def, m_DerivedIV(m_ZeroInt(), m_VPValue(A), m_OneInt())) || + if ((match(Def, m_DerivedIV(m_ZeroInt(), m_VPValue(A), m_One())) || match(Def, m_DerivedIV(m_ZeroInt(), m_ZeroInt(), m_VPValue()))) && TypeInfo.inferScalarType(Def->getOperand(1)) == TypeInfo.inferScalarType(Def)) return Def->replaceAllUsesWith(Def->getOperand(1)); if (match(Def, m_VPInstruction(m_VPValue(X), - m_OneInt()))) { + m_One()))) { Type *WideStepTy = TypeInfo.inferScalarType(Def); if (TypeInfo.inferScalarType(X) != WideStepTy) X = Builder.createWidenCast(Instruction::Trunc, X, WideStepTy); diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp index b431aee79b407..233539ae88c2e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp @@ -238,7 +238,7 @@ void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R, if (Part != 1) continue; VPValue *StartV; - if (match(VPI->getOperand(2), m_OneInt())) { + if (match(VPI->getOperand(2), m_One())) { StartV = VPI->getOperand(1); } else { auto *C = VPI->clone(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp index 0ff66462f0cb1..ddc4ad1977401 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp @@ -65,10 +65,10 @@ bool vputils::isHeaderMask(const VPValue *V, VPlan &Plan) { VPValue *A, *B; using namespace VPlanPatternMatch; - if (match(V, m_ActiveLaneMask(m_VPValue(A), m_VPValue(B), m_OneInt()))) + if (match(V, m_ActiveLaneMask(m_VPValue(A), m_VPValue(B), m_One()))) return B == Plan.getTripCount() && - (match(A, m_ScalarIVSteps(m_Specific(Plan.getCanonicalIV()), - m_OneInt(), m_Specific(&Plan.getVF()))) || + (match(A, m_ScalarIVSteps(m_Specific(Plan.getCanonicalIV()), m_One(), + m_Specific(&Plan.getVF()))) || IsWideCanonicalIV(A)); return match(V, m_Binary(m_VPValue(A), m_VPValue(B))) && From b7c270e688519019af59a374a06b263534f6cdfc Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Thu, 11 Sep 2025 08:54:25 +0100 Subject: [PATCH 3/4] [VPlan/PM] Improve comment --- llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index 2b22c1ab6fe27..42cb42e2be35e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -155,8 +155,8 @@ struct is_one { /// Match an integer 0 or a vector with all elements equal to 0. /// For vectors, this includes constants with undefined elements. -/// The reason for not naming this m_Zero() is because PatternMatch::m_Zero() -/// also matches null constants. +/// This is different from PatternMatch::m_Zero(), which also matches null +/// constants. inline int_pred_ty m_ZeroInt() { return int_pred_ty(); } From 4d42f8a7d31cee9b476359c040877407718c59dd Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Mon, 15 Sep 2025 11:01:48 +0100 Subject: [PATCH 4/4] [VPlan] Strip a comment --- llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index 42cb42e2be35e..8b94378467706 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -155,8 +155,6 @@ struct is_one { /// Match an integer 0 or a vector with all elements equal to 0. /// For vectors, this includes constants with undefined elements. -/// This is different from PatternMatch::m_Zero(), which also matches null -/// constants. inline int_pred_ty m_ZeroInt() { return int_pred_ty(); }