From b304148f78c7e27412ba6bfa706b4e550dc1edf5 Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Mon, 17 Nov 2025 16:05:29 +0000 Subject: [PATCH 1/2] [IVDesc] Use SCEVPatternMatch to improve code (NFC) --- llvm/include/llvm/Analysis/IVDescriptors.h | 2 +- llvm/lib/Analysis/IVDescriptors.cpp | 49 +++++++------------ llvm/lib/Analysis/LoopInfo.cpp | 2 +- .../Vectorize/LoopVectorizationLegality.cpp | 4 +- 4 files changed, 23 insertions(+), 34 deletions(-) diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h index 654a5f10cea96..2b7707db9d16b 100644 --- a/llvm/include/llvm/Analysis/IVDescriptors.h +++ b/llvm/include/llvm/Analysis/IVDescriptors.h @@ -401,7 +401,7 @@ class InductionDescriptor { InductionKind getKind() const { return IK; } const SCEV *getStep() const { return Step; } BinaryOperator *getInductionBinOp() const { return InductionBinOp; } - LLVM_ABI ConstantInt *getConstIntStepValue() const; + LLVM_ABI const APInt *getStepValue() const; /// Returns true if \p Phi is an induction in the loop \p L. If \p Phi is an /// induction, the induction descriptor \p D will contain the data describing diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp index 641850b46bbd8..409a45d6fc542 100644 --- a/llvm/lib/Analysis/IVDescriptors.cpp +++ b/llvm/lib/Analysis/IVDescriptors.cpp @@ -15,6 +15,7 @@ #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" +#include "llvm/Analysis/ScalarEvolutionPatternMatch.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Instructions.h" @@ -25,6 +26,7 @@ using namespace llvm; using namespace llvm::PatternMatch; +using namespace llvm::SCEVPatternMatch; #define DEBUG_TYPE "iv-descriptors" @@ -719,11 +721,12 @@ RecurrenceDescriptor::isFindIVPattern(RecurKind Kind, Loop *TheLoop, if (!SE.isSCEVable(Ty)) return std::nullopt; - auto *AR = dyn_cast(SE.getSCEV(V)); - if (!AR || AR->getLoop() != TheLoop) + auto *AR = SE.getSCEV(V); + const SCEV *Step; + if (!match(AR, m_scev_AffineAddRec(m_SCEV(), m_SCEV(Step), + m_SpecificLoop(TheLoop)))) return std::nullopt; - const SCEV *Step = AR->getStepRecurrence(SE); if ((isFindFirstIVRecurrenceKind(Kind) && !SE.isKnownNegative(Step)) || (isFindLastIVRecurrenceKind(Kind) && !SE.isKnownPositive(Step))) return std::nullopt; @@ -1377,7 +1380,7 @@ InductionDescriptor::InductionDescriptor(Value *Start, InductionKind K, "StartValue is not an integer for integer induction"); // Check the Step Value. It should be non-zero integer value. - assert((!getConstIntStepValue() || !getConstIntStepValue()->isZero()) && + assert((!getStepValue() || !getStepValue()->isZero()) && "Step value is zero"); assert((IK == IK_FpInduction || Step->getType()->isIntegerTy()) && @@ -1395,10 +1398,11 @@ InductionDescriptor::InductionDescriptor(Value *Start, InductionKind K, llvm::append_range(RedundantCasts, *Casts); } -ConstantInt *InductionDescriptor::getConstIntStepValue() const { - if (isa(Step)) - return dyn_cast(cast(Step)->getValue()); - return nullptr; +const APInt *InductionDescriptor::getStepValue() const { + const APInt *StepC; + if (!match(Step, m_scev_APInt(StepC))) + return nullptr; + return StepC; } bool InductionDescriptor::isFPInductionPHI(PHINode *Phi, const Loop *TheLoop, @@ -1614,41 +1618,26 @@ bool InductionDescriptor::isInductionPHI( // Check that the PHI is consecutive. const SCEV *PhiScev = Expr ? Expr : SE->getSCEV(Phi); - const SCEVAddRecExpr *AR = dyn_cast(PhiScev); - - if (!AR) { + const SCEV *Step; + if (!match(PhiScev, m_scev_AffineAddRec(m_SCEV(), m_SCEV(Step), + m_SpecificLoop(TheLoop)))) { LLVM_DEBUG(dbgs() << "LV: PHI is not a poly recurrence.\n"); return false; } - if (AR->getLoop() != TheLoop) { - // FIXME: We should treat this as a uniform. Unfortunately, we - // don't currently know how to handled uniform PHIs. - LLVM_DEBUG( - dbgs() << "LV: PHI is a recurrence with respect to an outer loop.\n"); - return false; - } - // This function assumes that InductionPhi is called only on Phi nodes // present inside loop headers. Check for the same, and throw an assert if // the current Phi is not present inside the loop header. - assert(Phi->getParent() == AR->getLoop()->getHeader() - && "Invalid Phi node, not present in loop header"); + assert(Phi->getParent() == TheLoop->getHeader() && + "Invalid Phi node, not present in loop header"); Value *StartValue = - Phi->getIncomingValueForBlock(AR->getLoop()->getLoopPreheader()); + Phi->getIncomingValueForBlock(TheLoop->getLoopPreheader()); - BasicBlock *Latch = AR->getLoop()->getLoopLatch(); + BasicBlock *Latch = TheLoop->getLoopLatch(); if (!Latch) return false; - const SCEV *Step = AR->getStepRecurrence(*SE); - // Calculate the pointer stride and check if it is consecutive. - // The stride may be a constant or a loop invariant integer value. - const SCEVConstant *ConstStep = dyn_cast(Step); - if (!ConstStep && !SE->isLoopInvariant(Step, TheLoop)) - return false; - if (PhiTy->isIntegerTy()) { BinaryOperator *BOp = dyn_cast(Phi->getIncomingValueForBlock(Latch)); diff --git a/llvm/lib/Analysis/LoopInfo.cpp b/llvm/lib/Analysis/LoopInfo.cpp index d84721b7f8f4b..7f261f21e1e5b 100644 --- a/llvm/lib/Analysis/LoopInfo.cpp +++ b/llvm/lib/Analysis/LoopInfo.cpp @@ -421,7 +421,7 @@ bool Loop::isCanonical(ScalarEvolution &SE) const { if (IndDesc.getInductionOpcode() != Instruction::Add) return false; - ConstantInt *Step = IndDesc.getConstIntStepValue(); + const APInt *Step = IndDesc.getStepValue(); if (!Step || !Step->isOne()) return false; diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp index 03112c67dda7b..bc57ef5e71725 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -706,7 +706,7 @@ void LoopVectorizationLegality::addInductionPhi( // Int inductions are special because we only allow one IV. if (ID.getKind() == InductionDescriptor::IK_IntInduction && - ID.getConstIntStepValue() && ID.getConstIntStepValue()->isOne() && + ID.getStepValue() && ID.getStepValue()->isOne() && isa(ID.getStartValue()) && cast(ID.getStartValue())->isNullValue()) { @@ -891,7 +891,7 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) { if (AllowStridedPointerIVs) return false; return ID.getKind() == InductionDescriptor::IK_PtrInduction && - ID.getConstIntStepValue() == nullptr; + ID.getStepValue() == nullptr; }; // TODO: Instead of recording the AllowedExit, it would be good to From 8749faa5402225c86aa332635e34b2dea92f4daf Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Tue, 18 Nov 2025 10:15:53 +0000 Subject: [PATCH 2/2] [IVDesc] Address review --- llvm/include/llvm/Analysis/IVDescriptors.h | 2 +- llvm/lib/Analysis/IVDescriptors.cpp | 15 +++++++++------ llvm/lib/Analysis/LoopInfo.cpp | 2 +- .../Vectorize/LoopVectorizationLegality.cpp | 4 ++-- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h index 2b7707db9d16b..654a5f10cea96 100644 --- a/llvm/include/llvm/Analysis/IVDescriptors.h +++ b/llvm/include/llvm/Analysis/IVDescriptors.h @@ -401,7 +401,7 @@ class InductionDescriptor { InductionKind getKind() const { return IK; } const SCEV *getStep() const { return Step; } BinaryOperator *getInductionBinOp() const { return InductionBinOp; } - LLVM_ABI const APInt *getStepValue() const; + LLVM_ABI ConstantInt *getConstIntStepValue() const; /// Returns true if \p Phi is an induction in the loop \p L. If \p Phi is an /// induction, the induction descriptor \p D will contain the data describing diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp index 409a45d6fc542..9b8efc020769c 100644 --- a/llvm/lib/Analysis/IVDescriptors.cpp +++ b/llvm/lib/Analysis/IVDescriptors.cpp @@ -1380,7 +1380,7 @@ InductionDescriptor::InductionDescriptor(Value *Start, InductionKind K, "StartValue is not an integer for integer induction"); // Check the Step Value. It should be non-zero integer value. - assert((!getStepValue() || !getStepValue()->isZero()) && + assert((!getConstIntStepValue() || !getConstIntStepValue()->isZero()) && "Step value is zero"); assert((IK == IK_FpInduction || Step->getType()->isIntegerTy()) && @@ -1398,11 +1398,10 @@ InductionDescriptor::InductionDescriptor(Value *Start, InductionKind K, llvm::append_range(RedundantCasts, *Casts); } -const APInt *InductionDescriptor::getStepValue() const { - const APInt *StepC; - if (!match(Step, m_scev_APInt(StepC))) - return nullptr; - return StepC; +ConstantInt *InductionDescriptor::getConstIntStepValue() const { + if (auto *C = dyn_cast(Step)) + return dyn_cast(C->getValue()); + return nullptr; } bool InductionDescriptor::isFPInductionPHI(PHINode *Phi, const Loop *TheLoop, @@ -1619,6 +1618,10 @@ bool InductionDescriptor::isInductionPHI( // Check that the PHI is consecutive. const SCEV *PhiScev = Expr ? Expr : SE->getSCEV(Phi); const SCEV *Step; + + // FIXME: We are currently matching the specific loop TheLoop; if it doesn't + // match, we should treat it as a uniform. Unfortunately, we don't currently + // know how to handled uniform PHIs. if (!match(PhiScev, m_scev_AffineAddRec(m_SCEV(), m_SCEV(Step), m_SpecificLoop(TheLoop)))) { LLVM_DEBUG(dbgs() << "LV: PHI is not a poly recurrence.\n"); diff --git a/llvm/lib/Analysis/LoopInfo.cpp b/llvm/lib/Analysis/LoopInfo.cpp index 7f261f21e1e5b..d84721b7f8f4b 100644 --- a/llvm/lib/Analysis/LoopInfo.cpp +++ b/llvm/lib/Analysis/LoopInfo.cpp @@ -421,7 +421,7 @@ bool Loop::isCanonical(ScalarEvolution &SE) const { if (IndDesc.getInductionOpcode() != Instruction::Add) return false; - const APInt *Step = IndDesc.getStepValue(); + ConstantInt *Step = IndDesc.getConstIntStepValue(); if (!Step || !Step->isOne()) return false; diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp index bc57ef5e71725..03112c67dda7b 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -706,7 +706,7 @@ void LoopVectorizationLegality::addInductionPhi( // Int inductions are special because we only allow one IV. if (ID.getKind() == InductionDescriptor::IK_IntInduction && - ID.getStepValue() && ID.getStepValue()->isOne() && + ID.getConstIntStepValue() && ID.getConstIntStepValue()->isOne() && isa(ID.getStartValue()) && cast(ID.getStartValue())->isNullValue()) { @@ -891,7 +891,7 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) { if (AllowStridedPointerIVs) return false; return ID.getKind() == InductionDescriptor::IK_PtrInduction && - ID.getStepValue() == nullptr; + ID.getConstIntStepValue() == nullptr; }; // TODO: Instead of recording the AllowedExit, it would be good to