Skip to content

Commit 1601aff

Browse files
artagnonartagnon
committed
[SCEVPatternMatch] Introduce m_scev_AddRec
Introduce m_scev_AddRec to match affine AddRecs, a class_match for SCEVConstant, and demonstrate their utility in LSR and SCEV. While at it, rename m_Specific to m_scev_Specific for clarity.
1 parent 3764ba2 commit 1601aff

File tree

3 files changed

+72
-80
lines changed

3 files changed

+72
-80
lines changed

‎llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ template <typename Class> struct class_match {
6161
};
6262

6363
inline class_match<const SCEV> m_SCEV() { return class_match<const SCEV>(); }
64+
inline class_match<const SCEVConstant> m_SCEVConstant() {
65+
return class_match<const SCEVConstant>();
66+
}
6467

6568
template <typename Class> struct bind_ty {
6669
Class *&VR;
@@ -95,7 +98,7 @@ struct specificscev_ty {
9598
};
9699

97100
/// Match if we have a specific specified SCEV.
98-
inline specificscev_ty m_Specific(const SCEV *S) { return S; }
101+
inline specificscev_ty m_scev_Specific(const SCEV *S) { return S; }
99102

100103
struct is_specific_cst {
101104
uint64_t CV;
@@ -192,6 +195,13 @@ inline SCEVBinaryExpr_match<SCEVUDivExpr, Op0_t, Op1_t>
192195
m_scev_UDiv(const Op0_t &Op0, const Op1_t &Op1) {
193196
return m_scev_Binary<SCEVUDivExpr>(Op0, Op1);
194197
}
198+
199+
// Matches affine AddRecs.
200+
template <typename Op0_t, typename Op1_t>
201+
inline SCEVBinaryExpr_match<SCEVAddRecExpr, Op0_t, Op1_t>
202+
m_scev_AddRec(const Op0_t &Op0, const Op1_t &Op1) {
203+
return m_scev_Binary<SCEVAddRecExpr>(Op0, Op1);
204+
}
195205
} // namespace SCEVPatternMatch
196206
} // namespace llvm
197207

‎llvm/lib/Analysis/ScalarEvolution.cpp‎

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12480,26 +12480,21 @@ static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE,
1248012480
if (!ICmpInst::isRelational(Pred))
1248112481
return false;
1248212482

12483-
const SCEVAddRecExpr *LAR = dyn_cast<SCEVAddRecExpr>(LHS);
12484-
if (!LAR)
12485-
return false;
12486-
const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(RHS);
12487-
if (!RAR)
12483+
const SCEV *LStart, *RStart, *Step;
12484+
if (!match(LHS, m_scev_AddRec(m_SCEV(LStart), m_SCEV(Step))) ||
12485+
!match(RHS, m_scev_AddRec(m_SCEV(RStart), m_scev_Specific(Step))))
1248812486
return false;
12487+
const SCEVAddRecExpr *LAR = cast<SCEVAddRecExpr>(LHS);
12488+
const SCEVAddRecExpr *RAR = cast<SCEVAddRecExpr>(RHS);
1248912489
if (LAR->getLoop() != RAR->getLoop())
1249012490
return false;
12491-
if (!LAR->isAffine() || !RAR->isAffine())
12492-
return false;
12493-
12494-
if (LAR->getStepRecurrence(SE) != RAR->getStepRecurrence(SE))
12495-
return false;
1249612491

1249712492
SCEV::NoWrapFlags NW = ICmpInst::isSigned(Pred) ?
1249812493
SCEV::FlagNSW : SCEV::FlagNUW;
1249912494
if (!LAR->getNoWrapFlags(NW) || !RAR->getNoWrapFlags(NW))
1250012495
return false;
1250112496

12502-
return SE.isKnownPredicate(Pred, LAR->getStart(), RAR->getStart());
12497+
return SE.isKnownPredicate(Pred, LStart, RStart);
1250312498
}
1250412499

1250512500
/// Is LHS `Pred` RHS true on the virtue of LHS or RHS being a Min or Max
@@ -12716,15 +12711,15 @@ static bool isKnownPredicateExtendIdiom(CmpPredicate Pred, const SCEV *LHS,
1271612711
case ICmpInst::ICMP_SLE: {
1271712712
// If operand >=s 0 then ZExt == SExt. If operand <s 0 then SExt <s ZExt.
1271812713
return match(LHS, m_scev_SExt(m_SCEV(Op))) &&
12719-
match(RHS, m_scev_ZExt(m_Specific(Op)));
12714+
match(RHS, m_scev_ZExt(m_scev_Specific(Op)));
1272012715
}
1272112716
case ICmpInst::ICMP_UGE:
1272212717
std::swap(LHS, RHS);
1272312718
[[fallthrough]];
1272412719
case ICmpInst::ICMP_ULE: {
1272512720
// If operand >=u 0 then ZExt == SExt. If operand <u 0 then ZExt <u SExt.
1272612721
return match(LHS, m_scev_ZExt(m_SCEV(Op))) &&
12727-
match(RHS, m_scev_SExt(m_Specific(Op)));
12722+
match(RHS, m_scev_SExt(m_scev_Specific(Op)));
1272812723
}
1272912724
default:
1273012725
return false;

‎llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp‎

Lines changed: 53 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -77,11 +77,11 @@
7777
#include "llvm/Analysis/ScalarEvolution.h"
7878
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
7979
#include "llvm/Analysis/ScalarEvolutionNormalization.h"
80+
#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
8081
#include "llvm/Analysis/TargetLibraryInfo.h"
8182
#include "llvm/Analysis/TargetTransformInfo.h"
8283
#include "llvm/Analysis/ValueTracking.h"
8384
#include "llvm/BinaryFormat/Dwarf.h"
84-
#include "llvm/Config/llvm-config.h"
8585
#include "llvm/IR/BasicBlock.h"
8686
#include "llvm/IR/Constant.h"
8787
#include "llvm/IR/Constants.h"
@@ -128,6 +128,7 @@
128128
#include <utility>
129129

130130
using namespace llvm;
131+
using namespace SCEVPatternMatch;
131132

132133
#define DEBUG_TYPE "loop-reduce"
133134

@@ -556,16 +557,17 @@ static void DoInitialMatch(const SCEV *S, Loop *L,
556557
}
557558

558559
// Look at addrec operands.
559-
if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S))
560-
if (!AR->getStart()->isZero() && AR->isAffine()) {
561-
DoInitialMatch(AR->getStart(), L, Good, Bad, SE);
562-
DoInitialMatch(SE.getAddRecExpr(SE.getConstant(AR->getType(), 0),
563-
AR->getStepRecurrence(SE),
564-
// FIXME: AR->getNoWrapFlags()
565-
AR->getLoop(), SCEV::FlagAnyWrap),
566-
L, Good, Bad, SE);
567-
return;
568-
}
560+
const SCEV *Start, *Step;
561+
if (match(S, m_scev_AddRec(m_SCEV(Start), m_SCEV(Step))) &&
562+
!Start->isZero()) {
563+
DoInitialMatch(Start, L, Good, Bad, SE);
564+
DoInitialMatch(SE.getAddRecExpr(SE.getConstant(S->getType(), 0), Step,
565+
// FIXME: AR->getNoWrapFlags()
566+
cast<SCEVAddRecExpr>(S)->getLoop(),
567+
SCEV::FlagAnyWrap),
568+
L, Good, Bad, SE);
569+
return;
570+
}
569571

570572
// Handle a multiplication by -1 (negation) if it didn't fold.
571573
if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S))
@@ -1436,22 +1438,16 @@ void Cost::RateRegister(const Formula &F, const SCEV *Reg,
14361438
unsigned LoopCost = 1;
14371439
if (TTI->isIndexedLoadLegal(TTI->MIM_PostInc, AR->getType()) ||
14381440
TTI->isIndexedStoreLegal(TTI->MIM_PostInc, AR->getType())) {
1439-
1440-
// If the step size matches the base offset, we could use pre-indexed
1441-
// addressing.
1442-
if (AMK == TTI::AMK_PreIndexed && F.BaseOffset.isFixed()) {
1443-
if (auto *Step = dyn_cast<SCEVConstant>(AR->getStepRecurrence(*SE)))
1444-
if (Step->getAPInt() == F.BaseOffset.getFixedValue())
1445-
LoopCost = 0;
1446-
} else if (AMK == TTI::AMK_PostIndexed) {
1447-
const SCEV *LoopStep = AR->getStepRecurrence(*SE);
1448-
if (isa<SCEVConstant>(LoopStep)) {
1449-
const SCEV *LoopStart = AR->getStart();
1450-
if (!isa<SCEVConstant>(LoopStart) &&
1451-
SE->isLoopInvariant(LoopStart, L))
1452-
LoopCost = 0;
1453-
}
1454-
}
1441+
const SCEV *Start;
1442+
const SCEVConstant *Step;
1443+
if (match(AR, m_scev_AddRec(m_SCEV(Start), m_SCEVConstant(Step))))
1444+
// If the step size matches the base offset, we could use pre-indexed
1445+
// addressing.
1446+
if ((AMK == TTI::AMK_PreIndexed && F.BaseOffset.isFixed() &&
1447+
Step->getAPInt() == F.BaseOffset.getFixedValue()) ||
1448+
(AMK == TTI::AMK_PostIndexed && !isa<SCEVConstant>(Start) &&
1449+
SE->isLoopInvariant(Start, L)))
1450+
LoopCost = 0;
14551451
}
14561452
C.AddRecCost += LoopCost;
14571453

@@ -2544,13 +2540,10 @@ ICmpInst *LSRInstance::OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse) {
25442540
// Check the relevant induction variable for conformance to
25452541
// the pattern.
25462542
const SCEV *IV = SE.getSCEV(Cond->getOperand(0));
2547-
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(IV);
2548-
if (!AR || !AR->isAffine() ||
2549-
AR->getStart() != One ||
2550-
AR->getStepRecurrence(SE) != One)
2543+
if (!match(IV, m_scev_AddRec(m_scev_SpecificInt(1), m_scev_SpecificInt(1))))
25512544
return Cond;
25522545

2553-
assert(AR->getLoop() == L &&
2546+
assert(cast<SCEVAddRecExpr>(IV)->getLoop() == L &&
25542547
"Loop condition operand is an addrec in a different loop!");
25552548

25562549
// Check the right operand of the select, and remember it, as it will
@@ -3345,7 +3338,7 @@ void LSRInstance::CollectChains() {
33453338
void LSRInstance::FinalizeChain(IVChain &Chain) {
33463339
assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
33473340
LLVM_DEBUG(dbgs() << "Final Chain: " << *Chain.Incs[0].UserInst << "\n");
3348-
3341+
33493342
for (const IVInc &Inc : Chain) {
33503343
LLVM_DEBUG(dbgs() << " Inc: " << *Inc.UserInst << "\n");
33513344
auto UseI = find(Inc.UserInst->operands(), Inc.IVOperand);
@@ -3848,26 +3841,27 @@ static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C,
38483841
Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder);
38493842
}
38503843
return nullptr;
3851-
} else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
3844+
}
3845+
const SCEV *Start, *Step;
3846+
if (match(S, m_scev_AddRec(m_SCEV(Start), m_SCEV(Step)))) {
38523847
// Split a non-zero base out of an addrec.
3853-
if (AR->getStart()->isZero() || !AR->isAffine())
3848+
if (Start->isZero())
38543849
return S;
38553850

3856-
const SCEV *Remainder = CollectSubexprs(AR->getStart(),
3857-
C, Ops, L, SE, Depth+1);
3851+
const SCEV *Remainder = CollectSubexprs(Start, C, Ops, L, SE, Depth + 1);
38583852
// Split the non-zero AddRec unless it is part of a nested recurrence that
38593853
// does not pertain to this loop.
3860-
if (Remainder && (AR->getLoop() == L || !isa<SCEVAddRecExpr>(Remainder))) {
3854+
if (Remainder && (cast<SCEVAddRecExpr>(S)->getLoop() == L ||
3855+
!isa<SCEVAddRecExpr>(Remainder))) {
38613856
Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder);
38623857
Remainder = nullptr;
38633858
}
3864-
if (Remainder != AR->getStart()) {
3859+
if (Remainder != Start) {
38653860
if (!Remainder)
3866-
Remainder = SE.getConstant(AR->getType(), 0);
3867-
return SE.getAddRecExpr(Remainder,
3868-
AR->getStepRecurrence(SE),
3869-
AR->getLoop(),
3870-
//FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
3861+
Remainder = SE.getConstant(S->getType(), 0);
3862+
return SE.getAddRecExpr(Remainder, Step,
3863+
cast<SCEVAddRecExpr>(S)->getLoop(),
3864+
// FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
38713865
SCEV::FlagAnyWrap);
38723866
}
38733867
} else if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
@@ -3895,17 +3889,13 @@ static bool mayUsePostIncMode(const TargetTransformInfo &TTI,
38953889
if (LU.Kind != LSRUse::Address ||
38963890
!LU.AccessTy.getType()->isIntOrIntVectorTy())
38973891
return false;
3898-
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S);
3899-
if (!AR)
3900-
return false;
3901-
const SCEV *LoopStep = AR->getStepRecurrence(SE);
3902-
if (!isa<SCEVConstant>(LoopStep))
3892+
const SCEV *Start;
3893+
if (!match(S, m_scev_AddRec(m_SCEV(Start), m_SCEVConstant())))
39033894
return false;
39043895
// Check if a post-indexed load/store can be used.
3905-
if (TTI.isIndexedLoadLegal(TTI.MIM_PostInc, AR->getType()) ||
3906-
TTI.isIndexedStoreLegal(TTI.MIM_PostInc, AR->getType())) {
3907-
const SCEV *LoopStart = AR->getStart();
3908-
if (!isa<SCEVConstant>(LoopStart) && SE.isLoopInvariant(LoopStart, L))
3896+
if (TTI.isIndexedLoadLegal(TTI.MIM_PostInc, S->getType()) ||
3897+
TTI.isIndexedStoreLegal(TTI.MIM_PostInc, S->getType())) {
3898+
if (!isa<SCEVConstant>(Start) && SE.isLoopInvariant(Start, L))
39093899
return true;
39103900
}
39113901
return false;
@@ -4164,18 +4154,15 @@ void LSRInstance::GenerateConstantOffsetsImpl(
41644154
// base pointer for each iteration of the loop, resulting in no extra add/sub
41654155
// instructions for pointer updating.
41664156
if (AMK == TTI::AMK_PreIndexed && LU.Kind == LSRUse::Address) {
4167-
if (auto *GAR = dyn_cast<SCEVAddRecExpr>(G)) {
4168-
if (auto *StepRec =
4169-
dyn_cast<SCEVConstant>(GAR->getStepRecurrence(SE))) {
4170-
const APInt &StepInt = StepRec->getAPInt();
4171-
int64_t Step = StepInt.isNegative() ?
4172-
StepInt.getSExtValue() : StepInt.getZExtValue();
4173-
4174-
for (Immediate Offset : Worklist) {
4175-
if (Offset.isFixed()) {
4176-
Offset = Immediate::getFixed(Offset.getFixedValue() - Step);
4177-
GenerateOffset(G, Offset);
4178-
}
4157+
const APInt *StepInt;
4158+
if (match(G, m_scev_AddRec(m_SCEV(), m_scev_APInt(StepInt)))) {
4159+
int64_t Step = StepInt->isNegative() ? StepInt->getSExtValue()
4160+
: StepInt->getZExtValue();
4161+
4162+
for (Immediate Offset : Worklist) {
4163+
if (Offset.isFixed()) {
4164+
Offset = Immediate::getFixed(Offset.getFixedValue() - Step);
4165+
GenerateOffset(G, Offset);
41794166
}
41804167
}
41814168
}
@@ -6647,7 +6634,7 @@ struct SCEVDbgValueBuilder {
66476634
if (Op.getOp() != dwarf::DW_OP_LLVM_arg) {
66486635
Op.appendToVector(DestExpr);
66496636
continue;
6650-
}
6637+
}
66516638

66526639
DestExpr.push_back(dwarf::DW_OP_LLVM_arg);
66536640
// `DW_OP_LLVM_arg n` represents the nth LocationOp in this SCEV,

0 commit comments

Comments
 (0)