Skip to content

Commit 2312a06

Browse files
alinasalinas
committed
[SCEV] Add option to forget everything in SCEV.
Summary: Create a method to forget everything in SCEV. Add a cl::opt and PassManagerBuilder option to use this in LoopUnroll. Motivation: Certain Halide applications spend a very long time compiling in forgetLoop, and prefer to forget everything and rebuild SCEV from scratch. Sample difference in compile time reduction: 21.04 to 14.78 using current ToT release build. Testcase showcasing this cannot be opensourced and is fairly large. The option disabled by default, but it may be desirable to enable by default. Evidence in favor (two difference runs on different days/ToT state): File Before (s) After (s) clang-9.bc 7267.91 6639.14 llvm-as.bc 194.12 194.12 llvm-dis.bc 62.50 62.50 opt.bc 1855.85 1857.53 File Before (s) After (s) clang-9.bc 8588.70 7812.83 llvm-as.bc 196.20 194.78 llvm-dis.bc 61.55 61.97 opt.bc 1739.78 1886.26 Reviewers: sanjoy Subscribers: mehdi_amini, jlebar, zzheng, javed.absar, dmgreen, jdoerfert, llvm-commits Tags: #llvm Differential Revision: https://reviews.llvm.org/D60144 llvm-svn: 358304
1 parent 5776938 commit 2312a06

File tree

11 files changed

+106
-48
lines changed

11 files changed

+106
-48
lines changed

‎llvm/include/llvm/Analysis/ScalarEvolution.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -782,6 +782,13 @@ class ScalarEvolution {
782782
/// backedge-taken count.
783783
bool hasLoopInvariantBackedgeTakenCount(const Loop *L);
784784

785+
// This method should be called by the client when it made any change that
786+
// would invalidate SCEV's answers, and the client wants to remove all loop
787+
// information held internally by ScalarEvolution. This is intended to be used
788+
// when the alternative to forget a loop is too expensive (i.e. large loop
789+
// bodies).
790+
void forgetAllLoops();
791+
785792
/// This method should be called by the client when it has changed a loop in
786793
/// a way that may effect ScalarEvolution's ability to compute a trip count,
787794
/// or if the loop is deleted. This call is potentially expensive for large

‎llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ class PassManagerBuilder {
148148
bool RerollLoops;
149149
bool NewGVN;
150150
bool DisableGVNLoadPRE;
151+
bool ForgetAllSCEVInLoopUnroll;
151152
bool VerifyInput;
152153
bool VerifyOutput;
153154
bool MergeFunctions;

‎llvm/include/llvm/Transforms/Scalar.h

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -183,11 +183,13 @@ Pass *createLoopInstSimplifyPass();
183183
// LoopUnroll - This pass is a simple loop unrolling pass.
184184
//
185185
Pass *createLoopUnrollPass(int OptLevel = 2, bool OnlyWhenForced = false,
186-
int Threshold = -1, int Count = -1,
187-
int AllowPartial = -1, int Runtime = -1,
188-
int UpperBound = -1, int AllowPeeling = -1);
186+
bool ForgetAllSCEV = false, int Threshold = -1,
187+
int Count = -1, int AllowPartial = -1,
188+
int Runtime = -1, int UpperBound = -1,
189+
int AllowPeeling = -1);
189190
// Create an unrolling pass for full unrolling that uses exact trip count only.
190-
Pass *createSimpleLoopUnrollPass(int OptLevel = 2, bool OnlyWhenForced = false);
191+
Pass *createSimpleLoopUnrollPass(int OptLevel = 2, bool OnlyWhenForced = false,
192+
bool ForgetAllSCEV = false);
191193

192194
//===----------------------------------------------------------------------===//
193195
//

‎llvm/include/llvm/Transforms/Utils/UnrollLoop.h

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -67,17 +67,17 @@ LoopUnrollResult UnrollLoop(Loop *L, unsigned Count, unsigned TripCount,
6767
bool AllowExpensiveTripCount, bool PreserveCondBr,
6868
bool PreserveOnlyFirst, unsigned TripMultiple,
6969
unsigned PeelCount, bool UnrollRemainder,
70-
LoopInfo *LI, ScalarEvolution *SE,
71-
DominatorTree *DT, AssumptionCache *AC,
72-
OptimizationRemarkEmitter *ORE, bool PreserveLCSSA,
73-
Loop **RemainderLoop = nullptr);
70+
bool ForgetAllSCEV, LoopInfo *LI,
71+
ScalarEvolution *SE, DominatorTree *DT,
72+
AssumptionCache *AC, OptimizationRemarkEmitter *ORE,
73+
bool PreserveLCSSA, Loop **RemainderLoop = nullptr);
7474

7575
bool UnrollRuntimeLoopRemainder(Loop *L, unsigned Count,
7676
bool AllowExpensiveTripCount,
7777
bool UseEpilogRemainder, bool UnrollRemainder,
78-
LoopInfo *LI, ScalarEvolution *SE,
79-
DominatorTree *DT, AssumptionCache *AC,
80-
bool PreserveLCSSA,
78+
bool ForgetAllSCEV, LoopInfo *LI,
79+
ScalarEvolution *SE, DominatorTree *DT,
80+
AssumptionCache *AC, bool PreserveLCSSA,
8181
Loop **ResultLoop = nullptr);
8282

8383
void computePeelCount(Loop *L, unsigned LoopSize,

‎llvm/lib/Analysis/ScalarEvolution.cpp

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6789,6 +6789,28 @@ ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
67896789
return BackedgeTakenCounts.find(L)->second = std::move(Result);
67906790
}
67916791

6792+
void ScalarEvolution::forgetAllLoops() {
6793+
// This method is intended to forget all info about loops. It should
6794+
// invalidate caches as if the following happened:
6795+
// - The trip counts of all loops have changed arbitrarily
6796+
// - Every llvm::Value has been updated in place to produce a different
6797+
// result.
6798+
BackedgeTakenCounts.clear();
6799+
PredicatedBackedgeTakenCounts.clear();
6800+
LoopPropertiesCache.clear();
6801+
ConstantEvolutionLoopExitValue.clear();
6802+
ValueExprMap.clear();
6803+
ValuesAtScopes.clear();
6804+
LoopDispositions.clear();
6805+
BlockDispositions.clear();
6806+
UnsignedRanges.clear();
6807+
SignedRanges.clear();
6808+
ExprValueMap.clear();
6809+
HasRecMap.clear();
6810+
MinTrailingZerosCache.clear();
6811+
PredicatedSCEVRewrites.clear();
6812+
}
6813+
67926814
void ScalarEvolution::forgetLoop(const Loop *L) {
67936815
// Drop any stored trip count value.
67946816
auto RemoveLoopFromBackedgeMap =

‎llvm/lib/Transforms/IPO/PassManagerBuilder.cpp

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,12 @@ cl::opt<bool> EnableOrderFileInstrumentation(
158158
"enable-order-file-instrumentation", cl::init(false), cl::Hidden,
159159
cl::desc("Enable order file instrumentation (default = off)"));
160160

161+
cl::opt<bool> ForgetSCEVInLoopUnroll(
162+
"forget-scev-loop-unroll", cl::init(false), cl::Hidden,
163+
cl::desc("Forget everything in SCEV when doing LoopUnroll, instead of just"
164+
" the current top-most loop. This is somtimes preferred to reduce"
165+
" compile time."));
166+
161167
PassManagerBuilder::PassManagerBuilder() {
162168
OptLevel = 2;
163169
SizeLevel = 0;
@@ -169,6 +175,7 @@ PassManagerBuilder::PassManagerBuilder() {
169175
RerollLoops = RunLoopRerolling;
170176
NewGVN = RunNewGVN;
171177
DisableGVNLoadPRE = false;
178+
ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
172179
VerifyInput = false;
173180
VerifyOutput = false;
174181
MergeFunctions = false;
@@ -386,8 +393,9 @@ void PassManagerBuilder::addFunctionSimplificationPasses(
386393
if (EnableLoopInterchange)
387394
MPM.add(createLoopInterchangePass()); // Interchange loops
388395

389-
MPM.add(createSimpleLoopUnrollPass(OptLevel,
390-
DisableUnrollLoops)); // Unroll small loops
396+
// Unroll small loops
397+
MPM.add(createSimpleLoopUnrollPass(OptLevel, DisableUnrollLoops,
398+
ForgetAllSCEVInLoopUnroll));
391399
addExtensionsToPM(EP_LoopOptimizerEnd, MPM);
392400
// This ends the loop pass pipelines.
393401

@@ -724,8 +732,9 @@ void PassManagerBuilder::populateModulePassManager(
724732
MPM.add(createLoopUnrollAndJamPass(OptLevel));
725733
}
726734

727-
MPM.add(createLoopUnrollPass(OptLevel,
728-
DisableUnrollLoops)); // Unroll small loops
735+
// Unroll small loops
736+
MPM.add(createLoopUnrollPass(OptLevel, DisableUnrollLoops,
737+
ForgetAllSCEVInLoopUnroll));
729738

730739
if (!DisableUnrollLoops) {
731740
// LoopUnroll may generate some redundency to cleanup.
@@ -919,11 +928,13 @@ void PassManagerBuilder::addLTOOptimizationPasses(legacy::PassManagerBase &PM) {
919928
if (EnableLoopInterchange)
920929
PM.add(createLoopInterchangePass());
921930

922-
PM.add(createSimpleLoopUnrollPass(OptLevel,
923-
DisableUnrollLoops)); // Unroll small loops
931+
// Unroll small loops
932+
PM.add(createSimpleLoopUnrollPass(OptLevel, DisableUnrollLoops,
933+
ForgetAllSCEVInLoopUnroll));
924934
PM.add(createLoopVectorizePass(true, !LoopVectorize));
925935
// The vectorizer may have significantly shortened a loop body; unroll again.
926-
PM.add(createLoopUnrollPass(OptLevel, DisableUnrollLoops));
936+
PM.add(createLoopUnrollPass(OptLevel, DisableUnrollLoops,
937+
ForgetAllSCEVInLoopUnroll));
927938

928939
PM.add(createWarnMissedTransformationsPass());
929940

‎llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -964,7 +964,7 @@ static LoopUnrollResult tryToUnrollLoop(
964964
Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
965965
const TargetTransformInfo &TTI, AssumptionCache &AC,
966966
OptimizationRemarkEmitter &ORE, bool PreserveLCSSA, int OptLevel,
967-
bool OnlyWhenForced, Optional<unsigned> ProvidedCount,
967+
bool OnlyWhenForced, bool ForgetAllSCEV, Optional<unsigned> ProvidedCount,
968968
Optional<unsigned> ProvidedThreshold, Optional<bool> ProvidedAllowPartial,
969969
Optional<bool> ProvidedRuntime, Optional<bool> ProvidedUpperBound,
970970
Optional<bool> ProvidedAllowPeeling) {
@@ -1082,7 +1082,7 @@ static LoopUnrollResult tryToUnrollLoop(
10821082
LoopUnrollResult UnrollResult = UnrollLoop(
10831083
L, UP.Count, TripCount, UP.Force, UP.Runtime, UP.AllowExpensiveTripCount,
10841084
UseUpperBound, MaxOrZero, TripMultiple, UP.PeelCount, UP.UnrollRemainder,
1085-
LI, &SE, &DT, &AC, &ORE, PreserveLCSSA, &RemainderLoop);
1085+
ForgetAllSCEV, LI, &SE, &DT, &AC, &ORE, PreserveLCSSA, &RemainderLoop);
10861086
if (UnrollResult == LoopUnrollResult::Unmodified)
10871087
return LoopUnrollResult::Unmodified;
10881088

@@ -1131,6 +1131,11 @@ class LoopUnroll : public LoopPass {
11311131
/// metadata are considered. All other loops are skipped.
11321132
bool OnlyWhenForced;
11331133

1134+
/// If false, when SCEV is invalidated, only forget everything in the
1135+
/// top-most loop (call forgetTopMostLoop), of the loop being processed.
1136+
/// Otherwise, forgetAllLoops and rebuild when needed next.
1137+
bool ForgetAllSCEV;
1138+
11341139
Optional<unsigned> ProvidedCount;
11351140
Optional<unsigned> ProvidedThreshold;
11361141
Optional<bool> ProvidedAllowPartial;
@@ -1139,15 +1144,16 @@ class LoopUnroll : public LoopPass {
11391144
Optional<bool> ProvidedAllowPeeling;
11401145

11411146
LoopUnroll(int OptLevel = 2, bool OnlyWhenForced = false,
1142-
Optional<unsigned> Threshold = None,
1147+
bool ForgetAllSCEV = false, Optional<unsigned> Threshold = None,
11431148
Optional<unsigned> Count = None,
11441149
Optional<bool> AllowPartial = None, Optional<bool> Runtime = None,
11451150
Optional<bool> UpperBound = None,
11461151
Optional<bool> AllowPeeling = None)
11471152
: LoopPass(ID), OptLevel(OptLevel), OnlyWhenForced(OnlyWhenForced),
1148-
ProvidedCount(std::move(Count)), ProvidedThreshold(Threshold),
1149-
ProvidedAllowPartial(AllowPartial), ProvidedRuntime(Runtime),
1150-
ProvidedUpperBound(UpperBound), ProvidedAllowPeeling(AllowPeeling) {
1153+
ForgetAllSCEV(ForgetAllSCEV), ProvidedCount(std::move(Count)),
1154+
ProvidedThreshold(Threshold), ProvidedAllowPartial(AllowPartial),
1155+
ProvidedRuntime(Runtime), ProvidedUpperBound(UpperBound),
1156+
ProvidedAllowPeeling(AllowPeeling) {
11511157
initializeLoopUnrollPass(*PassRegistry::getPassRegistry());
11521158
}
11531159

@@ -1171,8 +1177,8 @@ class LoopUnroll : public LoopPass {
11711177

11721178
LoopUnrollResult Result = tryToUnrollLoop(
11731179
L, DT, LI, SE, TTI, AC, ORE, PreserveLCSSA, OptLevel, OnlyWhenForced,
1174-
ProvidedCount, ProvidedThreshold, ProvidedAllowPartial, ProvidedRuntime,
1175-
ProvidedUpperBound, ProvidedAllowPeeling);
1180+
ForgetAllSCEV, ProvidedCount, ProvidedThreshold, ProvidedAllowPartial,
1181+
ProvidedRuntime, ProvidedUpperBound, ProvidedAllowPeeling);
11761182

11771183
if (Result == LoopUnrollResult::FullyUnrolled)
11781184
LPM.markLoopAsDeleted(*L);
@@ -1202,14 +1208,14 @@ INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
12021208
INITIALIZE_PASS_END(LoopUnroll, "loop-unroll", "Unroll loops", false, false)
12031209

12041210
Pass *llvm::createLoopUnrollPass(int OptLevel, bool OnlyWhenForced,
1205-
int Threshold, int Count, int AllowPartial,
1206-
int Runtime, int UpperBound,
1211+
bool ForgetAllSCEV, int Threshold, int Count,
1212+
int AllowPartial, int Runtime, int UpperBound,
12071213
int AllowPeeling) {
12081214
// TODO: It would make more sense for this function to take the optionals
12091215
// directly, but that's dangerous since it would silently break out of tree
12101216
// callers.
12111217
return new LoopUnroll(
1212-
OptLevel, OnlyWhenForced,
1218+
OptLevel, OnlyWhenForced, ForgetAllSCEV,
12131219
Threshold == -1 ? None : Optional<unsigned>(Threshold),
12141220
Count == -1 ? None : Optional<unsigned>(Count),
12151221
AllowPartial == -1 ? None : Optional<bool>(AllowPartial),
@@ -1218,8 +1224,10 @@ Pass *llvm::createLoopUnrollPass(int OptLevel, bool OnlyWhenForced,
12181224
AllowPeeling == -1 ? None : Optional<bool>(AllowPeeling));
12191225
}
12201226

1221-
Pass *llvm::createSimpleLoopUnrollPass(int OptLevel, bool OnlyWhenForced) {
1222-
return createLoopUnrollPass(OptLevel, OnlyWhenForced, -1, -1, 0, 0, 0, 0);
1227+
Pass *llvm::createSimpleLoopUnrollPass(int OptLevel, bool OnlyWhenForced,
1228+
bool ForgetAllSCEV) {
1229+
return createLoopUnrollPass(OptLevel, OnlyWhenForced, ForgetAllSCEV, -1, -1,
1230+
0, 0, 0, 0);
12231231
}
12241232

12251233
PreservedAnalyses LoopFullUnrollPass::run(Loop &L, LoopAnalysisManager &AM,
@@ -1250,7 +1258,7 @@ PreservedAnalyses LoopFullUnrollPass::run(Loop &L, LoopAnalysisManager &AM,
12501258
bool Changed =
12511259
tryToUnrollLoop(&L, AR.DT, &AR.LI, AR.SE, AR.TTI, AR.AC, *ORE,
12521260
/*PreserveLCSSA*/ true, OptLevel, OnlyWhenForced,
1253-
/*Count*/ None,
1261+
/*ForgetAllSCEV*/ false, /*Count*/ None,
12541262
/*Threshold*/ None, /*AllowPartial*/ false,
12551263
/*Runtime*/ false, /*UpperBound*/ false,
12561264
/*AllowPeeling*/ false) != LoopUnrollResult::Unmodified;
@@ -1388,7 +1396,7 @@ PreservedAnalyses LoopUnrollPass::run(Function &F,
13881396
LoopUnrollResult Result = tryToUnrollLoop(
13891397
&L, DT, &LI, SE, TTI, AC, ORE,
13901398
/*PreserveLCSSA*/ true, UnrollOpts.OptLevel, UnrollOpts.OnlyWhenForced,
1391-
/*Count*/ None,
1399+
/*ForgetAllSCEV*/ false, /*Count*/ None,
13921400
/*Threshold*/ None, UnrollOpts.AllowPartial, UnrollOpts.AllowRuntime,
13931401
UnrollOpts.AllowUpperBound, LocalAllowPeeling);
13941402
Changed |= Result != LoopUnrollResult::Unmodified;

‎llvm/lib/Transforms/Utils/LoopUnroll.cpp

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -335,8 +335,9 @@ LoopUnrollResult llvm::UnrollLoop(
335335
Loop *L, unsigned Count, unsigned TripCount, bool Force, bool AllowRuntime,
336336
bool AllowExpensiveTripCount, bool PreserveCondBr, bool PreserveOnlyFirst,
337337
unsigned TripMultiple, unsigned PeelCount, bool UnrollRemainder,
338-
LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
339-
OptimizationRemarkEmitter *ORE, bool PreserveLCSSA, Loop **RemainderLoop) {
338+
bool ForgetAllSCEV, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT,
339+
AssumptionCache *AC, OptimizationRemarkEmitter *ORE, bool PreserveLCSSA,
340+
Loop **RemainderLoop) {
340341

341342
BasicBlock *Preheader = L->getLoopPreheader();
342343
if (!Preheader) {
@@ -469,8 +470,9 @@ LoopUnrollResult llvm::UnrollLoop(
469470

470471
if (RuntimeTripCount && TripMultiple % Count != 0 &&
471472
!UnrollRuntimeLoopRemainder(L, Count, AllowExpensiveTripCount,
472-
EpilogProfitability, UnrollRemainder, LI, SE,
473-
DT, AC, PreserveLCSSA, RemainderLoop)) {
473+
EpilogProfitability, UnrollRemainder,
474+
ForgetAllSCEV, LI, SE, DT, AC, PreserveLCSSA,
475+
RemainderLoop)) {
474476
if (Force)
475477
RuntimeTripCount = false;
476478
else {
@@ -554,8 +556,12 @@ LoopUnrollResult llvm::UnrollLoop(
554556
// and if something changes inside them then any of outer loops may also
555557
// change. When we forget outermost loop, we also forget all contained loops
556558
// and this is what we need here.
557-
if (SE)
558-
SE->forgetTopmostLoop(L);
559+
if (SE) {
560+
if (ForgetAllSCEV)
561+
SE->forgetAllLoops();
562+
else
563+
SE->forgetTopmostLoop(L);
564+
}
559565

560566
bool ContinueOnTrue = L->contains(BI->getSuccessor(0));
561567
BasicBlock *LoopExit = BI->getSuccessor(ContinueOnTrue);

‎llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -197,8 +197,8 @@ LoopUnrollResult llvm::UnrollAndJamLoop(
197197
if (TripMultiple == 1 || TripMultiple % Count != 0) {
198198
if (!UnrollRuntimeLoopRemainder(L, Count, /*AllowExpensiveTripCount*/ false,
199199
/*UseEpilogRemainder*/ true,
200-
UnrollRemainder, LI, SE, DT, AC, true,
201-
EpilogueLoop)) {
200+
UnrollRemainder, /*ForgetAllSCEV*/ false,
201+
LI, SE, DT, AC, true, EpilogueLoop)) {
202202
LLVM_DEBUG(dbgs() << "Won't unroll-and-jam; remainder loop could not be "
203203
"generated when assuming runtime trip count\n");
204204
return LoopUnrollResult::Unmodified;

‎llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -553,10 +553,10 @@ static bool canProfitablyUnrollMultiExitLoop(
553553
bool llvm::UnrollRuntimeLoopRemainder(Loop *L, unsigned Count,
554554
bool AllowExpensiveTripCount,
555555
bool UseEpilogRemainder,
556-
bool UnrollRemainder, LoopInfo *LI,
557-
ScalarEvolution *SE, DominatorTree *DT,
558-
AssumptionCache *AC, bool PreserveLCSSA,
559-
Loop **ResultLoop) {
556+
bool UnrollRemainder, bool ForgetAllSCEV,
557+
LoopInfo *LI, ScalarEvolution *SE,
558+
DominatorTree *DT, AssumptionCache *AC,
559+
bool PreserveLCSSA, Loop **ResultLoop) {
560560
LLVM_DEBUG(dbgs() << "Trying runtime unrolling on Loop: \n");
561561
LLVM_DEBUG(L->dump());
562562
LLVM_DEBUG(UseEpilogRemainder ? dbgs() << "Using epilog remainder.\n"
@@ -953,8 +953,8 @@ bool llvm::UnrollRuntimeLoopRemainder(Loop *L, unsigned Count,
953953
/*Force*/ false, /*AllowRuntime*/ false,
954954
/*AllowExpensiveTripCount*/ false, /*PreserveCondBr*/ true,
955955
/*PreserveOnlyFirst*/ false, /*TripMultiple*/ 1,
956-
/*PeelCount*/ 0, /*UnrollRemainder*/ false, LI, SE, DT, AC,
957-
/*ORE*/ nullptr, PreserveLCSSA);
956+
/*PeelCount*/ 0, /*UnrollRemainder*/ false, ForgetAllSCEV,
957+
LI, SE, DT, AC, /*ORE*/ nullptr, PreserveLCSSA);
958958
}
959959

960960
if (ResultLoop && UnrollResult != LoopUnrollResult::FullyUnrolled)

0 commit comments

Comments
 (0)