@@ -97,6 +97,7 @@ namespace {
9797 bool eliminateIVUser (Instruction *UseInst, Instruction *IVOperand);
9898 bool makeIVComparisonInvariant (ICmpInst *ICmp, Instruction *IVOperand);
9999 void eliminateIVComparison (ICmpInst *ICmp, Instruction *IVOperand);
100+ bool forceEqualityForICmp (ICmpInst *ICmp, Instruction *IVOperand);
100101 void simplifyIVRemainder (BinaryOperator *Rem, Instruction *IVOperand,
101102 bool IsSigned);
102103 void replaceRemWithNumerator (BinaryOperator *Rem);
@@ -244,6 +245,128 @@ bool SimplifyIndvar::makeIVComparisonInvariant(ICmpInst *ICmp,
244245 return true ;
245246}
246247
248+ // / Try to change predicate of ICmp to EQ/NE to facilitate better work of OSR.
249+ // / This can be done only if all possible IV values but one lead to the same
250+ // / produced comparison result, while the 'chosen one' value gives the opposite
251+ // / result.
252+ bool SimplifyIndvar::forceEqualityForICmp (ICmpInst *ICmp,
253+ Instruction *IVOperand) {
254+ if (ICmp->isEquality ()) {
255+ // nothing to do
256+ return false ;
257+ }
258+
259+ unsigned BoundOperandIdx = IVOperand == ICmp->getOperand (0 ) ? 1 : 0 ;
260+ const SCEV *BoundSCEV = SE->getSCEV (ICmp->getOperand (BoundOperandIdx));
261+ const SCEVConstant *BoundC = dyn_cast<SCEVConstant>(BoundSCEV);
262+ CmpInst::Predicate OrigPredicate = ICmp->getPredicate ();
263+ CmpInst::Predicate NewPredicate = CmpInst::BAD_ICMP_PREDICATE;
264+ Type *Ty = IVOperand->getType ();
265+ APInt NewBoundA;
266+
267+ if (BoundC) {
268+ // Try to find the 'chosen one' value basing on predicate type and bound
269+ const APInt &BoundA = BoundC->getAPInt ();
270+ ConstantRange ExactCR =
271+ ConstantRange::makeExactICmpRegion (OrigPredicate, BoundA);
272+ if (!ExactCR.getEquivalentICmp (NewPredicate, NewBoundA)) {
273+ NewPredicate = CmpInst::BAD_ICMP_PREDICATE;
274+ }
275+ }
276+
277+ if (!ICmpInst::isEquality (NewPredicate)) {
278+ const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(SE->getSCEV (IVOperand));
279+ if (!AR) {
280+ return false ;
281+ }
282+ const SCEVConstant *IVStart = dyn_cast<SCEVConstant>(AR->getStart ());
283+ const SCEVConstant *IVStep =
284+ dyn_cast<SCEVConstant>(AR->getStepRecurrence (*SE));
285+ if (!IVStart || !IVStep || !IVStep->getValue ()->getValue ()) {
286+ return false ;
287+ }
288+
289+ if (BoundC) {
290+ // Check to see the 'chosen one' value is the IV start value
291+ bool HasNoWrap = ICmpInst::isSigned (OrigPredicate)
292+ ? AR->hasNoSignedWrap ()
293+ : AR->hasNoUnsignedWrap ();
294+ if (HasNoWrap) {
295+ const DataLayout &DL = ICmp->getParent ()->getDataLayout ();
296+ Constant *SecondIterIV =
297+ ConstantInt::get (Ty, IVStart->getAPInt () + IVStep->getAPInt ());
298+ Constant *FirstIterResult = ConstantFoldCompareInstOperands (
299+ OrigPredicate, IVStart->getValue (), BoundC->getValue (), DL);
300+ Constant *SecondIterResult = ConstantFoldCompareInstOperands (
301+ OrigPredicate, SecondIterIV, BoundC->getValue (), DL);
302+ if (FirstIterResult != SecondIterResult) {
303+ NewBoundA = IVStart->getAPInt ();
304+ NewPredicate = FirstIterResult->isAllOnesValue () ? CmpInst::ICMP_EQ
305+ : CmpInst::ICMP_NE;
306+ }
307+ }
308+ }
309+
310+ if (!ICmpInst::isEquality (NewPredicate)) {
311+ // Check to see the 'chosen one' value is the very last IV value.
312+ // To put it differently, check to see if ICmp directly or indirectly
313+ // defines maximum loop trip count (or simply has aligned behavior by
314+ // accident). This is different from loop exit condition rewriting as here
315+ // not only ICmp instructions directly writing to exiting branch are
316+ // considered.
317+
318+ // check to see if max trip count and IV parameters are constant
319+ const SCEVConstant *MaxBackCount =
320+ dyn_cast<SCEVConstant>(SE->getConstantMaxBackedgeTakenCount (L));
321+ if (!MaxBackCount) {
322+ return false ;
323+ }
324+
325+ // compute the number of consecutive iterations in which produced
326+ // predicate value will be the same
327+ bool ExitIfTrue = false ;
328+ auto EL = SE->computeExitLimitFromCond (L, ICmp, ExitIfTrue, false );
329+ const SCEVConstant *SameIterCount =
330+ dyn_cast<SCEVConstant>(EL.ExactNotTaken );
331+ if (!SameIterCount || SameIterCount->getValue ()->isZero ()) {
332+ ExitIfTrue = !ExitIfTrue;
333+ EL = SE->computeExitLimitFromCond (L, ICmp, ExitIfTrue, false );
334+ SameIterCount = dyn_cast<SCEVConstant>(EL.ExactNotTaken );
335+ }
336+
337+ if (SameIterCount != MaxBackCount) {
338+ // ICmp isn't aligned with maximum trip count
339+ return false ;
340+ }
341+
342+ unsigned IVBitWigth = IVStep->getAPInt ().getBitWidth ();
343+ unsigned CountBitWigth = SameIterCount->getAPInt ().getBitWidth ();
344+ APInt SameIterCountA = SameIterCount->getAPInt ();
345+ if (IVBitWigth < CountBitWigth) {
346+ SameIterCountA = SameIterCountA.trunc (IVBitWigth);
347+ } else if (IVBitWigth > CountBitWigth) {
348+ SameIterCountA = SameIterCountA.zext (IVBitWigth);
349+ }
350+ NewBoundA = IVStart->getAPInt () + (IVStep->getAPInt () * SameIterCountA);
351+ NewPredicate = ExitIfTrue ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
352+ }
353+ }
354+
355+ if (!TTI->isLegalICmpImmediate (NewBoundA.getSExtValue ())) {
356+ return false ;
357+ }
358+
359+ LLVM_DEBUG (dbgs () << " INDVARS: Force EQ/NE predicate for max trip count: "
360+ << *ICmp << ' \n '
361+
362+ assert (Ty->getPrimitiveSizeInBits () == NewBoundA.getBitWidth () &&
363+ " bit widths should be aligned"
364+ ICmp->setOperand (BoundOperandIdx, ConstantInt::get (Ty, NewBoundA));
365+ ICmp->setPredicate (NewPredicate);
366+
367+ return true ;
368+ }
369+
247370// / SimplifyIVUsers helper for eliminating useless
248371// / comparisons against an induction variable.
249372void SimplifyIndvar::eliminateIVComparison (ICmpInst *ICmp,
@@ -267,33 +390,43 @@ void SimplifyIndvar::eliminateIVComparison(ICmpInst *ICmp,
267390 // If the condition is always true or always false in the given context,
268391 // replace it with a constant value.
269392 SmallVector<Instruction *, 4 > Users;
393+ bool IsDead = false ;
270394 for (auto *U : ICmp->users ())
271395 Users.push_back (cast<Instruction>(U));
272396 const Instruction *CtxI = findCommonDominator (Users, *DT);
273397 if (auto Ev = SE->evaluatePredicateAt (Pred, S, X, CtxI)) {
274398 SE->forgetValue (ICmp);
275399 ICmp->replaceAllUsesWith (ConstantInt::getBool (ICmp->getContext (), *Ev));
276400 DeadInsts.emplace_back (ICmp);
401+ IsDead = true ;
277402 LLVM_DEBUG (dbgs () << " INDVARS: Eliminated comparison: " ' \n '
278403 } else if (makeIVComparisonInvariant (ICmp, IVOperand)) {
279- // fallthrough to end of function
280- } else if (ICmpInst::isSigned (OriginalPred) &&
281- SE->isKnownNonNegative (S) && SE->isKnownNonNegative (X)) {
404+ IsDead = true ;
405+ } else {
282406 // If we were unable to make anything above, all we can is to canonicalize
283407 // the comparison hoping that it will open the doors for other
284- // optimizations. If we find out that we compare two non-negative values,
285- // we turn the instruction's predicate to its unsigned version. Note that
286- // we cannot rely on Pred here unless we check if we have swapped it.
287- assert (ICmp->getPredicate () == OriginalPred && " Predicate changed?"
288- LLVM_DEBUG (dbgs () << " INDVARS: Turn to unsigned comparison: "
289- << ' \n '
290- ICmp->setPredicate (ICmpInst::getUnsignedPredicate (OriginalPred));
291- ICmp->setSameSign ();
292- } else
293- return ;
408+ // optimizations.
409+ if (ICmpInst::isSigned (OriginalPred) && SE->isKnownNonNegative (S) &&
410+ SE->isKnownNonNegative (X)) {
411+ // If we find out that we compare two non-negative values,
412+ // we turn the instruction's predicate to its unsigned version. Note that
413+ // we cannot rely on Pred here unless we check if we have swapped it.
414+ assert (ICmp->getPredicate () == OriginalPred && " Predicate changed?"
415+ LLVM_DEBUG (dbgs () << " INDVARS: Turn to unsigned comparison: "
416+ << ' \n '
417+ ICmp->setPredicate (ICmpInst::getUnsignedPredicate (OriginalPred));
418+ ICmp->setSameSign ();
419+ Changed = true ;
420+ }
421+ if (forceEqualityForICmp (ICmp, IVOperand)) {
422+ Changed = true ;
423+ }
424+ }
294425
295- ++NumElimCmp;
296- Changed = true ;
426+ if (IsDead) {
427+ NumElimCmp++;
428+ Changed = true ;
429+ }
297430}
298431
299432bool SimplifyIndvar::eliminateSDiv (BinaryOperator *SDiv) {
0 commit comments