!fixup Undo SCEV changes, change SCEV invalidation order in LV.

Author: fhahn

llvm/lib/Analysis/ScalarEvolution.cpp

@@ -8441,7 +8441,7 @@ ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
   // recusive call to getBackedgeTakenInfo (on a different
   // loop), which would invalidate the iterator computed
   // earlier.
-  return BackedgeTakenCounts[L] = std::move(Result);
+  return BackedgeTakenCounts.find(L)->second = std::move(Result);
 }
 
 void ScalarEvolution::forgetAllLoops() {
@@ -9866,10 +9866,8 @@ const SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) {
   for (auto &LS : reverse(ValuesAtScopes[V]))
     if (LS.first == L) {
       LS.second = C;
-      if (!isa<SCEVConstant>(C)) {
+      if (!isa<SCEVConstant>(C))
         ValuesAtScopesUsers[C].push_back({L, V});
-        SCEVUsers[V].insert(C);
-      }
       break;
     }
   return C;
@@ -14202,8 +14200,7 @@ void ScalarEvolution::forgetBackedgeTakenCounts(const Loop *L,
       for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
         if (!isa<SCEVConstant>(S)) {
           auto UserIt = BECountUsers.find(S);
-          if (UserIt == BECountUsers.end())
-            continue;
+          assert(UserIt != BECountUsers.end());
           UserIt->second.erase({L, Predicated});
         }
       }

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -2928,10 +2928,6 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
   if (EnableVPlanNativePath)
     fixNonInductionPHIs(State);
 
-  // Forget the original basic block.
-  PSE.getSE()->forgetLoop(OrigLoop);
-  PSE.getSE()->forgetBlockAndLoopDispositions();
-
   // After vectorization, the exit blocks of the original loop will have
   // additional predecessors. Invalidate SCEVs for the exit phis in case SE
   // looked through single-entry phis.
@@ -2941,6 +2937,10 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
     for (PHINode &PN : Exit->phis())
       PSE.getSE()->forgetLcssaPhiWithNewPredecessor(OrigLoop, &PN);
 
+  // Forget the original basic block.
+  PSE.getSE()->forgetLoop(OrigLoop);
+  PSE.getSE()->forgetBlockAndLoopDispositions();
+
   // Don't apply optimizations below when no vector region remains, as they all
   // require a vector loop at the moment.
   if (!State.Plan->getVectorLoopRegion())