[LoopVersioning] Invalidate SCEV for phi if new values are added.

After 20d798b, SCEV looks through PHIs with a single incoming
value. This means adding a new incoming value may change the SCEV for a
phi. Add missing invalidation when an existing PHI is reused during
LoopVersioning. New incoming values will be added later from the
versioned loop.

Similar issues have been fixed by also adding missing invalidation.

Fixes #57825.

Note that the test case unfortunately requires running loop-vectorize
followed by loop-load-elimination, which does the actual versioning. I
don't think it is possible to reproduce the failure without that
combination.

(cherry picked from commit 623c4a7)

Author: fhahn

llvm/lib/Transforms/Utils/LoopVersioning.cpp

@@ -137,8 +137,10 @@ void LoopVersioning::addPHINodes(
     // See if we have a single-operand PHI with the value defined by the
     // original loop.
     for (auto I = PHIBlock->begin(); (PN = dyn_cast<PHINode>(I)); ++I) {
-      if (PN->getIncomingValue(0) == Inst)
+      if (PN->getIncomingValue(0) == Inst) {
+        SE->forgetValue(PN);
         break;
+      }
     }
     // If not create it.
     if (!PN) {

llvm/test/Transforms/LoopLoadElim/versioning-scev-invalidation.ll

@@ -0,0 +1,125 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -force-vector-width=4 -force-vector-interleave=1 -passes='loop-vectorize,loop-load-elim' -S %s | FileCheck %s
+
+@glob.1 = external global [100 x double]
+@glob.2 = external global [100 x double]
+
+define void @g(ptr %dst.1, ptr %start, i64 %N) {
+; CHECK-LABEL: @g(
+; CHECK-NEXT:  loop.1.lver.check:
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[N:%.*]], 3
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 16
+; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, ptr @glob.2, i64 [[TMP1]]
+; CHECK-NEXT:    [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[DST_1:%.*]], i64 8
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr @glob.2, [[UGLYGEP2]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[DST_1]], [[UGLYGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[LOOP_1_PH_LVER_ORIG:%.*]], label [[LOOP_1_PH:%.*]]
+; CHECK:       loop.1.ph.lver.orig:
+; CHECK-NEXT:    br label [[LOOP_1_LVER_ORIG:%.*]]
+; CHECK:       loop.1.lver.orig:
+; CHECK-NEXT:    [[IV_LVER_ORIG:%.*]] = phi i64 [ 0, [[LOOP_1_PH_LVER_ORIG]] ], [ [[IV_NEXT_LVER_ORIG:%.*]], [[LOOP_1_LVER_ORIG]] ]
+; CHECK-NEXT:    [[PTR_IV_1_LVER_ORIG:%.*]] = phi ptr [ @glob.1, [[LOOP_1_PH_LVER_ORIG]] ], [ [[PTR_IV_1_NEXT_LVER_ORIG:%.*]], [[LOOP_1_LVER_ORIG]] ]
+; CHECK-NEXT:    [[GEP_IV_LVER_ORIG:%.*]] = getelementptr inbounds double, ptr @glob.2, i64 [[IV_LVER_ORIG]]
+; CHECK-NEXT:    [[L_1_LVER_ORIG:%.*]] = load double, ptr [[GEP_IV_LVER_ORIG]], align 8
+; CHECK-NEXT:    [[GEP_IV_1_LVER_ORIG:%.*]] = getelementptr inbounds double, ptr getelementptr inbounds (double, ptr @glob.2, i64 1), i64 [[IV_LVER_ORIG]]
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[GEP_IV_1_LVER_ORIG]], align 8
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[DST_1]], align 8
+; CHECK-NEXT:    [[PTR_IV_1_NEXT_LVER_ORIG]] = getelementptr inbounds double, ptr [[PTR_IV_1_LVER_ORIG]], i64 1
+; CHECK-NEXT:    [[IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[IV_LVER_ORIG]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT_LVER_ORIG:%.*]] = icmp eq i64 [[IV_LVER_ORIG]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT_LVER_ORIG]], label [[LOOP_2_PH_LOOPEXIT:%.*]], label [[LOOP_1_LVER_ORIG]]
+; CHECK:       loop.1.ph:
+; CHECK-NEXT:    [[LOAD_INITIAL:%.*]] = load double, ptr @glob.2, align 8
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop.1:
+; CHECK-NEXT:    [[STORE_FORWARDED:%.*]] = phi double [ [[LOAD_INITIAL]], [[LOOP_1_PH]] ], [ 0.000000e+00, [[LOOP_1]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[LOOP_1_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_1]] ]
+; CHECK-NEXT:    [[PTR_IV_1:%.*]] = phi ptr [ @glob.1, [[LOOP_1_PH]] ], [ [[PTR_IV_1_NEXT:%.*]], [[LOOP_1]] ]
+; CHECK-NEXT:    [[GEP_IV:%.*]] = getelementptr inbounds double, ptr @glob.2, i64 [[IV]]
+; CHECK-NEXT:    [[L_1:%.*]] = load double, ptr [[GEP_IV]], align 8
+; CHECK-NEXT:    [[GEP_IV_1:%.*]] = getelementptr inbounds double, ptr getelementptr inbounds (double, ptr @glob.2, i64 1), i64 [[IV]]
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[GEP_IV_1]], align 8
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[DST_1]], align 8
+; CHECK-NEXT:    [[PTR_IV_1_NEXT]] = getelementptr inbounds double, ptr [[PTR_IV_1]], i64 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[LOOP_2_PH_LOOPEXIT3:%.*]], label [[LOOP_1]]
+; CHECK:       loop.2.ph.loopexit:
+; CHECK-NEXT:    [[LCSSA_PTR_IV_1_PH:%.*]] = phi ptr [ [[PTR_IV_1_LVER_ORIG]], [[LOOP_1_LVER_ORIG]] ]
+; CHECK-NEXT:    br label [[LOOP_2_PH:%.*]]
+; CHECK:       loop.2.ph.loopexit3:
+; CHECK-NEXT:    [[LCSSA_PTR_IV_1_PH4:%.*]] = phi ptr [ [[PTR_IV_1]], [[LOOP_1]] ]
+; CHECK-NEXT:    br label [[LOOP_2_PH]]
+; CHECK:       loop.2.ph:
+; CHECK-NEXT:    [[LCSSA_PTR_IV_1:%.*]] = phi ptr [ [[LCSSA_PTR_IV_1_PH]], [[LOOP_2_PH_LOOPEXIT]] ], [ [[LCSSA_PTR_IV_1_PH4]], [[LOOP_2_PH_LOOPEXIT3]] ]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[N_VEC]], 8
+; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[LCSSA_PTR_IV_1]], i64 [[TMP2]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[LCSSA_PTR_IV_1]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr double, ptr [[NEXT_GEP]], i32 0
+; CHECK-NEXT:    store <4 x double> zeroinitializer, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_2_PH]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[LCSSA_PTR_IV_1]], [[LOOP_2_PH]] ]
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop.2:
+; CHECK-NEXT:    [[IV_2:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], [[LOOP_2]] ]
+; CHECK-NEXT:    [[PTR_IV_2:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[PTR_IV_2_NEXT:%.*]], [[LOOP_2]] ]
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[PTR_IV_2]], align 8
+; CHECK-NEXT:    [[PTR_IV_2_NEXT]] = getelementptr inbounds double, ptr [[PTR_IV_2]], i64 1
+; CHECK-NEXT:    [[IV_2_NEXT]] = add nuw nsw i64 [[IV_2]], 1
+; CHECK-NEXT:    [[EXITCOND_1_NOT:%.*]] = icmp eq i64 [[IV_2_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_1_NOT]], label [[EXIT_LOOPEXIT:%.*]], label [[LOOP_2]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.1
+
+loop.1:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.1 ]
+  %ptr.iv.1 = phi ptr [ @glob.1, %entry ], [ %ptr.iv.1.next, %loop.1 ]
+  %gep.iv = getelementptr inbounds double, ptr @glob.2, i64 %iv
+  %l.1 = load double, ptr %gep.iv, align 8
+  %gep.iv.1 = getelementptr inbounds double, ptr getelementptr inbounds (double, ptr @glob.2, i64 1), i64 %iv
+  store double 0.000000e+00, ptr %gep.iv.1, align 8
+  store double 0.000000e+00, ptr %dst.1, align 8
+  %ptr.iv.1.next = getelementptr inbounds double, ptr %ptr.iv.1, i64 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv, %N
+  br i1 %exitcond.not, label %loop.2.ph, label %loop.1
+
+loop.2.ph:
+  %lcssa.ptr.iv.1 = phi ptr [ %ptr.iv.1, %loop.1 ]
+  br label %loop.2
+
+loop.2:
+  %iv.2 = phi i64 [ 0, %loop.2.ph ] , [ %iv.2.next, %loop.2 ]
+  %ptr.iv.2 = phi ptr [ %lcssa.ptr.iv.1, %loop.2.ph ], [ %ptr.iv.2.next, %loop.2 ]
+  store double 0.000000e+00, ptr %ptr.iv.2, align 8
+  %ptr.iv.2.next = getelementptr inbounds double, ptr %ptr.iv.2, i64 1
+  %iv.2.next = add nuw nsw i64 %iv.2, 1
+  %exitcond.1.not = icmp eq i64 %iv.2.next, %N
+  br i1 %exitcond.1.not, label %exit, label %loop.2
+
+exit:
+  ret void
+}