[SLP] Recalculate cleared deps for potential control schedule data nodes

Need to recalculate the dependencies for all potential control data
schedule nodes to prevent compiler crash.

Fixes #153571

Author: alexey-bataev

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

@@ -20796,6 +20796,10 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
                         << BB->getName() << "\n");
       calculateDependencies(Bundle, /*InsertInReadyList=*/!ReSchedule, SLP,
                             ControlDependentMembers);
+    } else if (!ControlDependentMembers.empty()) {
+      ScheduleBundle Invalid = ScheduleBundle::invalid();
+      calculateDependencies(Invalid, /*InsertInReadyList=*/!ReSchedule, SLP,
+                            ControlDependentMembers);
     }
 
     if (ReSchedule) {
@@ -20891,6 +20895,7 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
       }
     }
     ScheduledBundlesList.pop_back();
+    SmallVector<ScheduleData *> ControlDependentMembers;
     for (Value *V : VL) {
       if (S.isNonSchedulable(V))
         continue;
@@ -20930,11 +20935,20 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
           ScheduleCopyableDataMapByUsers.erase(I);
         ScheduleCopyableDataMap.erase(KV);
         // Need to recalculate dependencies for the actual schedule data.
-        if (ScheduleData *OpSD = getScheduleData(I))
+        if (ScheduleData *OpSD = getScheduleData(I)) {
           OpSD->clearDirectDependencies();
+          if (RegionHasStackSave ||
+              !isGuaranteedToTransferExecutionToSuccessor(OpSD->getInst()))
+            ControlDependentMembers.push_back(OpSD);
+        }
         continue;
       }
       ScheduledBundles.find(I)->getSecond().pop_back();
+      if (!ControlDependentMembers.empty()) {
+        ScheduleBundle Invalid = ScheduleBundle::invalid();
+        calculateDependencies(Invalid, /*InsertInReadyList=*/false, SLP,
+                              ControlDependentMembers);
+      }
     }
     return std::nullopt;
   }
@@ -21304,7 +21318,10 @@ void BoUpSLP::BlockScheduling::calculateDependencies(
     }
   };
 
-  WorkList.push_back(Bundle.getBundle().front());
+  assert((Bundle || !ControlDeps.empty()) &&
+         "expected at least one instruction to schedule");
+  if (Bundle)
+    WorkList.push_back(Bundle.getBundle().front());
   WorkList.append(ControlDeps.begin(), ControlDeps.end());
   SmallPtrSet<ScheduleBundle *, 16> Visited;
   while (!WorkList.empty()) {

llvm/test/Transforms/SLPVectorizer/X86/control-deps-schedule-data-recalculate.ll

@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define i32 @test(i32 %mul) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: i32 [[MUL:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[H:%.*]] = alloca [4 x i32], align 16
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 0, 0
+; CHECK-NEXT:    [[ADD4:%.*]] = add i32 [[ADD]], 0
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @f1(i32 [[ADD4]])
+; CHECK-NEXT:    [[MUL1:%.*]] = shl i32 0, 1
+; CHECK-NEXT:    [[ADD5:%.*]] = add i32 [[CALL]], [[MUL1]]
+; CHECK-NEXT:    store i32 [[ADD5]], ptr [[H]], align 16
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr i8, ptr [[H]], i64 4
+; CHECK-NEXT:    [[ADD6:%.*]] = add i32 0, 0
+; CHECK-NEXT:    [[ADD7:%.*]] = add i32 [[ADD6]], [[MUL]]
+; CHECK-NEXT:    [[ADD9:%.*]] = add i32 [[ADD7]], [[ADD4]]
+; CHECK-NEXT:    store i32 [[ADD9]], ptr [[ARRAYINIT_ELEMENT]], align 4
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT10:%.*]] = getelementptr i8, ptr [[H]], i64 8
+; CHECK-NEXT:    [[ADD11:%.*]] = or i32 [[ADD]], 0
+; CHECK-NEXT:    [[ADD12:%.*]] = add i32 [[ADD11]], [[ADD4]]
+; CHECK-NEXT:    store i32 [[ADD12]], ptr [[ARRAYINIT_ELEMENT10]], align 8
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT13:%.*]] = getelementptr i8, ptr [[H]], i64 12
+; CHECK-NEXT:    store i32 0, ptr [[ARRAYINIT_ELEMENT13]], align 4
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %h = alloca [4 x i32], align 16
+  %add = add i32 0, 0
+  %add4 = add i32 %add, 0
+  %call = tail call i32 @f1(i32 %add4)
+  %mul1 = shl i32 0, 1
+  %add5 = add i32 %call, %mul1
+  store i32 %add5, ptr %h, align 16
+  %arrayinit.element = getelementptr i8, ptr %h, i64 4
+  %add6 = add i32 0, 0
+  %add7 = add i32 %add6, %mul
+  %add9 = add i32 %add7, %add4
+  store i32 %add9, ptr %arrayinit.element, align 4
+  %arrayinit.element10 = getelementptr i8, ptr %h, i64 8
+  %add11 = or i32 %add, 0
+  %add12 = add i32 %add11, %add4
+  store i32 %add12, ptr %arrayinit.element10, align 8
+  %arrayinit.element13 = getelementptr i8, ptr %h, i64 12
+  store i32 0, ptr %arrayinit.element13, align 4
+  ret i32 0
+}
+
+declare i32 @f1(i32)
+