Merge branch 'release/rocm-rel-7.0' into amd/dev/chfang/cherry

Author: vamovsik

llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp

@@ -77,8 +77,6 @@ class LiveRegOptimizer {
   const GCNSubtarget &ST;
   /// The scalar type to convert to
   Type *const ConvertToScalar;
-  /// The set of visited Instructions
-  SmallPtrSet<Instruction *, 4> Visited;
   /// Map of Value -> Converted Value
   ValueToValueMap ValMap;
   /// Map of containing conversions from Optimal Type -> Original Type per BB.
@@ -248,6 +246,7 @@ bool LiveRegOptimizer::optimizeLiveType(
   SmallPtrSet<PHINode *, 4> PhiNodes;
   SmallPtrSet<Instruction *, 4> Defs;
   SmallPtrSet<Instruction *, 4> Uses;
+  SmallPtrSet<Instruction *, 4> Visited;
 
   Worklist.push_back(cast<Instruction>(I));
   while (!Worklist.empty()) {

llvm/lib/Target/AMDGPU/VOP1Instructions.td

@@ -391,8 +391,8 @@ def VOP_PERMLANE_SWAP : VOPProfile<[i32, i32, untyped, untyped]> {
   let HasExtDPP = 0;
   let HasExtSDWA = 0;
 
-  let Ins32 = (ins Src0RC64:$vdst_in, Src0RC32:$src0);
-  let Ins64 = (ins Src0RC64:$vdst_in, Src0RC64:$src0, Dpp16FI:$fi, DppBoundCtrl:$bound_ctrl);
+  let Ins32 = (ins DstRC:$vdst_in, Src0RC32:$src0);
+  let Ins64 = (ins DstRC64:$vdst_in, Src0RC64:$src0, Dpp16FI:$fi, DppBoundCtrl:$bound_ctrl);
   let InsVOP3OpSel = (ins Src0RC64:$vdst_in, Src0RC64:$src0, Dpp16FI:$fi, DppBoundCtrl:$bound_ctrl);
   let Asm64 = "$vdst, $src0$bound_ctrl$fi";
   let AsmVOP3OpSel = "$vdst, $src0$bound_ctrl$fi";

llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp

@@ -278,6 +278,15 @@ bool PointerReplacer::collectUsers() {
           Worklist.emplace_back(I);
   };
 
+  auto TryPushInstOperand = [&](Instruction *InstOp) {
+    if (!UsersToReplace.contains(InstOp)) {
+      if (!ValuesToRevisit.insert(InstOp))
+        return false;
+      Worklist.emplace_back(InstOp);
+    }
+    return true;
+  };
+
   PushUsersToWorklist(&Root);
   while (!Worklist.empty()) {
     Instruction *Inst = Worklist.pop_back_val();
@@ -310,21 +319,26 @@ bool PointerReplacer::collectUsers() {
       // incoming values.
       Worklist.emplace_back(PHI);
       for (unsigned Idx = 0; Idx < PHI->getNumIncomingValues(); ++Idx) {
-        auto *IncomingValue = cast<Instruction>(PHI->getIncomingValue(Idx));
-        if (UsersToReplace.contains(IncomingValue))
-          continue;
-        if (!ValuesToRevisit.insert(IncomingValue))
+        if (!TryPushInstOperand(cast<Instruction>(PHI->getIncomingValue(Idx))))
           return false;
-        Worklist.emplace_back(IncomingValue);
       }
     } else if (auto *SI = dyn_cast<SelectInst>(Inst)) {
       auto *TrueInst = dyn_cast<Instruction>(SI->getTrueValue());
       auto *FalseInst = dyn_cast<Instruction>(SI->getFalseValue());
       if (!TrueInst || !FalseInst)
         return false;
 
-      UsersToReplace.insert(SI);
-      PushUsersToWorklist(SI);
+      if (isAvailable(TrueInst) && isAvailable(FalseInst)) {
+        UsersToReplace.insert(SI);
+        PushUsersToWorklist(SI);
+        continue;
+      }
+
+      // Push select back onto the stack, followed by unavailable true/false
+      // value.
+      Worklist.emplace_back(SI);
+      if (!TryPushInstOperand(TrueInst) || !TryPushInstOperand(FalseInst))
+        return false;
     } else if (auto *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
       UsersToReplace.insert(GEP);
       PushUsersToWorklist(GEP);

llvm/lib/Transforms/Scalar/ConstraintElimination.cpp

@@ -313,8 +313,7 @@ class ConstraintInfo {
   /// New variables that need to be added to the system are collected in
   /// \p NewVariables.
   ConstraintTy getConstraint(CmpInst::Predicate Pred, Value *Op0, Value *Op1,
-                             SmallVectorImpl<Value *> &NewVariables,
-                             bool ForceSignedSystem = false) const;
+                             SmallVectorImpl<Value *> &NewVariables) const;
 
   /// Turns a comparison of the form \p Op0 \p Pred \p Op1 into a vector of
   /// constraints using getConstraint. Returns an empty constraint if the result
@@ -331,14 +330,6 @@ class ConstraintInfo {
   void transferToOtherSystem(CmpInst::Predicate Pred, Value *A, Value *B,
                              unsigned NumIn, unsigned NumOut,
                              SmallVectorImpl<StackEntry> &DFSInStack);
-
-private:
-  /// Adds facts into constraint system. \p ForceSignedSystem can be set when
-  /// the \p Pred is eq/ne, and signed constraint system is used when it's
-  /// specified.
-  void addFactImpl(CmpInst::Predicate Pred, Value *A, Value *B, unsigned NumIn,
-                   unsigned NumOut, SmallVectorImpl<StackEntry> &DFSInStack,
-                   bool ForceSignedSystem);
 };
 
 /// Represents a (Coefficient * Variable) entry after IR decomposition.
@@ -645,12 +636,8 @@ static Decomposition decompose(Value *V,
 
 ConstraintTy
 ConstraintInfo::getConstraint(CmpInst::Predicate Pred, Value *Op0, Value *Op1,
-                              SmallVectorImpl<Value *> &NewVariables,
-                              bool ForceSignedSystem) const {
+                              SmallVectorImpl<Value *> &NewVariables) const {
   assert(NewVariables.empty() && "NewVariables must be empty when passed in");
-  assert((!ForceSignedSystem || CmpInst::isEquality(Pred)) &&
-         "signed system can only be forced on eq/ne");
-
   bool IsEq = false;
   bool IsNe = false;
 
@@ -665,15 +652,15 @@ ConstraintInfo::getConstraint(CmpInst::Predicate Pred, Value *Op0, Value *Op1,
     break;
   }
   case CmpInst::ICMP_EQ:
-    if (!ForceSignedSystem && match(Op1, m_Zero())) {
+    if (match(Op1, m_Zero())) {
       Pred = CmpInst::ICMP_ULE;
     } else {
       IsEq = true;
       Pred = CmpInst::ICMP_ULE;
     }
     break;
   case CmpInst::ICMP_NE:
-    if (!ForceSignedSystem && match(Op1, m_Zero())) {
+    if (match(Op1, m_Zero())) {
       Pred = CmpInst::getSwappedPredicate(CmpInst::ICMP_UGT);
       std::swap(Op0, Op1);
     } else {
@@ -690,7 +677,7 @@ ConstraintInfo::getConstraint(CmpInst::Predicate Pred, Value *Op0, Value *Op1,
     return {};
 
   SmallVector<ConditionTy, 4> Preconditions;
-  bool IsSigned = ForceSignedSystem || CmpInst::isSigned(Pred);
+  bool IsSigned = CmpInst::isSigned(Pred);
   auto &Value2Index = getValue2Index(IsSigned);
   auto ADec = decompose(Op0->stripPointerCastsSameRepresentation(),
                         Preconditions, IsSigned, DL);
@@ -750,7 +737,7 @@ ConstraintInfo::getConstraint(CmpInst::Predicate Pred, Value *Op0, Value *Op1,
   int64_t OffsetSum;
   if (AddOverflow(Offset1, Offset2, OffsetSum))
     return {};
-  if (Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_ULT)
+  if (Pred == (IsSigned ? CmpInst::ICMP_SLT : CmpInst::ICMP_ULT))
     if (AddOverflow(OffsetSum, int64_t(-1), OffsetSum))
       return {};
   R[0] = OffsetSum;
@@ -1593,20 +1580,10 @@ static bool checkOrAndOpImpliedByOther(
 void ConstraintInfo::addFact(CmpInst::Predicate Pred, Value *A, Value *B,
                              unsigned NumIn, unsigned NumOut,
                              SmallVectorImpl<StackEntry> &DFSInStack) {
-  addFactImpl(Pred, A, B, NumIn, NumOut, DFSInStack, false);
-  // If the Pred is eq/ne, also add the fact to signed system.
-  if (CmpInst::isEquality(Pred))
-    addFactImpl(Pred, A, B, NumIn, NumOut, DFSInStack, true);
-}
-
-void ConstraintInfo::addFactImpl(CmpInst::Predicate Pred, Value *A, Value *B,
-                                 unsigned NumIn, unsigned NumOut,
-                                 SmallVectorImpl<StackEntry> &DFSInStack,
-                                 bool ForceSignedSystem) {
   // If the constraint has a pre-condition, skip the constraint if it does not
   // hold.
   SmallVector<Value *> NewVariables;
-  auto R = getConstraint(Pred, A, B, NewVariables, ForceSignedSystem);
+  auto R = getConstraint(Pred, A, B, NewVariables);
 
   // TODO: Support non-equality for facts as well.
   if (!R.isValid(*this) || R.isNe())

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

@@ -12799,25 +12799,47 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
   InstructionCost SpillCost = getSpillCost();
   Cost += SpillCost + ExtractCost;
   auto &&ResizeToVF = [this, &Cost](const TreeEntry *TE, ArrayRef<int> Mask,
-                                    bool) {
+                                    bool ForSingleMask) {
     InstructionCost C = 0;
     unsigned VF = Mask.size();
     unsigned VecVF = TE->getVectorFactor();
-    if (VF != VecVF &&
-        (any_of(Mask, [VF](int Idx) { return Idx >= static_cast<int>(VF); }) ||
-         !ShuffleVectorInst::isIdentityMask(Mask, VF))) {
-      SmallVector<int> OrigMask(VecVF, PoisonMaskElem);
-      std::copy(Mask.begin(), std::next(Mask.begin(), std::min(VF, VecVF)),
-                OrigMask.begin());
-      C = ::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
-                           getWidenedType(TE->getMainOp()->getType(), VecVF),
-                           OrigMask);
-      LLVM_DEBUG(
-          dbgs() << "SLP: Adding cost " << C
-                 << " for final shuffle of insertelement external users.\n";
-          TE->dump(); dbgs() << "SLP: Current total cost = " << Cost << "\n");
-      Cost += C;
-      return std::make_pair(TE, true);
+    bool HasLargeIndex =
+        any_of(Mask, [VF](int Idx) { return Idx >= static_cast<int>(VF); });
+    if ((VF != VecVF && HasLargeIndex) ||
+        !ShuffleVectorInst::isIdentityMask(Mask, VF)) {
+
+      if (HasLargeIndex) {
+        SmallVector<int> OrigMask(VecVF, PoisonMaskElem);
+        std::copy(Mask.begin(), std::next(Mask.begin(), std::min(VF, VecVF)),
+                  OrigMask.begin());
+        C = ::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
+                             getWidenedType(TE->getMainOp()->getType(), VecVF),
+                             OrigMask);
+        LLVM_DEBUG(
+            dbgs() << "SLP: Adding cost " << C
+                   << " for final shuffle of insertelement external users.\n";
+            TE->dump(); dbgs() << "SLP: Current total cost = " << Cost << "\n");
+        Cost += C;
+        return std::make_pair(TE, true);
+      }
+
+      if (!ForSingleMask) {
+        SmallVector<int> ResizeMask(VF, PoisonMaskElem);
+        for (unsigned I = 0; I < VF; ++I) {
+          if (Mask[I] != PoisonMaskElem)
+            ResizeMask[Mask[I]] = Mask[I];
+        }
+        if (!ShuffleVectorInst::isIdentityMask(ResizeMask, VF))
+          C = ::getShuffleCost(
+              *TTI, TTI::SK_PermuteSingleSrc,
+              getWidenedType(TE->getMainOp()->getType(), VecVF), ResizeMask);
+        LLVM_DEBUG(
+            dbgs() << "SLP: Adding cost " << C
+                   << " for final shuffle of insertelement external users.\n";
+            TE->dump(); dbgs() << "SLP: Current total cost = " << Cost << "\n");
+
+        Cost += C;
+      }
     }
     return std::make_pair(TE, false);
   };

llvm/test/CodeGen/AMDGPU/amdgpu-late-codegenprepare-crash-splat.ll

@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-late-codegenprepare %s  | FileCheck %s
+
+; This crashed because the PHI with a splat was rejected, but then we marked the PHI
+; as visited and tried to convert one of its user afterwards.
+
+define amdgpu_kernel void @widget(ptr %arg, ptr %arg1, ptr %arg2) {
+; CHECK-LABEL: define amdgpu_kernel void @widget(
+; CHECK-SAME: ptr [[ARG:%.*]], ptr [[ARG1:%.*]], ptr [[ARG2:%.*]]) {
+; CHECK-NEXT:  [[BB:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARG]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[TMP0]] to i1
+; CHECK-NEXT:    [[ARG1_LOAD:%.*]] = load <4 x i8>, ptr [[ARG1]], align 4
+; CHECK-NEXT:    [[ARG2_LOAD:%.*]] = load i64, ptr [[ARG2]], align 4
+; CHECK-NEXT:    br label %[[BB_1:.*]]
+; CHECK:       [[BB_1]]:
+; CHECK-NEXT:    [[PHI:%.*]] = phi ptr [ null, %[[BB]] ], [ [[ARG1]], %[[BB_6:.*]] ]
+; CHECK-NEXT:    [[PHI4:%.*]] = phi <4 x i8> [ splat (i8 1), %[[BB]] ], [ [[PHI15:%.*]], %[[BB_6]] ]
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[BB_2:.*]], label %[[BB_6]]
+; CHECK:       [[BB_2]]:
+; CHECK-NEXT:    [[PHI7:%.*]] = phi <4 x i8> [ [[PHI13:%.*]], %[[BB_5:.*]] ], [ [[PHI4]], %[[BB_1]] ]
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[BB_4:.*]], label %[[BB_5]]
+; CHECK:       [[BB_3:.*]]:
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[BB_4]], label %[[BB_EXIT:.*]]
+; CHECK:       [[BB_4]]:
+; CHECK-NEXT:    [[PHI11:%.*]] = phi <4 x i8> [ [[PHI7]], %[[BB_3]] ], [ zeroinitializer, %[[BB_2]] ]
+; CHECK-NEXT:    store <4 x i8> [[PHI11]], ptr [[PHI]], align 1
+; CHECK-NEXT:    br label %[[BB_5]]
+; CHECK:       [[BB_5]]:
+; CHECK-NEXT:    [[PHI13]] = phi <4 x i8> [ zeroinitializer, %[[BB_4]] ], [ [[PHI7]], %[[BB_2]] ]
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[BB_2]], label %[[BB_6]]
+; CHECK:       [[BB_6]]:
+; CHECK-NEXT:    [[PHI15]] = phi <4 x i8> [ [[ARG1_LOAD]], %[[BB_1]] ], [ zeroinitializer, %[[BB_5]] ]
+; CHECK-NEXT:    br label %[[BB_1]]
+; CHECK:       [[BB_EXIT]]:
+; CHECK-NEXT:    ret void
+;
+bb:
+  %ld = load i32, ptr %arg, align 4
+  %ld.trunc = trunc i32 %ld to i1
+  %arg1.load = load <4 x i8>, ptr %arg1, align 4
+  %arg2.load = load i64, ptr %arg2, align 4
+  br label %bb.1
+
+bb.1:
+  %phi = phi ptr [ null, %bb ], [ %arg1, %bb.6 ]
+  %phi4 = phi <4 x i8> [ splat (i8 1), %bb ], [ %phi15, %bb.6 ]
+  br i1 %ld.trunc, label %bb.2, label %bb.6
+
+bb.2:
+  %phi7 = phi <4 x i8> [ %phi13, %bb.5 ], [ %phi4, %bb.1 ]
+  br i1 %ld.trunc, label %bb.4, label %bb.5
+
+bb.3:
+  br i1 %ld.trunc, label %bb.4, label %bb.exit
+
+bb.4:
+  %phi11 = phi <4 x i8> [ %phi7, %bb.3 ], [ zeroinitializer, %bb.2 ]
+  store <4 x i8> %phi11, ptr %phi, align 1
+  br label %bb.5
+
+bb.5:
+  %phi13 = phi <4 x i8> [ zeroinitializer, %bb.4 ], [ %phi7, %bb.2 ]
+  br i1 %ld.trunc, label %bb.2, label %bb.6
+
+bb.6:
+  %phi15 = phi <4 x i8> [ %arg1.load, %bb.1 ], [ zeroinitializer, %bb.5 ]
+  br label %bb.1
+
+bb.exit:
+  ret void
+}