Provide and use new TII method

Change-Id: I4cbb79b168f451e2decd0775657dafba0243faab

Author: jrbyrnes

llvm/docs/AMDGPUUsage.rst

@@ -1347,7 +1347,7 @@ The AMDGPU backend implements the following LLVM IR intrinsics.
                                                    - 0x0100: All DS read instructions may be scheduled accoss sched_barrier.
                                                    - 0x0200: All DS write instructions may be scheduled across sched_barrier.
                                                    - 0x0400: All Transcendental (e.g. V_EXP) instructions may be scheduled across sched_barrier.
-                                                   - 0x0800: All Packed Arithmetic (e.g. V_PK_MOV, V_DOT, etc) instructions may be scheduled across sched_barrier.
+                                                   - 0x0800: All "Never-Coissuable" (e.g. V_PK_ADD, V_DOT, etc) instructions may be scheduled across sched_barrier.
 
   llvm.amdgcn.sched.group.barrier                  Creates schedule groups with specific properties to create custom scheduling
                                                    pipelines. The ordering between groups is enforced by the instruction scheduler.

llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp

@@ -75,9 +75,9 @@ enum class SchedGroupMask {
   DS_READ = 1u << 8,
   DS_WRITE = 1u << 9,
   TRANS = 1u << 10,
-  PACK = 1u << 11,
+  NONCOISSUE = 1u << 11,
   ALL = ALU | VALU | SALU | MFMA | VMEM | VMEM_READ | VMEM_WRITE | DS |
-        DS_READ | DS_WRITE | TRANS | PACK,
+        DS_READ | DS_WRITE | TRANS | NONCOISSUE,
   LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ ALL)
 };
 
@@ -2457,8 +2457,9 @@ bool SchedGroup::canAddMI(const MachineInstr &MI) const {
            TII->isTRANS(MI))
     Result = true;
 
-  else if (((SGMask & SchedGroupMask::PACK) != SchedGroupMask::NONE) &&
-           TII->isVOP3P(MI) && !TII->isMFMAorWMMA(MI))
+  else if (((SGMask & SchedGroupMask::NONCOISSUE) != SchedGroupMask::NONE) &&
+           TII->isNeverCoissue(MI) && !TII->isMFMAorWMMA(MI) &&
+           !TII->isTRANS(MI))
     Result = true;
 
   LLVM_DEBUG(
@@ -2640,17 +2641,17 @@ IGroupLPDAGMutation::invertSchedBarrierMask(SchedGroupMask Mask) const {
   // allowed past the SCHED_BARRIER.
   SchedGroupMask InvertedMask = ~Mask;
 
-  // ALU implies VALU, SALU, MFMA, TRANS, PACK.
+  // ALU implies VALU, SALU, MFMA, TRANS, NONCOISSUE.
   if ((InvertedMask & SchedGroupMask::ALU) == SchedGroupMask::NONE)
     InvertedMask &= ~SchedGroupMask::VALU & ~SchedGroupMask::SALU &
                     ~SchedGroupMask::MFMA & ~SchedGroupMask::TRANS &
-                    ~SchedGroupMask::PACK;
-  // VALU, SALU, MFMA, TRANS, PACK implies ALU.
+                    ~SchedGroupMask::NONCOISSUE;
+  // VALU, SALU, MFMA, TRANS, NONCOISSUE implies ALU.
   else if ((InvertedMask & SchedGroupMask::VALU) == SchedGroupMask::NONE ||
            (InvertedMask & SchedGroupMask::SALU) == SchedGroupMask::NONE ||
            (InvertedMask & SchedGroupMask::MFMA) == SchedGroupMask::NONE ||
            (InvertedMask & SchedGroupMask::TRANS) == SchedGroupMask::NONE ||
-           (InvertedMask & SchedGroupMask::PACK) == SchedGroupMask::NONE)
+           (InvertedMask & SchedGroupMask::NONCOISSUE) == SchedGroupMask::NONE)
     InvertedMask &= ~SchedGroupMask::ALU;
 
   // VMEM implies VMEM_READ, VMEM_WRITE.

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

@@ -8957,6 +8957,64 @@ uint64_t SIInstrInfo::getScratchRsrcWords23() const {
   return Rsrc23;
 }
 
+bool SIInstrInfo::isNeverCoissue(const MachineInstr &MI) const {
+  bool IsGFX942Only = ST.hasGFX940Insts() && !ST.hasGFX950Insts();
+  if (!IsGFX942Only)
+    return false;
+
+  if (!isVALU(MI))
+    return false;
+
+  // V_COS, V_EXP, V_RCP, etc.
+  if (isTRANS(MI))
+    return true;
+
+  // DOT2, DOT2C, DOT4, etc.
+  if (isDOT(MI))
+    return true;
+
+  // MFMA, SMFMA
+  if (isMFMA(MI))
+    return true;
+
+  unsigned Opcode = MI.getOpcode();
+  switch (Opcode) {
+  case AMDGPU::V_CVT_PK_BF8_F32_e64:
+  case AMDGPU::V_CVT_PK_FP8_F32_e64:
+  case AMDGPU::V_MQSAD_PK_U16_U8_e64:
+  case AMDGPU::V_MQSAD_U32_U8_e64:
+  case AMDGPU::V_PK_ADD_F16:
+  case AMDGPU::V_PK_ADD_F32:
+  case AMDGPU::V_PK_ADD_I16:
+  case AMDGPU::V_PK_ADD_U16:
+  case AMDGPU::V_PK_ASHRREV_I16:
+  case AMDGPU::V_PK_FMA_F16:
+  case AMDGPU::V_PK_FMA_F32:
+  case AMDGPU::V_PK_FMAC_F16_e32:
+  case AMDGPU::V_PK_FMAC_F16_e64:
+  case AMDGPU::V_PK_LSHLREV_B16:
+  case AMDGPU::V_PK_LSHRREV_B16:
+  case AMDGPU::V_PK_MAD_I16:
+  case AMDGPU::V_PK_MAD_U16:
+  case AMDGPU::V_PK_MAX_F16:
+  case AMDGPU::V_PK_MAX_I16:
+  case AMDGPU::V_PK_MAX_U16:
+  case AMDGPU::V_PK_MIN_F16:
+  case AMDGPU::V_PK_MIN_I16:
+  case AMDGPU::V_PK_MIN_U16:
+  case AMDGPU::V_PK_MOV_B32:
+  case AMDGPU::V_PK_MUL_F16:
+  case AMDGPU::V_PK_MUL_F32:
+  case AMDGPU::V_PK_MUL_LO_U16:
+  case AMDGPU::V_PK_SUB_I16:
+  case AMDGPU::V_PK_SUB_U16:
+  case AMDGPU::V_QSAD_PK_U16_U8_e64:
+    return true;
+  default:
+    return false;
+  }
+}
+
 bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const {
   unsigned Opc = MI.getOpcode();
 

llvm/lib/Target/AMDGPU/SIInstrInfo.h

@@ -1031,6 +1031,8 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
     }
   }
 
+  bool isNeverCoissue(const MachineInstr &MI) const;
+
   bool isVGPRCopy(const MachineInstr &MI) const {
     assert(isCopyInstr(MI));
     Register Dest = MI.getOperand(0).getReg();