[AMDGPU] misched: avoid subregister dependencies

llvm/include/llvm/CodeGen/TargetRegisterInfo.h

@@ -473,6 +473,28 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
     return false;
   }
 
+  /// Returns true if the two subregisters are equal or overlap.
+  /// The registers may be virtual registers.
+  bool subRegsOverlap(Register RegA, unsigned SubA, Register RegB,
+                      unsigned SubB) const {
+    if (RegA == RegB && SubA == SubB)
+      return true;
+    if (RegA.isVirtual() && RegB.isVirtual()) {
+      if (RegA != RegB)
+        return false;
+      LaneBitmask LA = getSubRegIndexLaneMask(SubA);
+      LaneBitmask LB = getSubRegIndexLaneMask(SubB);
+      return (LA & LB).any();
+    }
+    if (RegA.isPhysical() && RegB.isPhysical()) {
+      MCRegister MCRegA = SubA ? getSubReg(RegA, SubA) : RegA.asMCReg();
+      MCRegister MCRegB = SubB ? getSubReg(RegB, SubB) : RegB.asMCReg();
+      assert(MCRegB.isValid() && MCRegA.isValid() && "invalid subregister");
+      return MCRegisterInfo::regsOverlap(MCRegA, MCRegB);
+    }
+    return false;
+  }
+
   /// Returns true if Reg contains RegUnit.
   bool hasRegUnit(MCRegister Reg, MCRegUnit RegUnit) const {
     return llvm::is_contained(regunits(Reg), RegUnit);

llvm/lib/Target/AMDGPU/GCNSubtarget.cpp

@@ -627,6 +627,124 @@ GCNSubtarget::getMaxNumVectorRegs(const Function &F) const {
   return std::pair(MaxNumVGPRs, MaxNumAGPRs);
 }
 
+// Check to which source operand UseOpIdx points to and return a pointer to the
+// operand of the corresponding source modifier.
+// Return nullptr if UseOpIdx either doesn't point to src0/1/2 or if there is no
+// operand for the corresponding source modifier.
+static const MachineOperand *
+getVOP3PSourceModifierFromOpIdx(const MachineInstr *UseI, int UseOpIdx,
+                                const SIInstrInfo &InstrInfo) {
+  AMDGPU::OpName UseModName;
+  AMDGPU::OpName UseName =
+      AMDGPU::getOperandIdxName(UseI->getOpcode(), UseOpIdx);
+  switch (UseName) {
+  case AMDGPU::OpName::src0:
+    UseModName = AMDGPU::OpName::src0_modifiers;
+    break;
+  case AMDGPU::OpName::src1:
+    UseModName = AMDGPU::OpName::src1_modifiers;
+    break;
+  case AMDGPU::OpName::src2:
+    UseModName = AMDGPU::OpName::src2_modifiers;
+    break;
+  default:
+    return nullptr;
+  }
+  return InstrInfo.getNamedOperand(*UseI, UseModName);
+}
+
+// Get the subreg idx of the subreg that is used by the given instruction
+// operand, considering the given op_sel modifier.
+// Return 0 if the whole register is used or as a conservative fallback.
+static unsigned getEffectiveSubRegIdx(const SIRegisterInfo &TRI,
+                                      const SIInstrInfo &InstrInfo,
+                                      const MachineOperand &Op) {
+  const MachineInstr *I = Op.getParent();
+  if (!InstrInfo.isVOP3P(*I) || InstrInfo.isWMMA(*I) || InstrInfo.isSWMMAC(*I))
+    return 0;
+
+  const MachineOperand *OpMod =
+      getVOP3PSourceModifierFromOpIdx(I, Op.getOperandNo(), InstrInfo);
+  if (!OpMod)
+    return 0;
+
+  // Note: the FMA_MIX* and MAD_MIX* instructions have different semantics for
+  // the op_sel and op_sel_hi source modifiers:
+  // - op_sel: selects low/high operand bits as input to the operation;
+  //           has only meaning for 16-bit source operands
+  // - op_sel_hi: specifies the size of the source operands (16 or 32 bits);
+  //              a value of 0 indicates 32 bit, 1 indicates 16 bit
+  // For the other VOP3P instructions, the semantics are:
+  // - op_sel: selects low/high operand bits as input to the operation which
+  //           results in the lower-half of the destination
+  // - op_sel_hi: selects the low/high operand bits as input to the operation
+  //              which results in the higher-half of the destination
+  int64_t OpSel = OpMod->getImm() & SISrcMods::OP_SEL_0;
+  int64_t OpSelHi = OpMod->getImm() & SISrcMods::OP_SEL_1;
+
+  // Check if all parts of the register are being used (= op_sel and op_sel_hi
+  // differ for VOP3P or op_sel_hi=0 for VOP3PMix). In that case we can return
+  // early.
+  if ((!InstrInfo.isVOP3PMix(*I) && (!OpSel || !OpSelHi) &&
+       (OpSel || OpSelHi)) ||
+      (InstrInfo.isVOP3PMix(*I) && !OpSelHi))
+    return 0;
+
+  const MachineRegisterInfo &MRI = I->getParent()->getParent()->getRegInfo();
+  const TargetRegisterClass *RC = TRI.getRegClassForOperandReg(MRI, Op);
+
+  if (unsigned SubRegIdx = OpSel ? AMDGPU::sub1 : AMDGPU::sub0;
+      TRI.getSubClassWithSubReg(RC, SubRegIdx) == RC)
+    return SubRegIdx;
+  if (unsigned SubRegIdx = OpSel ? AMDGPU::hi16 : AMDGPU::lo16;
+      TRI.getSubClassWithSubReg(RC, SubRegIdx) == RC)
+    return SubRegIdx;
+
+  return 0;
+}
+
+Register GCNSubtarget::getRealSchedDependency(const MachineInstr *DefI,
+                                              int DefOpIdx,
+                                              const MachineInstr *UseI,
+                                              int UseOpIdx) const {
+  const SIRegisterInfo *TRI = getRegisterInfo();
+  const MachineOperand &DefOp = DefI->getOperand(DefOpIdx);
+  const MachineOperand &UseOp = UseI->getOperand(UseOpIdx);
+  Register DefReg = DefOp.getReg();
+  Register UseReg = UseOp.getReg();
+
+  // If the registers aren't restricted to a sub-register, there is no point in
+  // further analysis. This check makes only sense for virtual registers because
+  // physical registers may form a tuple and thus be part of a superregister
+  // although they are not a subregister themselves (vgpr0 is a "subreg" of
+  // vgpr0_vgpr1 without being a subreg in itself).
+  unsigned DefSubRegIdx = DefOp.getSubReg();
+  if (DefReg.isVirtual() && !DefSubRegIdx)
+    return DefReg;
+  unsigned UseSubRegIdx = getEffectiveSubRegIdx(*TRI, InstrInfo, UseOp);
+  if (UseReg.isVirtual() && !UseSubRegIdx)
+    return DefReg;
+
+  if (!TRI->subRegsOverlap(DefReg, DefSubRegIdx, UseReg, UseSubRegIdx))
+    return 0; // no real dependency
+
+  // UseReg might be smaller or larger than DefReg, depending on the subreg and
+  // on whether DefReg is a subreg, too. -> Find the smaller one.  This does not
+  // apply to virtual registers because we cannot construct a subreg for them.
+  if (DefReg.isVirtual())
+    return DefReg;
+  MCRegister DefMCReg =
+      DefSubRegIdx ? TRI->getSubReg(DefReg, DefSubRegIdx) : DefReg.asMCReg();
+  MCRegister UseMCReg =
+      UseSubRegIdx ? TRI->getSubReg(UseReg, UseSubRegIdx) : UseReg.asMCReg();
+  const TargetRegisterClass *DefRC = TRI->getPhysRegBaseClass(DefMCReg);
+  const TargetRegisterClass *UseRC = TRI->getPhysRegBaseClass(UseMCReg);
+  // Some registers, such as SGPR[0-9]+_HI16, do not have a register class.
+  if (!DefRC || !UseRC)
+    return DefReg;
+  return DefRC->hasSubClass(UseRC) ? UseMCReg : DefMCReg;
+}
+
 void GCNSubtarget::adjustSchedDependency(
     SUnit *Def, int DefOpIdx, SUnit *Use, int UseOpIdx, SDep &Dep,
     const TargetSchedModel *SchedModel) const {
@@ -637,6 +755,13 @@ void GCNSubtarget::adjustSchedDependency(
   MachineInstr *DefI = Def->getInstr();
   MachineInstr *UseI = Use->getInstr();
 
+  if (Register Reg = getRealSchedDependency(DefI, DefOpIdx, UseI, UseOpIdx)) {
+    Dep.setReg(Reg);
+  } else {
+    Dep = SDep(Def, SDep::Artificial);
+    return; // this is not a data dependency anymore
+  }
+
   if (DefI->isBundle()) {
     const SIRegisterInfo *TRI = getRegisterInfo();
     auto Reg = Dep.getReg();

llvm/lib/Target/AMDGPU/GCNSubtarget.h

@@ -299,6 +299,15 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   SITargetLowering TLInfo;
   SIFrameLowering FrameLowering;
 
+  /// Get the register that represents the actual dependency between the
+  /// definition and the use. The definition might only affect a subregister
+  /// that is not actually used. Works for both virtual and physical registers.
+  /// Note: Currently supports VOP3P instructions (without WMMA an SWMMAC).
+  /// Returns the definition register if there is a real dependency and no
+  /// better match is found.
+  Register getRealSchedDependency(const MachineInstr *DefI, int DefOpIdx,
+                                  const MachineInstr *UseI, int UseOpIdx) const;
+
 public:
   GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
                const GCNTargetMachine &TM);

llvm/lib/Target/AMDGPU/SIInstrInfo.h

@@ -909,6 +909,26 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
     return get(Opcode).TSFlags & SIInstrFlags::VOP3P;
   }
 
+  bool isVOP3PMix(const MachineInstr &MI) const {
+    return isVOP3PMix(MI.getOpcode());
+  }
+
+  bool isVOP3PMix(uint16_t Opcode) const {
+    if (!isVOP3P(Opcode))
+      return false;
+    switch (Opcode) {
+    case AMDGPU::V_FMA_MIXHI_F16:
+    case AMDGPU::V_FMA_MIXLO_F16:
+    case AMDGPU::V_FMA_MIX_F32:
+    case AMDGPU::V_MAD_MIXHI_F16:
+    case AMDGPU::V_MAD_MIXLO_F16:
+    case AMDGPU::V_MAD_MIX_F32:
+      return true;
+    default:
+      return false;
+    }
+  }
+
   static bool isVINTRP(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & SIInstrFlags::VINTRP;
   }

llvm/lib/Target/AMDGPU/VOP3PInstructions.td

@@ -358,8 +358,8 @@ let SubtargetPredicate = HasMadMixInsts in {
 let OtherPredicates = [NoFP32Denormals] in {
 
 // These are VOP3a-like opcodes which accept no omod.
-// Size of src arguments (16/32) is controlled by op_sel.
-// For 16-bit src arguments their location (hi/lo) are controlled by op_sel_hi.
+// Size of src arguments (16/32) is controlled by op_sel_hi.
+// For 16-bit src arguments their location (hi/lo) are controlled by op_sel.
 let isCommutable = 1, mayRaiseFPException = 0 in {
 let isReMaterializable = 1 in
 defm V_MAD_MIX_F32 : VOP3_VOP3PInst<"v_mad_mix_f32", VOP3P_Mix_Profile<VOP_F32_F16_F16_F16, VOP3_OPSEL>>;

llvm/test/CodeGen/AMDGPU/calling-conventions.ll

@@ -965,11 +965,11 @@ define amdgpu_ps void @ps_mesa_inreg_v5i32(<5 x i32> inreg %arg0) {
 ;
 ; GFX11-LABEL: ps_mesa_inreg_v5i32:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_add_i32 s3, s3, 4
-; GFX11-NEXT:    s_add_i32 s2, s2, 3
 ; GFX11-NEXT:    s_add_i32 s1, s1, 2
 ; GFX11-NEXT:    s_add_i32 s4, s4, 5
 ; GFX11-NEXT:    s_add_i32 s0, s0, 1
+; GFX11-NEXT:    s_add_i32 s3, s3, 4
+; GFX11-NEXT:    s_add_i32 s2, s2, 3
 ; GFX11-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v1, s1
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
 ; GFX11-NEXT:    v_mov_b32_e32 v2, s2
@@ -980,12 +980,11 @@ define amdgpu_ps void @ps_mesa_inreg_v5i32(<5 x i32> inreg %arg0) {
 ;
 ; GFX1250-LABEL: ps_mesa_inreg_v5i32:
 ; GFX1250:       ; %bb.0:
-; GFX1250-NEXT:    s_add_co_i32 s3, s3, 4
-; GFX1250-NEXT:    s_add_co_i32 s2, s2, 3
 ; GFX1250-NEXT:    s_add_co_i32 s1, s1, 2
 ; GFX1250-NEXT:    s_add_co_i32 s4, s4, 5
 ; GFX1250-NEXT:    s_add_co_i32 s0, s0, 1
-; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_add_co_i32 s3, s3, 4
+; GFX1250-NEXT:    s_add_co_i32 s2, s2, 3
 ; GFX1250-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v0, s0
 ; GFX1250-NEXT:    v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
 ; GFX1250-NEXT:    v_mov_b32_e32 v3, s3
@@ -1014,36 +1013,36 @@ define amdgpu_ps void @ps_mesa_inreg_v5f32(<5 x float> inreg %arg0) {
 ;
 ; VI-LABEL: ps_mesa_inreg_v5f32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_add_f32_e64 v3, s3, -1.0
-; VI-NEXT:    v_add_f32_e64 v2, s2, 4.0
 ; VI-NEXT:    v_add_f32_e64 v1, s1, 2.0
 ; VI-NEXT:    v_add_f32_e64 v0, s0, 1.0
 ; VI-NEXT:    v_add_f32_e64 v4, s4, 0.5
+; VI-NEXT:    v_add_f32_e64 v3, s3, -1.0
+; VI-NEXT:    v_add_f32_e64 v2, s2, 4.0
 ; VI-NEXT:    flat_store_dword v[0:1], v4
 ; VI-NEXT:    flat_store_dwordx4 v[0:1], v[0:3]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: ps_mesa_inreg_v5f32:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_add_f32_e64 v3, s3, -1.0
-; GFX11-NEXT:    v_add_f32_e64 v2, s2, 4.0
 ; GFX11-NEXT:    v_add_f32_e64 v1, s1, 2.0
 ; GFX11-NEXT:    v_add_f32_e64 v4, s4, 0.5
 ; GFX11-NEXT:    v_add_f32_e64 v0, s0, 1.0
+; GFX11-NEXT:    v_add_f32_e64 v3, s3, -1.0
+; GFX11-NEXT:    v_add_f32_e64 v2, s2, 4.0
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    global_store_b32 v[0:1], v4, off
 ; GFX11-NEXT:    global_store_b128 v[0:1], v[0:3], off
 ; GFX11-NEXT:    s_endpgm
 ;
 ; GFX1250-LABEL: ps_mesa_inreg_v5f32:
 ; GFX1250:       ; %bb.0:
-; GFX1250-NEXT:    s_add_f32 s3, s3, -1.0
 ; GFX1250-NEXT:    s_add_f32 s4, s4, 0.5
 ; GFX1250-NEXT:    s_add_f32 s0, s0, 1.0
 ; GFX1250-NEXT:    s_add_f32 s1, s1, 2.0
+; GFX1250-NEXT:    s_add_f32 s3, s3, -1.0
 ; GFX1250-NEXT:    s_add_f32 s2, s2, 4.0
-; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_2)
 ; GFX1250-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v0, s0
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_2)
 ; GFX1250-NEXT:    v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
 ; GFX1250-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX1250-NEXT:    s_clause 0x1
@@ -1148,32 +1147,32 @@ define amdgpu_ps void @ps_mesa_v5i32(<5 x i32> %arg0) {
 ;
 ; VI-LABEL: ps_mesa_v5i32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_add_u32_e32 v3, vcc, 4, v3
-; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, 2, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 1, v0
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 5, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 4, v3
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v2
 ; VI-NEXT:    flat_store_dword v[0:1], v4
 ; VI-NEXT:    flat_store_dwordx4 v[0:1], v[0:3]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: ps_mesa_v5i32:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_add_nc_u32_e32 v3, 4, v3
-; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
 ; GFX11-NEXT:    v_add_nc_u32_e32 v1, 2, v1
 ; GFX11-NEXT:    v_add_nc_u32_e32 v4, 5, v4
 ; GFX11-NEXT:    v_add_nc_u32_e32 v0, 1, v0
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 4, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 3, v2
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    global_store_b32 v[0:1], v4, off
 ; GFX11-NEXT:    global_store_b128 v[0:1], v[0:3], off
 ; GFX11-NEXT:    s_endpgm
 ;
 ; GFX1250-LABEL: ps_mesa_v5i32:
 ; GFX1250:       ; %bb.0:
-; GFX1250-NEXT:    v_dual_add_nc_u32 v3, 4, v3 :: v_dual_add_nc_u32 v2, 3, v2
 ; GFX1250-NEXT:    v_dual_add_nc_u32 v1, 2, v1 :: v_dual_add_nc_u32 v4, 5, v4
-; GFX1250-NEXT:    v_add_nc_u32_e32 v0, 1, v0
+; GFX1250-NEXT:    v_dual_add_nc_u32 v0, 1, v0 :: v_dual_add_nc_u32 v3, 4, v3
+; GFX1250-NEXT:    v_add_nc_u32_e32 v2, 3, v2
 ; GFX1250-NEXT:    s_clause 0x1
 ; GFX1250-NEXT:    global_store_b32 v[0:1], v4, off
 ; GFX1250-NEXT:    global_store_b128 v[0:1], v[0:3], off
@@ -1199,30 +1198,30 @@ define amdgpu_ps void @ps_mesa_v5f32(<5 x float> %arg0) {
 ;
 ; VI-LABEL: ps_mesa_v5f32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    v_add_f32_e32 v3, -1.0, v3
-; VI-NEXT:    v_add_f32_e32 v2, 4.0, v2
 ; VI-NEXT:    v_add_f32_e32 v1, 2.0, v1
 ; VI-NEXT:    v_add_f32_e32 v0, 1.0, v0
 ; VI-NEXT:    v_add_f32_e32 v4, 0.5, v4
+; VI-NEXT:    v_add_f32_e32 v3, -1.0, v3
+; VI-NEXT:    v_add_f32_e32 v2, 4.0, v2
 ; VI-NEXT:    flat_store_dword v[0:1], v4
 ; VI-NEXT:    flat_store_dwordx4 v[0:1], v[0:3]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: ps_mesa_v5f32:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_dual_add_f32 v3, -1.0, v3 :: v_dual_add_f32 v2, 4.0, v2
 ; GFX11-NEXT:    v_dual_add_f32 v1, 2.0, v1 :: v_dual_add_f32 v4, 0.5, v4
-; GFX11-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX11-NEXT:    v_dual_add_f32 v0, 1.0, v0 :: v_dual_add_f32 v3, -1.0, v3
+; GFX11-NEXT:    v_add_f32_e32 v2, 4.0, v2
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    global_store_b32 v[0:1], v4, off
 ; GFX11-NEXT:    global_store_b128 v[0:1], v[0:3], off
 ; GFX11-NEXT:    s_endpgm
 ;
 ; GFX1250-LABEL: ps_mesa_v5f32:
 ; GFX1250:       ; %bb.0:
-; GFX1250-NEXT:    v_dual_add_f32 v3, -1.0, v3 :: v_dual_add_f32 v2, 4.0, v2
 ; GFX1250-NEXT:    v_dual_add_f32 v1, 2.0, v1 :: v_dual_add_f32 v4, 0.5, v4
-; GFX1250-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GFX1250-NEXT:    v_dual_add_f32 v0, 1.0, v0 :: v_dual_add_f32 v3, -1.0, v3
+; GFX1250-NEXT:    v_add_f32_e32 v2, 4.0, v2
 ; GFX1250-NEXT:    s_clause 0x1
 ; GFX1250-NEXT:    global_store_b32 v[0:1], v4, off
 ; GFX1250-NEXT:    global_store_b128 v[0:1], v[0:3], off