Merge amd-gfx13 and amd-gfx into amd-gfx-gfx13

Author: piotrAMD

llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp

@@ -5707,6 +5707,15 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
       OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, SizeSrc2);
       break;
     }
+    case Intrinsic::amdgcn_waterfall_last_use_vgpr: {
+      unsigned SizeDst = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
+      unsigned SizeSrc1 = getSizeInBits(MI.getOperand(2).getReg(), MRI, *TRI);
+      unsigned SizeSrc2 = getSizeInBits(MI.getOperand(3).getReg(), MRI, *TRI);
+      OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SizeDst);
+      OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, SizeSrc1);
+      OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SizeSrc2);
+      break;
+    }
     case Intrinsic::amdgcn_ds_gws_init:
     case Intrinsic::amdgcn_ds_gws_barrier:
     case Intrinsic::amdgcn_ds_gws_sema_br: {

llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

@@ -924,9 +924,9 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
         PM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
       });
 
-  PB.registerOptimizerEarlyEPCallback([this](ModulePassManager &MPM,
-                                             OptimizationLevel Level,
-                                             ThinOrFullLTOPhase Phase) {
+  PB.registerOptimizerEarlyEPCallback([](ModulePassManager &MPM,
+                                         OptimizationLevel Level,
+                                         ThinOrFullLTOPhase Phase) {
     MPM.addPass(AMDGPURankSpecializationPass());
   });
 

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

@@ -6861,15 +6861,19 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
         .addReg(WaveIDInWaveGroup)
         .addImm(Rank);
     BuildMI(*BB, &MI, DL, TII->get(AMDGPU::S_CBRANCH_SCC0)).addMBB(SplitBB);
+
     // Call inside the conditional branch.
     Register CalleeAddrReg = MI.getOperand(1).getReg();
-    BuildMI(*RankCallBB, RankCallBB->end(), DL, TII->get(AMDGPU::S_SETPC_B64))
-        .addReg(CalleeAddrReg);
+    auto CalleeAddrDef = MRI.getVRegDef(CalleeAddrReg);
+    assert(CalleeAddrDef->getOpcode() == AMDGPU::SI_PC_ADD_REL_OFFSET64);
+    // Use s_add_pc_i64, bypass the address computation.
+    BuildMI(*RankCallBB, RankCallBB->end(), DL, TII->get(AMDGPU::S_ADD_PC_I64))
+        .addGlobalAddress(CalleeAddrDef->getOperand(1).getGlobal(), 0,
+                          SIInstrInfo::MO_REL64);
+
     // Update IDX0 for the next rank-call. Use the global address of the rank
     // callee as the source. In AsmPrinter, it will be replaced with the
     // MCSymbol representing the number of VGPRs of that callee.
-    auto CalleeAddrDef =  MRI.getVRegDef(CalleeAddrReg);
-    assert(CalleeAddrDef->getOpcode() == AMDGPU::SI_PC_ADD_REL_OFFSET64);
     BuildMI(*SplitBB, SplitBB->begin(), DL, TII->get(AMDGPU::S_ADD_GPR_IDX_U32),
             AMDGPU::IDX0)
         .addGlobalAddress(CalleeAddrDef->getOperand(1).getGlobal(), 0,

llvm/lib/Target/AMDGPU/SIInstructions.td

@@ -2551,57 +2551,44 @@ def : GCNPat <
                    (V_RCP_IFLAG_F32_e32 (V_CVT_F32_U32_e32 $src0))))
 >;
 
-multiclass SI_WATERFALL_Pattern < ValueType dvt, ValueType svt, string VecSize> {
-  def : GCNPat<
-    (i32 (int_amdgcn_waterfall_begin i32:$tok, svt:$idx)),
-    (!cast<Instruction>("SI_WATERFALL_BEGIN_"#VecSize) i32:$tok, svt:$idx)
-  >;
-
-  def : GCNPat<
-    (dvt (int_amdgcn_waterfall_readfirstlane i32:$tok, svt:$src)),
-    (!cast<Instruction>("SI_WATERFALL_READFIRSTLANE_"#VecSize) i32:$tok, svt:$src)
-  >;
-
-  def : GCNPat<
-    (svt (int_amdgcn_waterfall_end i32:$tok, svt:$src)),
-    (!cast<Instruction>("SI_WATERFALL_END_"#VecSize) i32:$tok, svt:$src)
-  >;
-}
-
-multiclass SI_WATERFALL_S_Pattern < ValueType svt, string VecSize> {
-  def : GCNPat<
-    (i32 (int_amdgcn_waterfall_begin i32:$tok, svt:$idx)),
-    (!cast<Instruction>("SI_WATERFALL_BEGIN_"#VecSize) i32:$tok, svt:$idx)
-  >;
-
-  def : GCNPat<
-    (svt (int_amdgcn_waterfall_readfirstlane i32:$tok, svt:$src)),
-    (!cast<Instruction>("SI_WATERFALL_READFIRSTLANE_"#VecSize) i32:$tok, svt:$src)
-  >;
-
-  def : GCNPat<
-    (svt (int_amdgcn_waterfall_end i32:$tok, svt:$src)),
-    (!cast<Instruction>("SI_WATERFALL_END_"#VecSize) i32:$tok, svt:$src)
-  >;
-
-  def : GCNPat<
-    (svt (int_amdgcn_waterfall_last_use i32:$tok, svt:$src)),
-    (!cast<Instruction>("SI_WATERFALL_LAST_USE_"#VecSize) i32:$tok, svt:$src)
-  >;
-  def : GCNPat<
-    (svt (int_amdgcn_waterfall_last_use_vgpr i32:$tok, svt:$src)),
-    (!cast<Instruction>("SI_WATERFALL_LAST_USE_"#VecSize#"_V") i32:$tok, svt:$src)
-  >;
+multiclass SI_WATERFALL_Pattern<ValueType dvt, ValueType svt, string VecSize> {
+  def : GCNPat<(dvt(int_amdgcn_waterfall_readfirstlane i32:$tok, svt:$src)),
+               (!cast<Instruction>("SI_WATERFALL_READFIRSTLANE_"#VecSize)
+                       i32:$tok,
+                   svt:$src)>;
+}
+
+multiclass SI_WATERFALL_S_Pattern<list<ValueType> vts, string VecSize> {
+  foreach vt = vts in {
+    def : GCNPat<(i32(int_amdgcn_waterfall_begin i32:$tok, vt:$idx)),
+                 (!cast<Instruction>("SI_WATERFALL_BEGIN_"#VecSize) i32:$tok,
+                     vt:$idx)>;
+
+    def : GCNPat<(vt(int_amdgcn_waterfall_readfirstlane i32:$tok, vt:$src)),
+                 (!cast<Instruction>("SI_WATERFALL_READFIRSTLANE_"#VecSize)
+                         i32:$tok,
+                     vt:$src)>;
+
+    def : GCNPat<(vt(int_amdgcn_waterfall_end i32:$tok, vt:$src)),
+                 (!cast<Instruction>("SI_WATERFALL_END_"#VecSize) i32:$tok,
+                     vt:$src)>;
+
+    def : GCNPat<(vt(int_amdgcn_waterfall_last_use i32:$tok, vt:$src)),
+                 (!cast<Instruction>("SI_WATERFALL_LAST_USE_"#VecSize) i32:$tok,
+                     vt:$src)>;
+
+    def : GCNPat<(vt(int_amdgcn_waterfall_last_use_vgpr i32:$tok, vt:$src)),
+                 (!cast<Instruction>("SI_WATERFALL_LAST_USE_"#VecSize#"_V")
+                         i32:$tok,
+                     vt:$src)>;
+  }
 }
 
-defm : SI_WATERFALL_S_Pattern <i16, "V1">;
-defm : SI_WATERFALL_S_Pattern <v2i16, "V1">;
-defm : SI_WATERFALL_S_Pattern <v4i16, "V2">;
-
-defm : SI_WATERFALL_S_Pattern <i32, "V1">;
-defm : SI_WATERFALL_S_Pattern <v2i32, "V2">;
-defm : SI_WATERFALL_S_Pattern <v4i32, "V4">;
-defm : SI_WATERFALL_S_Pattern <v8i32, "V8">;
+defm : SI_WATERFALL_S_Pattern<Reg16Types.types, "V1">;
+defm : SI_WATERFALL_S_Pattern<Reg32Types.types, "V1">;
+defm : SI_WATERFALL_S_Pattern<Reg64Types.types, "V2">;
+defm : SI_WATERFALL_S_Pattern<Reg128Types.types, "V4">;
+defm : SI_WATERFALL_S_Pattern<Reg256Types.types, "V8">;
 
 defm : SI_WATERFALL_Pattern <i16, f16, "V1">;
 defm : SI_WATERFALL_Pattern <v2i16, v2f16, "V1">;