[AMDGPU] CodeGen for 64-bit buffer atomic cmpswap intrinsics (#70475)

Implement codegen for:
llvm.amdgcn.raw.buffer.atomic.cmpswap.i64
llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i64
llvm.amdgcn.struct.buffer.atomic.cmpswap.i64
llvm.amdgcn.struct.ptr.buffer.atomic.cmpswap.i64

Author: jayfoad

llvm/lib/Target/AMDGPU/BUFInstructions.td

@@ -1659,73 +1659,76 @@ let SubtargetPredicate = isGFX90APlus in {
   defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">;
 } // End SubtargetPredicate = isGFX90APlus
 
-multiclass SIBufferAtomicCmpSwapPat<string Inst> {
-
+multiclass SIBufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> {
   foreach RtnMode = ["ret", "noret"] in {
-
     defvar Op = !cast<SDPatternOperator>(SIbuffer_atomic_cmpswap
                                          # !if(!eq(RtnMode, "ret"), "", "_noret"));
     defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", "");
     defvar CachePolicy = !if(!eq(RtnMode, "ret"), (set_glc $cachepolicy),
       (timm:$cachepolicy));
+    defvar SrcRC = getVregSrcForVT<vt>.ret;
+    defvar DataRC = getVregSrcForVT<data_vt>.ret;
+    defvar SubLo = !if(!eq(vt, i32), sub0, sub0_sub1);
+    defvar SubHi = !if(!eq(vt, i32), sub1, sub2_sub3);
 
     defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix)
-      (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1),
+      (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi),
       SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy);
     def : GCNPat<
-      (Op
-          i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset,
-          timm:$offset, timm:$cachepolicy, 0),
+      (vt (Op
+          vt:$data, vt:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset,
+          timm:$offset, timm:$cachepolicy, 0)),
       !if(!eq(RtnMode, "ret"),
-        (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffsetResDag, VReg_64)), sub0),
+        (EXTRACT_SUBREG OffsetResDag, SubLo),
         OffsetResDag)
     >;
 
     defvar IdxenResDag = (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix)
-      (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1),
+      (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi),
       VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,
       CachePolicy);
     def : GCNPat<
-      (Op
-          i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
+      (vt (Op
+          vt:$data, vt:$cmp, v4i32:$rsrc, i32:$vindex,
           0, i32:$soffset, timm:$offset,
-          timm:$cachepolicy, timm),
+          timm:$cachepolicy, timm)),
       !if(!eq(RtnMode, "ret"),
-        (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS IdxenResDag, VReg_64)), sub0),
+        (EXTRACT_SUBREG IdxenResDag, SubLo),
         IdxenResDag)
     >;
 
     defvar OffenResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix)
-      (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1),
+      (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi),
       VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,
       CachePolicy);
     def : GCNPat<
-      (Op
-          i32:$data, i32:$cmp, v4i32:$rsrc, 0,
+      (vt (Op
+          vt:$data, vt:$cmp, v4i32:$rsrc, 0,
           i32:$voffset, i32:$soffset, timm:$offset,
-          timm:$cachepolicy, 0),
+          timm:$cachepolicy, 0)),
       !if(!eq(RtnMode, "ret"),
-        (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffenResDag, VReg_64)), sub0),
+        (EXTRACT_SUBREG OffenResDag, SubLo),
         OffenResDag)
     >;
 
     defvar BothenResDag = (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix)
-      (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1),
+      (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi),
       (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
       SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy);
     def : GCNPat<
-      (Op
-          i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
+      (vt (Op
+          vt:$data, vt:$cmp, v4i32:$rsrc, i32:$vindex,
           i32:$voffset, i32:$soffset, timm:$offset,
-          timm:$cachepolicy, timm),
+          timm:$cachepolicy, timm)),
       !if(!eq(RtnMode, "ret"),
-        (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS BothenResDag, VReg_64)), sub0),
+        (EXTRACT_SUBREG BothenResDag, SubLo),
         BothenResDag)
     >;
   } // end foreach RtnMode
 }
 
-defm : SIBufferAtomicCmpSwapPat<"BUFFER_ATOMIC_CMPSWAP">;
+defm : SIBufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">;
+defm : SIBufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">;
 
 class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
                               PatFrag constant_ld> : GCNPat <

llvm/lib/Target/AMDGPU/SIInstrInfo.td

@@ -199,10 +199,7 @@ defm SIbuffer_atomic_fmax : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_FMAX">;
 
 def SIbuffer_atomic_cmpswap : SDNode <"AMDGPUISD::BUFFER_ATOMIC_CMPSWAP",
   SDTypeProfile<1, 9,
-    [SDTCisVT<0, i32>,   // dst
-     SDTCisVT<1, i32>,   // src
-     SDTCisVT<2, i32>,   // cmp
-     SDTCisVT<3, v4i32>, // rsrc
+    [SDTCisVT<3, v4i32>, // rsrc
      SDTCisVT<4, i32>,   // vindex(VGPR)
      SDTCisVT<5, i32>,   // voffset(VGPR)
      SDTCisVT<6, i32>,   // soffset(SGPR)