[RISCV][GlobalISel] Legalize and select G_ATOMICRMW_ADD instruction (llvm#153791)

This patch adds legalization and instruction selection support for the
`G_ATOMICRMW_ADD` opcode in the RISCV GlobalISel backend. Support for
other opcodes will be added in subsequent PRs.

Co-authored-by: Kane Wang <[email protected]>

Author: ReVe1uv

llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp

@@ -26,6 +26,8 @@
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsRISCV.h"
 #include "llvm/IR/Type.h"
 
 using namespace llvm;
@@ -692,6 +694,11 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
       .customIf(all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),
                     typeIsLegalIntOrFPVec(1, IntOrFPVecTys, ST)));
 
+  getActionDefinitionsBuilder(G_ATOMICRMW_ADD)
+      .legalFor(ST.hasStdExtA(), {{sXLen, p0}})
+      .libcallFor(!ST.hasStdExtA(), {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
+      .clampScalar(0, sXLen, sXLen);
+
   getLegacyLegalizerInfo().computeTables();
   verify(*ST.getInstrInfo());
 }
@@ -729,6 +736,8 @@ bool RISCVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
     MI.eraseFromParent();
     return true;
   }
+  case Intrinsic::riscv_masked_atomicrmw_add:
+    return true;
   }
 }
 

llvm/test/CodeGen/RISCV/GlobalISel/atomicrmw-add.ll

@@ -0,0 +1,299 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+a,+zabha -global-isel -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32IA-ZABHA
+; RUN: llc -mtriple=riscv32 -mattr=+a -global-isel -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32IA
+; RUN: llc -mtriple=riscv32 -global-isel -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32
+; RUN: llc -mtriple=riscv64 -mattr=+a,+zabha -global-isel -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64IA-ZABHA
+; RUN: llc -mtriple=riscv64 -mattr=+a -global-isel -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64IA
+; RUN: llc -mtriple=riscv64 -global-isel -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64
+
+define i8 @atomicrmw_add_i8(ptr %ptr, i8 %rhs) {
+; RV32IA-ZABHA-LABEL: atomicrmw_add_i8:
+; RV32IA-ZABHA:       # %bb.0:
+; RV32IA-ZABHA-NEXT:    amoadd.b.aqrl a0, a1, (a0)
+; RV32IA-ZABHA-NEXT:    ret
+;
+; RV32IA-LABEL: atomicrmw_add_i8:
+; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    li a2, 255
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    andi a0, a0, 3
+; RV32IA-NEXT:    zext.b a1, a1
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    sll a2, a2, a0
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    lr.w.aqrl a4, (a3)
+; RV32IA-NEXT:    add a5, a4, a1
+; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    and a5, a5, a2
+; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a3)
+; RV32IA-NEXT:    bnez a5, .LBB0_1
+; RV32IA-NEXT:  # %bb.2:
+; RV32IA-NEXT:    srl a0, a4, a0
+; RV32IA-NEXT:    ret
+;
+; RV32-LABEL: atomicrmw_add_i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    .cfi_offset ra, -4
+; RV32-NEXT:    li a2, 5
+; RV32-NEXT:    call __atomic_fetch_add_1
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    .cfi_restore ra
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_add_i8:
+; RV64IA-ZABHA:       # %bb.0:
+; RV64IA-ZABHA-NEXT:    amoadd.b.aqrl a0, a1, (a0)
+; RV64IA-ZABHA-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i8:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    li a2, 255
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    andi a0, a0, 3
+; RV64IA-NEXT:    zext.b a1, a1
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    sllw a2, a2, a0
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a3)
+; RV64IA-NEXT:    add a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a3)
+; RV64IA-NEXT:    bnez a5, .LBB0_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a0
+; RV64IA-NEXT:    ret
+;
+; RV64-LABEL: atomicrmw_add_i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    .cfi_offset ra, -8
+; RV64-NEXT:    li a2, 5
+; RV64-NEXT:    call __atomic_fetch_add_1
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    .cfi_restore ra
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-NEXT:    ret
+  %res = atomicrmw add ptr %ptr, i8 %rhs seq_cst
+  ret i8 %res
+}
+
+define i16 @atomicrmw_add_i16(ptr %ptr, i16 %rhs) {
+; RV32IA-ZABHA-LABEL: atomicrmw_add_i16:
+; RV32IA-ZABHA:       # %bb.0:
+; RV32IA-ZABHA-NEXT:    amoadd.h.aqrl a0, a1, (a0)
+; RV32IA-ZABHA-NEXT:    ret
+;
+; RV32IA-LABEL: atomicrmw_add_i16:
+; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    lui a2, 16
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    andi a0, a0, 3
+; RV32IA-NEXT:    addi a2, a2, -1
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    sll a4, a2, a0
+; RV32IA-NEXT:    and a1, a1, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV32IA-NEXT:    add a5, a2, a1
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a3)
+; RV32IA-NEXT:    bnez a5, .LBB1_1
+; RV32IA-NEXT:  # %bb.2:
+; RV32IA-NEXT:    srl a0, a2, a0
+; RV32IA-NEXT:    ret
+;
+; RV32-LABEL: atomicrmw_add_i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    .cfi_offset ra, -4
+; RV32-NEXT:    li a2, 5
+; RV32-NEXT:    call __atomic_fetch_add_2
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    .cfi_restore ra
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_add_i16:
+; RV64IA-ZABHA:       # %bb.0:
+; RV64IA-ZABHA-NEXT:    amoadd.h.aqrl a0, a1, (a0)
+; RV64IA-ZABHA-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i16:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    andi a0, a0, 3
+; RV64IA-NEXT:    addi a2, a2, -1
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    sllw a4, a2, a0
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV64IA-NEXT:    add a5, a2, a1
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a3)
+; RV64IA-NEXT:    bnez a5, .LBB1_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a2, a0
+; RV64IA-NEXT:    ret
+;
+; RV64-LABEL: atomicrmw_add_i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    .cfi_offset ra, -8
+; RV64-NEXT:    li a2, 5
+; RV64-NEXT:    call __atomic_fetch_add_2
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    .cfi_restore ra
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-NEXT:    ret
+  %res = atomicrmw add ptr %ptr, i16 %rhs seq_cst
+  ret i16 %res
+}
+
+define i32 @atomicrmw_add_i32(ptr %ptr, i32 %rhs) {
+; RV32IA-ZABHA-LABEL: atomicrmw_add_i32:
+; RV32IA-ZABHA:       # %bb.0:
+; RV32IA-ZABHA-NEXT:    amoadd.w.aqrl a0, a1, (a0)
+; RV32IA-ZABHA-NEXT:    ret
+;
+; RV32IA-LABEL: atomicrmw_add_i32:
+; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    amoadd.w.aqrl a0, a1, (a0)
+; RV32IA-NEXT:    ret
+;
+; RV32-LABEL: atomicrmw_add_i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    .cfi_offset ra, -4
+; RV32-NEXT:    li a2, 5
+; RV32-NEXT:    call __atomic_fetch_add_4
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    .cfi_restore ra
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_add_i32:
+; RV64IA-ZABHA:       # %bb.0:
+; RV64IA-ZABHA-NEXT:    amoadd.w.aqrl a0, a1, (a0)
+; RV64IA-ZABHA-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i32:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoadd.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
+;
+; RV64-LABEL: atomicrmw_add_i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    .cfi_offset ra, -8
+; RV64-NEXT:    li a2, 5
+; RV64-NEXT:    call __atomic_fetch_add_4
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    .cfi_restore ra
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-NEXT:    ret
+  %res = atomicrmw add ptr %ptr, i32 %rhs seq_cst
+  ret i32 %res
+}
+
+define i64 @atomicrmw_add_i64(ptr %ptr, i64 %rhs) {
+; RV32IA-ZABHA-LABEL: atomicrmw_add_i64:
+; RV32IA-ZABHA:       # %bb.0:
+; RV32IA-ZABHA-NEXT:    addi sp, sp, -16
+; RV32IA-ZABHA-NEXT:    .cfi_def_cfa_offset 16
+; RV32IA-ZABHA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IA-ZABHA-NEXT:    .cfi_offset ra, -4
+; RV32IA-ZABHA-NEXT:    li a3, 5
+; RV32IA-ZABHA-NEXT:    call __atomic_fetch_add_8
+; RV32IA-ZABHA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IA-ZABHA-NEXT:    .cfi_restore ra
+; RV32IA-ZABHA-NEXT:    addi sp, sp, 16
+; RV32IA-ZABHA-NEXT:    .cfi_def_cfa_offset 0
+; RV32IA-ZABHA-NEXT:    ret
+;
+; RV32IA-LABEL: atomicrmw_add_i64:
+; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    addi sp, sp, -16
+; RV32IA-NEXT:    .cfi_def_cfa_offset 16
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    .cfi_offset ra, -4
+; RV32IA-NEXT:    li a3, 5
+; RV32IA-NEXT:    call __atomic_fetch_add_8
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    .cfi_restore ra
+; RV32IA-NEXT:    addi sp, sp, 16
+; RV32IA-NEXT:    .cfi_def_cfa_offset 0
+; RV32IA-NEXT:    ret
+;
+; RV32-LABEL: atomicrmw_add_i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    .cfi_offset ra, -4
+; RV32-NEXT:    li a3, 5
+; RV32-NEXT:    call __atomic_fetch_add_8
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    .cfi_restore ra
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_add_i64:
+; RV64IA-ZABHA:       # %bb.0:
+; RV64IA-ZABHA-NEXT:    amoadd.d.aqrl a0, a1, (a0)
+; RV64IA-ZABHA-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i64:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoadd.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
+;
+; RV64-LABEL: atomicrmw_add_i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    .cfi_offset ra, -8
+; RV64-NEXT:    li a2, 5
+; RV64-NEXT:    call __atomic_fetch_add_8
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    .cfi_restore ra
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-NEXT:    ret
+  %res = atomicrmw add ptr %ptr, i64 %rhs seq_cst
+  ret i64 %res
+}