AMDGPU: Expand remaining system atomic operations

System scope atomics need to use cmpxchg
loops. aea5980 started this, this
expands the set to cover the remaining integer operations.

Don't expand xchg and add, those theoretically should work over PCIe.

Author: arsenm

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

@@ -16018,7 +16018,8 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
       SSID == SyncScope::System ||
       SSID == RMW->getContext().getOrInsertSyncScopeID("one-as");
 
-  switch (RMW->getOperation()) {
+  auto Op = RMW->getOperation();
+  switch (Op) {
   case AtomicRMWInst::FAdd: {
     Type *Ty = RMW->getType();
 
@@ -16096,19 +16097,29 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
 
     return AtomicExpansionKind::CmpXChg;
   }
-  case AtomicRMWInst::FMin:
-  case AtomicRMWInst::FMax:
-  case AtomicRMWInst::Min:
+  case AtomicRMWInst::Xchg:
+  case AtomicRMWInst::Add:
+    // PCIe supports add and xchg for system atomics.
+    break;
+  case AtomicRMWInst::Sub:
+  case AtomicRMWInst::And:
+  case AtomicRMWInst::Or:
+  case AtomicRMWInst::Xor:
   case AtomicRMWInst::Max:
+  case AtomicRMWInst::Min:
+  case AtomicRMWInst::UMax:
   case AtomicRMWInst::UMin:
-  case AtomicRMWInst::UMax: {
+  case AtomicRMWInst::FMin:
+  case AtomicRMWInst::FMax:
+  case AtomicRMWInst::UIncWrap:
+  case AtomicRMWInst::UDecWrap: {
     if (AMDGPU::isFlatGlobalAddrSpace(AS) ||
         AS == AMDGPUAS::BUFFER_FAT_POINTER) {
-      if (RMW->getType()->isFloatTy() &&
+      if (AtomicRMWInst::isFPOperation(Op) &&
           unsafeFPAtomicsDisabled(RMW->getFunction()))
         return AtomicExpansionKind::CmpXChg;
 
-      // Always expand system scope min/max atomics.
+      // Always expand system scope atomics.
       if (HasSystemScope)
         return AtomicExpansionKind::CmpXChg;
     }