[AMDGPU] Insert s_wait_xcnt(0) before atomics to work around write-combining miss hazard

This patch adds a workaround for a hazzard on GFX1250, which inserts an `s_wait_xcnt(0)` instruction before any atomic operation that might write to memory.

Fixes SWDEV-543703.

Author: shiltian

llvm/lib/Target/AMDGPU/AMDGPU.td

@@ -895,6 +895,12 @@ def FeatureCvtFP8VOP1Bug : SubtargetFeature<"cvt-fp8-vop1-bug",
   [FeatureFP8ConversionInsts]
 >;
 
+def FeatureWriteCombiningMissesHazards : SubtargetFeature<"write-combining-misses-hazards",
+  "HasWriteCombiningMissesHazards",
+  "true",
+  "Write combining misses hazards that require s_wait_cnt(0) before every atomic operation"
+>;
+
 def FeaturePkFmacF16Inst : SubtargetFeature<"pk-fmac-f16-inst",
   "HasPkFmacF16Inst",
   "true",
@@ -2145,6 +2151,7 @@ def FeatureISAVersion12_50 : FeatureSet<
    FeatureXNACK,
    FeatureClusters,
    FeatureD16Writes32BitVgpr,
+   FeatureWriteCombiningMissesHazards,
 ]>;
 
 def FeatureISAVersion12_51 : FeatureSet<
@@ -2945,6 +2952,8 @@ def HasGWS : Predicate<"Subtarget->hasGWS()">;
 def HasCvtFP8VOP1Bug : Predicate<"Subtarget->hasCvtFP8VOP1Bug()">;
 def HasNoCvtFP8VOP1Bug : Predicate<"!Subtarget->hasCvtFP8VOP1Bug()">;
 
+def HasWriteCombiningMissesHazards : Predicate<"Subtarget->hasWriteCombiningMissesHazards()">;
+
 def HasAtomicCSubNoRtnInsts : Predicate<"Subtarget->hasAtomicCSubNoRtnInsts()">;
 
 def HasScalarDwordx3Loads : Predicate<"Subtarget->hasScalarDwordx3Loads()">;

llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp

@@ -1269,6 +1269,8 @@ void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
     fixScratchBaseForwardingHazard(MI);
   if (ST.setRegModeNeedsVNOPs())
     fixSetRegMode(MI);
+  if (ST.hasWriteCombiningMissesHazards())
+    fixWriteCombiningMissesHazards(MI);
 }
 
 static bool isVCmpXWritesExec(const SIInstrInfo &TII, const SIRegisterInfo &TRI,
@@ -2177,6 +2179,29 @@ bool GCNHazardRecognizer::fixWMMACoexecutionHazards(MachineInstr *MI) {
   return true;
 }
 
+/// This function inserts an s_wait_cnt(0) before every atomic store/RMW
+/// operation to work around the write combining hazard.
+bool GCNHazardRecognizer::fixWriteCombiningMissesHazards(MachineInstr *MI) {
+  if (!SIInstrInfo::isAtomic(*MI) || !MI->mayStore())
+    return false;
+
+  // If the previous instruction is an s_wait_xcnt, and the count is 0, we don't
+  // need to do anything.
+  MachineBasicBlock &MBB = *MI->getParent();
+  auto Itr = MachineBasicBlock::iterator(MI);
+  auto PrevItr = std::prev(Itr);
+  if (Itr != MBB.begin() && (PrevItr->getOpcode() == AMDGPU::S_WAIT_XCNT_soft ||
+                             PrevItr->getOpcode() == AMDGPU::S_WAIT_XCNT)) {
+    int64_t Cnt = PrevItr->getOperand(0).getImm();
+    if (Cnt == 0)
+      return false;
+  }
+
+  BuildMI(MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::S_WAIT_XCNT_soft))
+      .addImm(0);
+  return true;
+}
+
 bool GCNHazardRecognizer::fixShift64HighRegBug(MachineInstr *MI) {
   if (!ST.hasShift64HighRegBug())
     return false;

llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h

@@ -114,6 +114,7 @@ class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
   bool fixDsAtomicAsyncBarrierArriveB64(MachineInstr *MI);
   bool fixScratchBaseForwardingHazard(MachineInstr *MI);
   bool fixSetRegMode(MachineInstr *MI);
+  bool fixWriteCombiningMissesHazards(MachineInstr *MI);
 
   int checkMAIHazards(MachineInstr *MI);
   int checkMAIHazards908(MachineInstr *MI);

llvm/lib/Target/AMDGPU/GCNSubtarget.h

@@ -282,7 +282,7 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   bool HasPointSampleAccel = false;
   bool HasLdsBarrierArriveAtomic = false;
   bool HasSetPrioIncWgInst = false;
-
+  bool HasWriteCombiningMissesHazards = false;
   bool RequiresCOV6 = false;
   bool UseBlockVGPROpsForCSR = false;
   bool HasGloballyAddressableScratch = false;
@@ -1836,6 +1836,10 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
     return getGeneration() == GFX12;
   }
 
+  bool hasWriteCombiningMissesHazards() const {
+    return HasWriteCombiningMissesHazards;
+  }
+
   // Requires s_wait_alu(0) after s102/s103 write and src_flat_scratch_base
   // read.
   bool hasScratchBaseForwardingHazard() const {