Adding a reg key to disable coalescing memory fences selectively

This change is to extend an opportunity to control over the SynchronizationObjectCoalescing pass. This allows to determine which fences cannot be processed by the pass based on the memory they synchronize.

Author: krystian-andrzejewski

IGC/Compiler/Optimizer/SynchronizationObjectCoalescing.cpp

@@ -69,6 +69,18 @@ enum InstructionMask : uint32_t
     LastMaskPlusOne,
 };
 
+////////////////////////////////////////////////////////////////////////////////
+enum class SyncInstMask : uint32_t
+{
+    None = 0x0,
+    UntypedMemoryFence = (1 << 0),
+    TypedMemoryFence = (1 << 1),
+    SharedMemoryFence = (1 << 2),
+    UrbMemoryFence = (1 << 3),
+    ThreadGroupBarrier = (1 << 4)
+};
+
+
 constexpr InstructionMask AllInstructionsMask =
     InstructionMask{ ((InstructionMask::LastMaskPlusOne - 1) << 1) - 1 };
 
@@ -84,6 +96,15 @@ inline constexpr InstructionMask& operator|=(InstructionMask& a, InstructionMask
     a = a | b;
     return a;
 }
+inline constexpr SyncInstMask operator|(SyncInstMask a, SyncInstMask b)
+{
+    return SyncInstMask(uint32_t(a) | uint32_t(b));
+}
+inline constexpr SyncInstMask& operator|=(SyncInstMask& a, SyncInstMask b)
+{
+    a = a | b;
+    return a;
+}
 
 inline constexpr InstructionMask operator&(InstructionMask a, InstructionMask b)
 {
@@ -442,9 +463,18 @@ class SynchronizationObjectCoalescing : public llvm::FunctionPass
     ////////////////////////////////////////////////////////////////////////
     bool IsUntypedMemoryLscFenceOperationForGlobalAccess(const llvm::Instruction* pInst) const;
 
+    ////////////////////////////////////////////////////////////////////////
+    bool IsSharedLscFenceOperation(const llvm::Instruction* pInst) const;
+
+    ////////////////////////////////////////////////////////////////////////
+    bool IsUrbLscFenceOperation(const llvm::Instruction* pInst) const;
+
     ////////////////////////////////////////////////////////////////////////
     static bool IsTypedMemoryFenceOperation(const llvm::Instruction* pInst);
 
+    ////////////////////////////////////////////////////////////////////////
+    bool IsTypedMemoryLscFenceOperation(const llvm::Instruction* pInst) const;
+
     ////////////////////////////////////////////////////////////////////////
     bool IsTypedMemoryFenceOperationWithInvalidationFunctionality(const llvm::Instruction* pInst) const;
 
@@ -663,6 +693,39 @@ bool SynchronizationObjectCoalescing::FindRedundancies()
 
         for (llvm::Instruction* pInst : synchronizationOperations)
         {
+            SyncInstMask instType = SyncInstMask::None;
+            if (IsUntypedMemoryFenceOperationForGlobalAccess(pInst) ||
+                IsUntypedMemoryLscFenceOperationForGlobalAccess(pInst))
+            {
+                instType |= SyncInstMask::UntypedMemoryFence;
+            }
+            if (IsTypedMemoryFenceOperation(pInst) ||
+                IsTypedMemoryLscFenceOperation(pInst))
+            {
+                instType |= SyncInstMask::TypedMemoryFence;
+            }
+            if (IsUntypedMemoryFenceOperationForSharedMemoryAccess(pInst) ||
+                IsSharedLscFenceOperation(pInst))
+            {
+                instType |= SyncInstMask::SharedMemoryFence;
+            }
+            if (IsUrbFenceOperation(pInst) ||
+                IsUrbLscFenceOperation(pInst))
+            {
+                instType |= SyncInstMask::UrbMemoryFence;
+            }
+            if (isBarrierIntrinsic(pInst))
+            {
+                instType |= SyncInstMask::ThreadGroupBarrier;
+            }
+            IGC_ASSERT(instType != SyncInstMask::None);
+            bool isDisabled = instType != SyncInstMask::None &&
+                (IGC_GET_FLAG_VALUE(DisableCoalescingSynchronizationObjectMask) & static_cast<DWORD>(instType)) == static_cast<DWORD>(instType);
+            if (isDisabled)
+            {
+                continue;
+            }
+
             if ((m_GlobalMemoryInstructionMask & GetDefaultWriteMemoryInstructionMask(pInst)) == 0)
             {
 #if _DEBUG
@@ -2615,6 +2678,28 @@ bool SynchronizationObjectCoalescing::IsUntypedMemoryLscFenceOperationForGlobalA
     return false;
 }
 
+////////////////////////////////////////////////////////////////////////
+bool SynchronizationObjectCoalescing::IsSharedLscFenceOperation(const llvm::Instruction* pInst) const
+{
+    if (IsLscFenceOperation(pInst))
+    {
+        LSC_SFID mem = GetLscMem(pInst);
+        return mem == LSC_SFID::LSC_SLM;
+    }
+    return false;
+}
+
+////////////////////////////////////////////////////////////////////////
+bool SynchronizationObjectCoalescing::IsTypedMemoryLscFenceOperation(const llvm::Instruction* pInst) const
+{
+    if (IsLscFenceOperation(pInst))
+    {
+        LSC_SFID mem = GetLscMem(pInst);
+        return mem == LSC_SFID::LSC_TGM;
+    }
+    return false;
+}
+
 ////////////////////////////////////////////////////////////////////////
 bool SynchronizationObjectCoalescing::IsTypedMemoryFenceOperation(const llvm::Instruction* pInst)
 {
@@ -2642,6 +2727,17 @@ bool SynchronizationObjectCoalescing::IsTypedMemoryFenceOperationWithInvalidatio
         llvm::cast<llvm::ConstantInt>(pInst->getOperand(L1CacheInvalidateArg))->getValue().getBoolValue();
 }
 
+////////////////////////////////////////////////////////////////////////
+bool SynchronizationObjectCoalescing::IsUrbLscFenceOperation(const llvm::Instruction* pInst) const
+{
+    if (IsLscFenceOperation(pInst))
+    {
+        LSC_SFID mem = GetLscMem(pInst);
+        return mem == LSC_SFID::LSC_URB;
+    }
+    return false;
+}
+
 ////////////////////////////////////////////////////////////////////////
 bool SynchronizationObjectCoalescing::IsUrbFenceOperation(const llvm::Instruction* pInst)
 {

IGC/common/igc_flags.h

@@ -210,6 +210,7 @@ DECLARE_IGC_REGKEY(bool, DisableBranchSwaping,          false, "Setting this to
 DECLARE_IGC_REGKEY(bool, DisableSynchronizationObjectCoalescingPass, false, "Disable SynchronizationObjectCoalescing pass", false)
 DECLARE_IGC_REGKEY(bool, EnableIndependentSharedMemoryFenceFunctionality, false, "Enable treating global memory fences as shared memory fences in SynchronizationObjectCoalescing pass", false)
 DECLARE_IGC_REGKEY(DWORD, SynchronizationObjectCoalescingConfig, 0, "Modify the default behavior of SynchronizationObjectCoalescing value is a bitmask bit0 – remove fences in read barrier write scenario", true)
+DECLARE_IGC_REGKEY(DWORD, DisableCoalescingSynchronizationObjectMask, 0, "The mask is casted to IGC::SyncInstMask and informs which synchronization objects should not be coalesced. Note that synchronization objects classified in multiple types are not disabled if any bit describing them is off.", true)
 DECLARE_IGC_REGKEY(DWORD,SetLoopUnrollThreshold,        0,     "Set the loop unroll threshold. Value 0 will use the default threshold.", false)
 DECLARE_IGC_REGKEY(DWORD,SetLoopUnrollThresholdForHighRegPressure,        0,     "Set the loop unroll threshold for shaders with high reg pressure. Value 0 will use the default threshold.", false)
 DECLARE_IGC_REGKEY(DWORD,SetRegisterPressureThresholdForLoopUnroll,       96,    "Set the register pressure threshold for limiting the loop unroll to smaller loops", false)