Revert: Limit the scope of post-atomic fences

PawelJurek · pszymich · commit 7ef153b0201b · 2022-08-26T16:10:54.000+02:00
By default the LSC scope of the fence was "gpu". For post-atomic fence "local" scope should be sufficient.
diff --git a/IGC/BiFModule/Implementation/IGCBiF_Intrinsics.cl b/IGC/BiFModule/Implementation/IGCBiF_Intrinsics.cl
@@ -103,7 +103,7 @@ local uchar* __builtin_IB_AllocLocalMemPool(bool allocAllWorkgroups, uint numAdd
 
 // Memory fences
 // See GenISAIntrinsics.td for documentation
-void     __builtin_IB_memfence(bool commitEnable, bool flushRW, bool flushConstant, bool flushTexture, bool flushIcache, bool isGlobal, bool invalidateL1, bool forceLocalLSCScope);
+void     __builtin_IB_memfence(bool commitEnable, bool flushRW, bool flushConstant, bool flushTexture, bool flushIcache, bool isGlobal, bool invalidateL1);
 void     __builtin_IB_flush_sampler_cache(void);
 void     __builtin_IB_typedmemfence(bool invalidateCache);
 
diff --git a/IGC/BiFModule/Implementation/atomics.cl b/IGC/BiFModule/Implementation/atomics.cl
@@ -28,7 +28,7 @@ extern __constant int __HasThreadPauseSupport;
   __local int* __builtin_IB_get_local_lock();
   __global int* __builtin_IB_get_global_lock();
   void __builtin_IB_eu_thread_pause(uint value);
-  void __intel_memfence_handler(bool flushRW, bool isGlobal, bool invalidateL1, bool forceLocalLSCScope);
+  void __intel_memfence_handler(bool flushRW, bool isGlobal, bool invalidateL1);
 
 #define LOCAL_SPINLOCK_START() \
   { \
@@ -59,14 +59,14 @@ extern __constant int __HasThreadPauseSupport;
   if( ( (Semantics) & ( SEMANTICS_PRE_OP_NEED_FENCE ) ) > 0 )                         \
   {                                                                                   \
       bool flushL3 = (isGlobal) && ((Scope) == Device || (Scope) == CrossDevice);     \
-      __intel_memfence_handler(flushL3, isGlobal, false, false);                      \
+      __intel_memfence_handler(flushL3, isGlobal, false);                             \
   }
 
 #define FENCE_POST_OP(Scope, Semantics, isGlobal)                                     \
   if( ( (Semantics) & ( SEMANTICS_POST_OP_NEEDS_FENCE ) ) > 0 )                       \
   {                                                                                   \
       bool flushL3 = (isGlobal) && ((Scope) == Device || (Scope) == CrossDevice);     \
-      __intel_memfence_handler(flushL3, isGlobal, isGlobal, true);                    \
+      __intel_memfence_handler(flushL3, isGlobal, isGlobal);                          \
   }
 
 // This fencing scheme allows us to obey the memory model when coherency is
diff --git a/IGC/BiFModule/Implementation/barrier.cl b/IGC/BiFModule/Implementation/barrier.cl
@@ -18,39 +18,37 @@ extern __constant int __OptDisable;
 
 // MEMFENCE IMPLEMENTATION
 
-void __attribute__((optnone)) __intel_memfence_optnone(bool flushRW, bool isGlobal, bool invalidateL1, bool forceLocalLSCScope)
+void __attribute__((optnone)) __intel_memfence_optnone(bool flushRW, bool isGlobal, bool invalidateL1)
 {
-#define MEMFENCE_IF(V1, V5, V6, V7)                                                             \
-if (flushRW == V1 && isGlobal == V5 && invalidateL1 == V6 && forceLocalLSCScope == V7)          \
-{                                                                                               \
-    __builtin_IB_memfence(true, V1, false, false, false, V5, V6, V7);                           \
+#define MEMFENCE_IF(V1, V5, V6)                                    \
+if (flushRW == V1 && isGlobal == V5 && invalidateL1 == V6)         \
+{                                                                  \
+    __builtin_IB_memfence(true, V1, false, false, false, V5, V6);  \
 } else
 
 // Generate combinations for all MEMFENCE_IF cases, e.g.:
-// true, true, true, true
-// true, true, true, false etc.
-#define MF_L3(...) MF_L2(__VA_ARGS__,false) MF_L2(__VA_ARGS__,true)
+// true, true, true
+// true, true, false etc.
 #define MF_L2(...) MF_L1(__VA_ARGS__,false) MF_L1(__VA_ARGS__,true)
 #define MF_L1(...) MEMFENCE_IF(__VA_ARGS__,false) MEMFENCE_IF(__VA_ARGS__,true)
-MF_L3(false)
-MF_L3(true) {}
+MF_L2(false)
+MF_L2(true) {}
 
 #undef MEMFENCE_IF
-#undef MF_L3
 #undef MF_L2
 #undef MF_L1
 }
-void __intel_memfence(bool flushRW, bool isGlobal, bool invalidateL1, bool forceLocalLSCScope)
+void __intel_memfence(bool flushRW, bool isGlobal, bool invalidateL1)
 {
-    __builtin_IB_memfence(true, flushRW, false, false, false, isGlobal, invalidateL1, forceLocalLSCScope);
+    __builtin_IB_memfence(true, flushRW, false, false, false, isGlobal, invalidateL1);
 }
 
-void __intel_memfence_handler(bool flushRW, bool isGlobal, bool invalidateL1, bool forceLocalLSCScope)
+void __intel_memfence_handler(bool flushRW, bool isGlobal, bool invalidateL1)
 {
     if (__OptDisable)
-        __intel_memfence_optnone(flushRW, isGlobal, invalidateL1, forceLocalLSCScope);
+        __intel_memfence_optnone(flushRW, isGlobal, invalidateL1);
     else
-        __intel_memfence(flushRW, isGlobal, invalidateL1, forceLocalLSCScope);
+        __intel_memfence(flushRW, isGlobal, invalidateL1);
 }
 
 // TYPEDMEMFENCE IMPLEMENTATION
@@ -99,12 +97,12 @@ static void __intel_atomic_work_item_fence( Scope_t Memory, uint Semantics )
         // although on some platforms they may be elided; platform-specific checks are performed in codegen
         if (Semantics & WorkgroupMemory)
         {
-           __intel_memfence_handler(false, false, false,false);
+           __intel_memfence_handler(false, false, false);
         }
         if (Semantics & CrossWorkgroupMemory)
         {
            bool flushL3 = Memory == Device || Memory == CrossDevice;
-           __intel_memfence_handler(flushL3, true, invalidateL1, false);
+           __intel_memfence_handler(flushL3, true, invalidateL1);
         }
     }
 }
diff --git a/IGC/Compiler/CISACodeGen/EmitVISAPass.cpp b/IGC/Compiler/CISACodeGen/EmitVISAPass.cpp
@@ -13520,12 +13520,12 @@ LSC_FENCE_OP EmitPass::getLSCMemoryFenceOp(bool IsGlobalMemFence, bool Invalidat
 
 void EmitPass::emitMemoryFence(llvm::Instruction* inst)
 {
-    static constexpr int ExpectedNumberOfArguments = 8;
+    static constexpr int ExpectedNumberOfArguments = 7;
     IGC_ASSERT(IGCLLVM::getNumArgOperands(cast<CallInst>(inst)) == ExpectedNumberOfArguments);
     CodeGenContext* ctx = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();
 
     // If passed a non-constant value for any of the parameters,
-    // be conservative and assume the default value defined below.
+    // be conservative and assume that the parameter is true.
     // This could happen in "optnone" scenarios.
     bool CommitEnable = true;
     bool L3_Flush_RW_Data = true;
@@ -13534,7 +13534,6 @@ void EmitPass::emitMemoryFence(llvm::Instruction* inst)
     bool L3_Flush_Instructions = true;
     bool Global_Mem_Fence = true;
     bool L1_Invalidate = ctx->platform.hasL1ReadOnlyCache();
-    bool Force_Local_LSC_Scope = false;
 
     std::array<reference_wrapper<bool>, ExpectedNumberOfArguments> MemFenceArguments{
       CommitEnable,
@@ -13544,13 +13543,12 @@ void EmitPass::emitMemoryFence(llvm::Instruction* inst)
       L3_Flush_Instructions,
       Global_Mem_Fence,
       L1_Invalidate,
-      Force_Local_LSC_Scope
     };
 
     for (size_t i = 0; i < MemFenceArguments.size(); ++i) {
         if (ConstantInt* CI = llvm::dyn_cast<llvm::ConstantInt>(inst->getOperand(i)))
         {
-            MemFenceArguments[i].get() = CI->getValue().getBoolValue();
+            MemFenceArguments[i] &= CI->getValue().getBoolValue();
         }
     }
 
@@ -13584,9 +13582,6 @@ void EmitPass::emitMemoryFence(llvm::Instruction* inst)
         LSC_SFID sfid = Global_Mem_Fence ? LSC_UGM : LSC_SLM;
         // ToDo: replace with fence instrinsics that take scope/op
         LSC_SCOPE scope = Global_Mem_Fence ? LSC_SCOPE_GPU : LSC_SCOPE_GROUP;
-        // When post-atomic fence is added with L1 invalidate, we may want to limit the scope to subslice.
-        if (Force_Local_LSC_Scope)
-            scope = (sfid == LSC_SLM) ? LSC_SCOPE_GROUP : LSC_SCOPE_LOCAL;
         // Change the scope from `GPU` to `Tile` on single-tile platforms to avoid L3 flush on DG2 and MTL
         if (scope == LSC_SCOPE_GPU &&
             !m_currShader->m_Platform->hasMultiTile() &&
diff --git a/IGC/Compiler/Optimizer/OpenCLPasses/Atomics/ResolveOCLAtomics.cpp b/IGC/Compiler/Optimizer/OpenCLPasses/Atomics/ResolveOCLAtomics.cpp
@@ -372,7 +372,6 @@ void ResolveOCLAtomics::generateLockInitilization(Function* F)
         falseValue,
         falseValue,
         trueValue,
-        falseValue
     };
     m_builder->SetInsertPoint(initSpinLockEndBB, initSpinLockEndBB->getFirstInsertionPt());
     Function* localMemFence = GenISAIntrinsic::getDeclaration(m_pModule, GenISAIntrinsic::GenISA_memoryfence);
diff --git a/IGC/Compiler/Optimizer/OpenCLPasses/NamedBarriers/NamedBarriersResolution.cpp b/IGC/Compiler/Optimizer/OpenCLPasses/NamedBarriers/NamedBarriersResolution.cpp
@@ -257,7 +257,6 @@ void NamedBarriersResolution::HandleNamedBarrierSyncHW(CallInst& NBarrierSyncCal
             falseValue, // bool flushIcache
             isGlobal,   // bool isGlobal
             falseValue, // bool invalidateL1
-            falseValue, // bool foceLocalLSCScope
         },
         "",
         &(NBarrierSyncCall));
diff --git a/IGC/Compiler/tests/ResolveOCLAtomics/local_spin_lock.ll b/IGC/Compiler/tests/ResolveOCLAtomics/local_spin_lock.ll
@@ -46,7 +46,7 @@ entry:
 ; CHECK:   br label %init_spinlock_var.end
 
 ; CHECK: init_spinlock_var.end:
-; CHECK:   call void @llvm.genx.GenISA.memoryfence(i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false)
+; CHECK:   call void @llvm.genx.GenISA.memoryfence(i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true)
 ; CHECK:   call void @llvm.genx.GenISA.threadgroupbarrier()
 
 ; ---------- LOCAL_SPINLOCK_START() --------------------
@@ -74,11 +74,11 @@ if.then3.i:                                       ; preds = %while.body.i
   store volatile i8 1, i8* %done_alloca, align 1
 ; CHECK-NOT: __builtin_IB_get_local_lock
   %call4.i = call spir_func i32 addrspace(3)* @__builtin_IB_get_local_lock()
-  call spir_func void @__builtin_IB_memfence(i1 zeroext true, i1 zeroext false, i1 zeroext false, i1 zeroext false, i1 zeroext false, i1 zeroext false, i1 zeroext false, i1 zeroext false)
+  call spir_func void @__builtin_IB_memfence(i1 zeroext true, i1 zeroext false, i1 zeroext false, i1 zeroext false, i1 zeroext false, i1 zeroext false, i1 zeroext false)
 ; CHECK: %[[SPINLOCK_AS_INT1:.*]] = ptrtoint i32 addrspace(3)* @spinlock to i32
 ; CHECK: call i32 @llvm.genx.GenISA.intatomicraw.i32.p3i32.i32(i32 addrspace(3)* @spinlock, i32 %[[SPINLOCK_AS_INT1]], i32 0, i32 6)
   %call.i1.i = call spir_func i32 @__builtin_IB_atomic_xchg_local_i32(i32 addrspace(3)* %call4.i, i32 0)
-  call spir_func void @__builtin_IB_memfence(i1 zeroext true, i1 zeroext false, i1 zeroext false, i1 zeroext false, i1 zeroext false, i1 zeroext false, i1 zeroext false, i1 zeroext false)
+  call spir_func void @__builtin_IB_memfence(i1 zeroext true, i1 zeroext false, i1 zeroext false, i1 zeroext false, i1 zeroext false, i1 zeroext false, i1 zeroext false)
   br label %if.end5.i
 
 if.end5.i:                                        ; preds = %if.then3.i, %while.body.i
@@ -94,12 +94,12 @@ test_spinlock.exit:                               ; preds = %entry, %if.end5.i
 declare spir_func i32 addrspace(3)* @__builtin_IB_get_local_lock()
 declare spir_func i32 @__builtin_IB_atomic_xchg_local_i32(i32 addrspace(3)*, i32)
 declare spir_func i32 @__builtin_IB_atomic_cmpxchg_local_i32(i32 addrspace(3)*, i32, i32)
-declare spir_func void @__builtin_IB_memfence(i1 zeroext, i1 zeroext, i1 zeroext, i1 zeroext, i1 zeroext, i1 zeroext, i1 zeroext, i1 zeroext)
+declare spir_func void @__builtin_IB_memfence(i1 zeroext, i1 zeroext, i1 zeroext, i1 zeroext, i1 zeroext, i1 zeroext, i1 zeroext)
 
 ; CHECK: declare i32 @llvm.genx.GenISA.icmpxchgatomicraw.i32.p3i32.i32(i32 addrspace(3)*, i32, i32, i32)
 ; CHECK: declare i32 @llvm.genx.GenISA.intatomicraw.i32.p3i32.i32(i32 addrspace(3)*, i32, i32, i32)
 ; CHECK: declare i32 @__builtin_IB_get_local_id_x()
 ; CHECK: declare i32 @__builtin_IB_get_local_id_y()
 ; CHECK: declare i32 @__builtin_IB_get_local_id_z()
-; CHECK: declare void @llvm.genx.GenISA.memoryfence(i1, i1, i1, i1, i1, i1, i1, i1)
+; CHECK: declare void @llvm.genx.GenISA.memoryfence(i1, i1, i1, i1, i1, i1, i1)
 ; CHECK: declare void @llvm.genx.GenISA.threadgroupbarrier()
diff --git a/IGC/GenISAIntrinsics/Intrinsic_definitions.py b/IGC/GenISAIntrinsics/Intrinsic_definitions.py
@@ -1426,8 +1426,7 @@
      ("bool",                          "L3_Flush_Texture_Data"),
      ("bool",                          "L3_Flush_Instructions"),
      ("bool",                          "Fence has global effect"),
-     ("bool",                          "L1 Invalidate"),
-     ("bool",                          "Force Local LSC scope")],
+     ("bool",                          "L1 Invalidate")],
     "Convergent"]],
 ####################################################################################################
 "GenISA_mov_identity": ["",
diff --git a/IGC/LLVM3DBuilder/BuiltinsFrontendDefinitions.hpp b/IGC/LLVM3DBuilder/BuiltinsFrontendDefinitions.hpp
@@ -874,7 +874,6 @@ llvm::Value* LLVM3DBuilder<preserveNames, T, Inserter>::Create_MemoryFence(
         this->getInt1(flushInstructionCache),
         this->getInt1(globalFence),
         this->getInt1(false),
-        this->getInt1(false),
     };
     llvm::Module* module = this->GetInsertBlock()->getParent()->getParent();
     return this->CreateCall(

Original file line number	Diff line number	Diff line change
`@@ -18,39 +18,37 @@ extern __constant int __OptDisable;`
`18`	`18`
`19`	`19`	`// MEMFENCE IMPLEMENTATION`
`20`	`20`
`21`		`-void __attribute__((optnone)) __intel_memfence_optnone(bool flushRW, bool isGlobal, bool invalidateL1, bool forceLocalLSCScope)`
	`21`	`+void __attribute__((optnone)) __intel_memfence_optnone(bool flushRW, bool isGlobal, bool invalidateL1)`
`22`	`22`	`{`
`23`		`-#define MEMFENCE_IF(V1, V5, V6, V7) \`
`24`		`-if (flushRW == V1 && isGlobal == V5 && invalidateL1 == V6 && forceLocalLSCScope == V7) \`
`25`		`-{ \`
`26`		`- __builtin_IB_memfence(true, V1, false, false, false, V5, V6, V7); \`
	`23`	`+#define MEMFENCE_IF(V1, V5, V6) \`
	`24`	`+if (flushRW == V1 && isGlobal == V5 && invalidateL1 == V6) \`
	`25`	`+{ \`
	`26`	`+ __builtin_IB_memfence(true, V1, false, false, false, V5, V6); \`
`27`	`27`	`} else`
`28`	`28`
`29`	`29`	`// Generate combinations for all MEMFENCE_IF cases, e.g.:`
`30`		`-// true, true, true, true`
`31`		`-// true, true, true, false etc.`
`32`		`-#define MF_L3(...) MF_L2(__VA_ARGS__,false) MF_L2(__VA_ARGS__,true)`
	`30`	`+// true, true, true`
	`31`	`+// true, true, false etc.`
`33`	`32`	`#define MF_L2(...) MF_L1(__VA_ARGS__,false) MF_L1(__VA_ARGS__,true)`
`34`	`33`	`#define MF_L1(...) MEMFENCE_IF(__VA_ARGS__,false) MEMFENCE_IF(__VA_ARGS__,true)`
`35`		`-MF_L3(false)`
`36`		`-MF_L3(true) {}`
	`34`	`+MF_L2(false)`
	`35`	`+MF_L2(true) {}`
`37`	`36`
`38`	`37`	`#undef MEMFENCE_IF`
`39`		`-#undef MF_L3`
`40`	`38`	`#undef MF_L2`
`41`	`39`	`#undef MF_L1`
`42`	`40`	`}`
`43`		`-void __intel_memfence(bool flushRW, bool isGlobal, bool invalidateL1, bool forceLocalLSCScope)`
	`41`	`+void __intel_memfence(bool flushRW, bool isGlobal, bool invalidateL1)`
`44`	`42`	`{`
`45`		`- __builtin_IB_memfence(true, flushRW, false, false, false, isGlobal, invalidateL1, forceLocalLSCScope);`
	`43`	`+ __builtin_IB_memfence(true, flushRW, false, false, false, isGlobal, invalidateL1);`
`46`	`44`	`}`
`47`	`45`
`48`		`-void __intel_memfence_handler(bool flushRW, bool isGlobal, bool invalidateL1, bool forceLocalLSCScope)`
	`46`	`+void __intel_memfence_handler(bool flushRW, bool isGlobal, bool invalidateL1)`
`49`	`47`	`{`
`50`	`48`	`if (__OptDisable)`
`51`		`- __intel_memfence_optnone(flushRW, isGlobal, invalidateL1, forceLocalLSCScope);`
	`49`	`+ __intel_memfence_optnone(flushRW, isGlobal, invalidateL1);`
`52`	`50`	`else`
`53`		`- __intel_memfence(flushRW, isGlobal, invalidateL1, forceLocalLSCScope);`
	`51`	`+ __intel_memfence(flushRW, isGlobal, invalidateL1);`
`54`	`52`	`}`
`55`	`53`
`56`	`54`	`// TYPEDMEMFENCE IMPLEMENTATION`
`@@ -99,12 +97,12 @@ static void __intel_atomic_work_item_fence( Scope_t Memory, uint Semantics )`
`99`	`97`	`// although on some platforms they may be elided; platform-specific checks are performed in codegen`
`100`	`98`	`if (Semantics & WorkgroupMemory)`
`101`	`99`	`{`
`102`		`- __intel_memfence_handler(false, false, false,false);`
	`100`	`+ __intel_memfence_handler(false, false, false);`
`103`	`101`	`}`
`104`	`102`	`if (Semantics & CrossWorkgroupMemory)`
`105`	`103`	`{`
`106`	`104`	`bool flushL3 = Memory == Device \|\| Memory == CrossDevice;`
`107`		`- __intel_memfence_handler(flushL3, true, invalidateL1, false);`
	`105`	`+ __intel_memfence_handler(flushL3, true, invalidateL1);`
`108`	`106`	`}`
`109`	`107`	`}`
`110`	`108`	`}`