subgroup stuff fixed, added better ballot barrier semantics

Author: devshgraphicsprogramming

include/irr/builtin/glsl/ext/LumaMeter/common.glsl

@@ -280,30 +280,9 @@ float irr_glsl_ext_LumaMeter_impl_getMeasuredLumaLog2(in irr_glsl_ext_LumaMeter_
 // TODO: figure out why the irr_glsl_workgroupExclusiveAdd function doesn't work
 uint irr_glsl_workgroupExclusiveAdd2(uint val)
 {
-#if 0
-    uint pingpong = uint(_IRR_GLSL_EXT_LUMA_METER_BIN_COUNT);
-    //! Bad INEFFICIENT Kogge-Stone adder, don't implement this way!
-    for (int pass = 1; pass < _IRR_GLSL_EXT_LUMA_METER_BIN_COUNT; pass <<= 1)
-    {
-        uint index = gl_LocalInvocationIndex + pingpong;
-        pingpong ^= _IRR_GLSL_EXT_LUMA_METER_BIN_COUNT;
-
-        _IRR_GLSL_SCRATCH_SHARED_DEFINED_[index] = val;
-        barrier();
-        memoryBarrierShared();
-        if (gl_LocalInvocationIndex >= pass)
-            val += _IRR_GLSL_SCRATCH_SHARED_DEFINED_[index - pass];
-    }
-    barrier();
-    memoryBarrierShared();
-    return val;
-#elif 1
-    barrier();
-    memoryBarrierShared();
-    _IRR_GLSL_SCRATCH_SHARED_DEFINED_[gl_LocalInvocationIndex] = 0u;
-    _IRR_GLSL_SCRATCH_SHARED_DEFINED_[_IRR_GLSL_EXT_LUMA_METER_BIN_COUNT + gl_LocalInvocationIndex] = 0u;
     barrier();
     memoryBarrierShared();
+#if 1
     const uint K = irr_glsl_SubgroupSize;
     const uint outIx = gl_LocalInvocationIndex/K;
     uint subScan = irr_glsl_subgroupInclusiveAdd(val);
@@ -322,6 +301,8 @@ uint irr_glsl_workgroupExclusiveAdd2(uint val)
 #else
     _IRR_GLSL_SCRATCH_SHARED_DEFINED_[gl_LocalInvocationIndex] = 0u;
     _IRR_GLSL_SCRATCH_SHARED_DEFINED_[_IRR_GLSL_EXT_LUMA_METER_BIN_COUNT+gl_LocalInvocationIndex] = 0u;
+    barrier();
+    memoryBarrierShared();
     return irr_glsl_workgroupExclusiveAdd(val);
 #endif
 }

include/irr/builtin/glsl/subgroup/arithmetic_portability.glsl

@@ -16,7 +16,7 @@
 #else
 */
 
-#define SUBGROUP_BARRIERS
+#define SUBGROUP_BARRIERS memoryBarrierShared()
 
 //#endif
 
@@ -141,28 +141,34 @@ TODO: Keep the pseudo subgroup and offset code DRY, move to a function.
 	const uint pseudoSubgroupID = gl_LocalInvocationIndex&hiMask; \
 	const uint scratchOffset = (pseudoSubgroupID<<1u)|pseudoSubgroupInvocation; \
 	const uint primaryOffset = scratchOffset+irr_glsl_HalfSubgroupSize; \
+	SUBGROUP_BARRIERS; \
 	_IRR_GLSL_SCRATCH_SHARED_DEFINED_[primaryOffset] = INVCONV (VALUE); \
 	if (CLEAR && pseudoSubgroupInvocation<irr_glsl_HalfSubgroupSize) \
 		_IRR_GLSL_SCRATCH_SHARED_DEFINED_[scratchOffset] = INVCONV (IDENTITY); \
 	SUBGROUP_BARRIERS; \
 	VALUE = OP (VALUE,CONV (_IRR_GLSL_SCRATCH_SHARED_DEFINED_[primaryOffset-1u])); \
+	SUBGROUP_BARRIERS; \
 	_IRR_GLSL_SCRATCH_SHARED_DEFINED_[primaryOffset] = INVCONV (VALUE); \
 	SUBGROUP_BARRIERS; \
 	VALUE = OP (VALUE,CONV (_IRR_GLSL_SCRATCH_SHARED_DEFINED_[primaryOffset-2u])); \
 	for (uint stp=irr_glsl_MinSubgroupSize; stp<irr_glsl_SubgroupSize; stp<<=1u) \
 	{ \
+		SUBGROUP_BARRIERS; \
 		_IRR_GLSL_SCRATCH_SHARED_DEFINED_[primaryOffset] = INVCONV (VALUE); \
 		SUBGROUP_BARRIERS; \
 		VALUE = OP (VALUE,CONV (_IRR_GLSL_SCRATCH_SHARED_DEFINED_[primaryOffset-stp])); \
-	}
+	} \
+	SUBGROUP_BARRIERS;
 
 
 #define IRR_GLSL_SUBGROUP_REDUCE(CONV,OP,VALUE,CLEAR,IDENTITY,INVCONV) IRR_GLSL_SUBGROUP_ARITHMETIC_IMPL(CONV,OP,VALUE,CLEAR,IDENTITY,INVCONV) \
 	_IRR_GLSL_SCRATCH_SHARED_DEFINED_[primaryOffset] = INVCONV (VALUE); \
 	SUBGROUP_BARRIERS; \
 	const uint maxPseudoSubgroupInvocation = (_IRR_GLSL_WORKGROUP_SIZE_-1u)&loMask; \
 	const uint maxPseudoSubgroupID = (_IRR_GLSL_WORKGROUP_SIZE_-1u)&hiMask; \
-	return CONV(_IRR_GLSL_SCRATCH_SHARED_DEFINED_[((maxPseudoSubgroupID<<1u)|maxPseudoSubgroupInvocation)+irr_glsl_HalfSubgroupSize])
+	const uint lastItem = _IRR_GLSL_SCRATCH_SHARED_DEFINED_[((maxPseudoSubgroupID<<1u)|maxPseudoSubgroupInvocation)+irr_glsl_HalfSubgroupSize]; \
+	SUBGROUP_BARRIERS; \
+	return CONV (lastItem);
 
 
 
@@ -267,7 +273,9 @@ float irr_glsl_subgroupMax_impl(in bool clearScratchToIdentity, float value)
 #define IRR_GLSL_SUBGROUP_EXCLUSIVE_SCAN(CONV,OP,VALUE,CLEAR,IDENTITY,INVCONV) IRR_GLSL_SUBGROUP_ARITHMETIC_IMPL(CONV,OP,VALUE,CLEAR,IDENTITY,INVCONV) \
 	_IRR_GLSL_SCRATCH_SHARED_DEFINED_[primaryOffset] = INVCONV (VALUE); \
 	SUBGROUP_BARRIERS; \
-	return CONV (_IRR_GLSL_SCRATCH_SHARED_DEFINED_[primaryOffset-1u])
+	const uint prevItem = _IRR_GLSL_SCRATCH_SHARED_DEFINED_[primaryOffset-1u]; \
+	SUBGROUP_BARRIERS; \
+	return CONV (prevItem);
 
 
 uint irr_glsl_subgroupInclusiveAnd_impl(in bool clearScratchToIdentity, uint value)
@@ -348,16 +356,7 @@ float irr_glsl_subgroupExclusiveOr_impl(in bool clearScratchToIdentity, float va
 
 uint irr_glsl_subgroupInclusiveAdd_impl(in bool clearScratchToIdentity, uint value)
 {
-	const uint loMask = 3u;
-    uint pseudoSubgroupInvocation = gl_LocalInvocationIndex & loMask;
-	uint pseudoSubgroupID = gl_LocalInvocationIndex & (~loMask);
-	_IRR_GLSL_SCRATCH_SHARED_DEFINED_[gl_LocalInvocationIndex] = value;
-	barrier();
-	memoryBarrierShared();
-	for (uint i=pseudoSubgroupID; i<gl_LocalInvocationIndex; i++)
-		value += _IRR_GLSL_SCRATCH_SHARED_DEFINED_[i];
-	return value;
-	//IRR_GLSL_SUBGROUP_INCLUSIVE_SCAN(irr_glsl_identityFunction,irr_glsl_add,value,clearScratchToIdentity,0u,irr_glsl_identityFunction);
+	IRR_GLSL_SUBGROUP_INCLUSIVE_SCAN(irr_glsl_identityFunction,irr_glsl_add,value,clearScratchToIdentity,0u,irr_glsl_identityFunction);
 }
 int irr_glsl_subgroupInclusiveAdd_impl(in bool clearScratchToIdentity, int value)
 {

include/irr/builtin/glsl/workgroup/ballot.glsl

@@ -59,7 +59,7 @@ If `GL_KHR_subgroup_arithmetic` is not available then these functions require em
 
 
 // puts the result into shared memory at offsets [0,_IRR_GLSL_WORKGROUP_SIZE_/32)
-void irr_glsl_workgroupBallot(in bool value)
+void irr_glsl_workgroupBallot_noBarriers(in bool value)
 {
 	// TODO: Optimization using subgroupBallot in an ifdef IRR_GL_something (need to do feature mapping first)
 	if (gl_LocalInvocationIndex<irr_glsl_workgroupBallot_impl_BitfieldDWORDs)
@@ -68,6 +68,12 @@ void irr_glsl_workgroupBallot(in bool value)
 	memoryBarrierShared();
 	if (value)
 		atomicOr(_IRR_GLSL_SCRATCH_SHARED_DEFINED_[irr_glsl_workgroupBallot_impl_getDWORD(gl_LocalInvocationIndex)],1u<<(gl_LocalInvocationIndex&31u));
+}
+void irr_glsl_workgroupBallot(in bool value)
+{
+	barrier();
+	memoryBarrierShared();
+	irr_glsl_workgroupBallot_noBarriers(value);
 	barrier();
 }
 
@@ -118,7 +124,7 @@ uint irr_glsl_workgroupBallotBitCount()
 }
 
 
-uint irr_glsl_workgroupBroadcast_noEndBarriers(in uint val, in uint id)
+uint irr_glsl_workgroupBroadcast_noBarriers(in uint val, in uint id)
 {
 	if (gl_LocalInvocationIndex==id)
 		_IRR_GLSL_SCRATCH_SHARED_DEFINED_[irr_glsl_workgroupBallot_impl_BitfieldDWORDs] = val;
@@ -128,13 +134,15 @@ uint irr_glsl_workgroupBroadcast_noEndBarriers(in uint val, in uint id)
 }
 uint irr_glsl_workgroupBroadcast(in uint val, in uint id)
 {
-	const uint retval = irr_glsl_workgroupBroadcast_noEndBarriers(val,id);
+	barrier();
+	memoryBarrierShared();
+	const uint retval = irr_glsl_workgroupBroadcast_noBarriers(val,id);
 	barrier();
 	memoryBarrierShared();
 	return retval;
 }
 
-uint irr_glsl_workgroupBroadcastFirst_noEndBarriers(in uint val)
+uint irr_glsl_workgroupBroadcastFirst_noBarriers(in uint val)
 {
 	if (irr_glsl_workgroupElect())
 		_IRR_GLSL_SCRATCH_SHARED_DEFINED_[irr_glsl_workgroupBallot_impl_BitfieldDWORDs] = val;
@@ -144,7 +152,9 @@ uint irr_glsl_workgroupBroadcastFirst_noEndBarriers(in uint val)
 }
 uint irr_glsl_workgroupBroadcastFirst(in uint val)
 {
-	const uint retval = irr_glsl_workgroupBroadcastFirst_noEndBarriers(val);
+	barrier();
+	memoryBarrierShared();
+	const uint retval = irr_glsl_workgroupBroadcastFirst_noBarriers(val);
 	barrier();
 	memoryBarrierShared();
 	return retval;