Add translucent & alpha-tested shadow support

Implement transparency-aware shadow mapping: add alpha-tested and hashed alpha casters plus an experimental translucent (moment-based) shadow accumulation path. New and updated GLSL programs provide caster/receiver support for projected and point lights (alpha-tested casters, hashed alpha coverage, and translucent moment packing), and receiver shaders sample translucent moment maps and blend them with existing shadowing. Renderer bindings and initialization were updated (src/renderer/*) to expose control CVars and bind new shadow textures, and documentation/README were added/updated (docs-user/docs-dev) describing the feature, CVars, and usage. This change enables optional translucent shadowing while preserving existing CSM and PCF-based shadow behavior.

Author: themuffinator

.install/openq4/glprogs/openq4_shadow_interaction.fs

@@ -1,9 +1,12 @@
+#version 110
+
 uniform sampler2D uBumpMap;
 uniform sampler2D uLightFalloffMap;
 uniform sampler2D uLightProjectionMap;
 uniform sampler2D uDiffuseMap;
 uniform sampler2D uSpecularMap;
 uniform sampler2D uShadowMap;
+uniform sampler2D uTranslucentShadowMap;
 
 uniform vec4 uDiffuseColor;
 uniform vec4 uSpecularColor;
@@ -17,6 +20,8 @@ uniform float uShadowCascadeBiasScale[4];
 uniform int uShadowCascadeCount;
 uniform float uShadowCascadeBlend;
 uniform float uShadowDebugMode;
+uniform float uTranslucentShadowEnabled;
+uniform float uTranslucentShadowDensity;
 
 varying vec2 vBumpTexCoord;
 varying vec2 vDiffuseTexCoord;
@@ -41,9 +46,12 @@ const float kShadowDebugProjectedUV = 3.0;
 const float kShadowDebugProjectedDepth = 4.0;
 const float kShadowDebugProjectedW = 5.0;
 const float kShadowDebugInvalidMask = 6.0;
+const float kTranslucentMomentMinVariance = 1.0e-5;
 
 float gShadowDebugState = 0.0;
 
+bool ProjectShadowCoord( vec4 shadowCoord, out vec2 localUv, out float depth );
+
 bool ShadowCoordComponentInvalid( float value ) {
 	return value != value || abs( value ) > kShadowCoordMaxMagnitude;
 }
@@ -56,6 +64,35 @@ vec3 SafeNormalize( vec3 value ) {
 	return value * inversesqrt( max( dot( value, value ), 1.0e-8 ) );
 }
 
+float ApproxErf( float x ) {
+	float s = sign( x );
+	float ax = abs( x );
+	float t = 1.0 / ( 1.0 + 0.3275911 * ax );
+	float y = 1.0 - ( ( ( ( ( 1.061405429 * t - 1.453152027 ) * t ) + 1.421413741 ) * t - 0.284496736 ) * t + 0.254829592 ) * t * exp( -ax * ax );
+	return s * y;
+}
+
+float NormalCdf( float x ) {
+	return 0.5 * ( 1.0 + ApproxErf( x * 0.70710678 ) );
+}
+
+float ResolveTranslucentShadowMoments( vec4 moments, float depth ) {
+	if ( uTranslucentShadowEnabled < 0.5 ) {
+		return 1.0;
+	}
+
+	float totalTau = max( moments.x, 0.0 );
+	if ( totalTau <= 1.0e-4 ) {
+		return 1.0;
+	}
+
+	float mean = moments.y / totalTau;
+	float variance = max( moments.z / totalTau - mean * mean, kTranslucentMomentMinVariance );
+	float sigma = sqrt( variance );
+	float tau = totalTau * clamp( NormalCdf( ( depth - mean ) / sigma ), 0.0, 1.0 );
+	return exp( -min( tau * max( uTranslucentShadowDensity, 0.0 ), 16.0 ) );
+}
+
 float CascadeBiasScale( int cascadeIndex ) {
 	if ( cascadeIndex <= 0 ) {
 		return uShadowCascadeBiasScale[0];
@@ -135,6 +172,34 @@ vec4 SampleShadowCascade( vec4 shadowCoord, vec4 atlasRect, int cascadeIndex ) {
 	return vec4( shadow * ( 1.0 / 13.0 ), localUv.x, localUv.y, depth );
 }
 
+float SampleTranslucentShadowCascade( vec4 shadowCoord, vec4 atlasRect ) {
+	if ( uTranslucentShadowEnabled < 0.5 ) {
+		return 1.0;
+	}
+
+	vec2 localUv;
+	float depth;
+	if ( !ProjectShadowCoord( shadowCoord, localUv, depth ) ) {
+		return 1.0;
+	}
+	if ( localUv.x <= 0.0 || localUv.x >= 1.0 || localUv.y <= 0.0 || localUv.y >= 1.0 ) {
+		return 1.0;
+	}
+	if ( depth <= 0.0 || depth >= 1.0 ) {
+		return 1.0;
+	}
+
+	vec2 atlasMin = atlasRect.xy;
+	vec2 atlasMax = atlasRect.zw;
+	vec2 uv = atlasMin + localUv * ( atlasMax - atlasMin );
+	vec2 clampMin = atlasMin + uShadowTexelSize * 0.5;
+	vec2 clampMax = atlasMax - uShadowTexelSize * 0.5;
+	clampMin = min( clampMin, clampMax );
+	uv = clamp( uv, clampMin, clampMax );
+
+	return ResolveTranslucentShadowMoments( texture2D( uTranslucentShadowMap, uv ), depth );
+}
+
 float CascadeSplitDepth( int index ) {
 	if ( index <= 0 ) {
 		return uShadowSplitDepths[0];
@@ -197,7 +262,7 @@ bool ProjectShadowCoord( vec4 shadowCoord, out vec2 localUv, out float depth ) {
 }
 
 int SelectCascade( float viewDepth ) {
-	int interiorSplitCount = max( uShadowCascadeCount - 1, 0 );
+	int interiorSplitCount = ( uShadowCascadeCount > 1 ) ? ( uShadowCascadeCount - 1 ) : 0;
 	if ( interiorSplitCount <= 0 || viewDepth < uShadowSplitDepths[0] ) {
 		return 0;
 	}
@@ -279,7 +344,8 @@ vec4 ShadowDebugOutput( vec4 shadowInfo ) {
 		}
 		vec4 cascadeColor = CascadeDebugColor( float( cascadeIndex ) );
 		if ( shadowInfo.z > 0.0 ) {
-			cascadeColor.rgb = mix( cascadeColor.rgb, CascadeDebugColor( float( min( cascadeIndex + 1, uShadowCascadeCount - 1 ) ) ).rgb, shadowInfo.z );
+			int nextCascadeIndex = ( cascadeIndex + 1 < uShadowCascadeCount ) ? ( cascadeIndex + 1 ) : ( uShadowCascadeCount - 1 );
+			cascadeColor.rgb = mix( cascadeColor.rgb, CascadeDebugColor( float( nextCascadeIndex ) ).rgb, shadowInfo.z );
 		}
 		return cascadeColor;
 	}
@@ -324,6 +390,45 @@ vec4 ShadowDebugOutput( vec4 shadowInfo ) {
 	return vec4( invalidColor, 1.0 );
 }
 
+float SampleTranslucentShadowCascadeByIndex( int index ) {
+	if ( index <= 0 ) {
+		return SampleTranslucentShadowCascade( vShadowCoord0, uShadowAtlasRect[0] );
+	}
+	if ( index == 1 ) {
+		return SampleTranslucentShadowCascade( vShadowCoord1, uShadowAtlasRect[1] );
+	}
+	if ( index == 2 ) {
+		return SampleTranslucentShadowCascade( vShadowCoord2, uShadowAtlasRect[2] );
+	}
+	return SampleTranslucentShadowCascade( vShadowCoord3, uShadowAtlasRect[3] );
+}
+
+float SampleTranslucentShadow() {
+	if ( uTranslucentShadowEnabled < 0.5 ) {
+		return 1.0;
+	}
+
+	int cascadeIndex = SelectCascade( vViewDepth );
+	float shadow = SampleTranslucentShadowCascadeByIndex( cascadeIndex );
+	int lastInteriorIndex = uShadowCascadeCount - 2;
+
+	if ( cascadeIndex > lastInteriorIndex || uShadowCascadeBlend <= 0.0 ) {
+		return shadow;
+	}
+
+	float previousSplit = ( cascadeIndex == 0 ) ? 0.0 : CascadeSplitDepth( cascadeIndex - 1 );
+	float currentSplit = CascadeSplitDepth( cascadeIndex );
+	float blendWidth = max( 1.0, ( currentSplit - previousSplit ) * uShadowCascadeBlend );
+	float blendStart = currentSplit - blendWidth;
+	if ( vViewDepth <= blendStart ) {
+		return shadow;
+	}
+
+	float nextShadow = SampleTranslucentShadowCascadeByIndex( cascadeIndex + 1 );
+	float blend = clamp( ( vViewDepth - blendStart ) / blendWidth, 0.0, 1.0 );
+	return mix( shadow, nextShadow, blend );
+}
+
 void main() {
 	vec4 bumpSample = texture2D( uBumpMap, vBumpTexCoord );
 	vec3 localNormal = vec3( bumpSample.a, bumpSample.g, bumpSample.b ) * 2.0 - 1.0;
@@ -339,6 +444,7 @@ void main() {
 	light *= texture2DProj( uLightProjectionMap, vLightProjectionTexCoord ).rgb;
 	vec4 shadowInfo = SampleShadow();
 	light *= shadowInfo.x;
+	light *= SampleTranslucentShadow();
 
 	vec3 diffuse = texture2D( uDiffuseMap, vDiffuseTexCoord ).rgb * uDiffuseColor.rgb;
 

.install/openq4/glprogs/openq4_shadow_interaction.vs

@@ -1,3 +1,5 @@
+#version 110
+
 attribute vec2 attr_TexCoord0;
 attribute vec3 attr_Tangent;
 attribute vec3 attr_Bitangent;

.install/openq4/glprogs/openq4_shadow_point_caster.fs

@@ -1,15 +1,48 @@
+#version 110
+
 uniform float uPointShadowFar;
+uniform sampler2D uAlphaMap;
+uniform float uAlphaRef;
+uniform float uAlphaScale;
+uniform float uAlphaTestEnabled;
+uniform float uAlphaHashEnabled;
 
 varying vec3 vPointShadowVector;
+varying vec2 vAlphaTexCoord;
 
 vec2 PackDepth16( float depth ) {
 	vec2 enc = fract( vec2( 1.0, 255.0 ) * clamp( depth, 0.0, 1.0 ) );
 	enc.x -= enc.y * ( 1.0 / 255.0 );
 	return enc;
 }
 
+float AlphaHashThreshold( vec2 fragmentCoord ) {
+	vec2 texel = floor( fragmentCoord );
+	return fract( 52.9829189 * fract( texel.x * 0.06711056 + texel.y * 0.00583715 ) );
+}
+
+float AlphaCoverage( float alpha ) {
+	float alphaRef = clamp( uAlphaRef, 0.0, 0.9999 );
+	float scaledAlpha = clamp( alpha, 0.0, 1.0 );
+	return clamp( ( scaledAlpha - alphaRef ) / max( 1.0 - alphaRef, 1.0e-4 ), 0.0, 1.0 );
+}
+
 void main() {
+	if ( uAlphaTestEnabled > 0.5 ) {
+		float alpha = texture2D( uAlphaMap, vAlphaTexCoord ).a * uAlphaScale;
+		if ( uAlphaHashEnabled > 0.5 ) {
+			float coverage = AlphaCoverage( alpha );
+			if ( coverage <= 0.0 || coverage <= AlphaHashThreshold( gl_FragCoord.xy ) ) {
+				discard;
+			}
+		} else {
+			if ( alpha <= uAlphaRef ) {
+				discard;
+			}
+		}
+	}
+
 	float depth = clamp( length( vPointShadowVector ) / uPointShadowFar, 0.0, 1.0 );
-	vec2 packed = PackDepth16( depth );
-	gl_FragColor = vec4( packed, 0.0, 1.0 );
+	vec2 packedDepth = PackDepth16( depth );
+	gl_FragColor = vec4( packedDepth, 0.0, 1.0 );
 }

.install/openq4/glprogs/openq4_shadow_point_caster.vs

@@ -1,16 +1,23 @@
+#version 110
+
 uniform vec4 uModelMatrixRow0;
 uniform vec4 uModelMatrixRow1;
 uniform vec4 uModelMatrixRow2;
 uniform vec4 uGlobalLightOrigin;
+uniform vec4 uAlphaTexCoordS;
+uniform vec4 uAlphaTexCoordT;
 
 varying vec3 vPointShadowVector;
+varying vec2 vAlphaTexCoord;
 
 void main() {
 	vec4 position = gl_Vertex;
+	vec4 alphaTexCoord = vec4( gl_MultiTexCoord0.xy, 0.0, 1.0 );
 	vec3 worldPos = vec3(
 		dot( position, uModelMatrixRow0 ),
 		dot( position, uModelMatrixRow1 ),
 		dot( position, uModelMatrixRow2 ) );
 	vPointShadowVector = worldPos - uGlobalLightOrigin.xyz;
+	vAlphaTexCoord = vec2( dot( alphaTexCoord, uAlphaTexCoordS ), dot( alphaTexCoord, uAlphaTexCoordT ) );
 	gl_Position = ftransform();
 }

.install/openq4/glprogs/openq4_shadow_point_interaction.fs

@@ -1,9 +1,12 @@
+#version 110
+
 uniform sampler2D uBumpMap;
 uniform sampler2D uLightFalloffMap;
 uniform sampler2D uLightProjectionMap;
 uniform sampler2D uDiffuseMap;
 uniform sampler2D uSpecularMap;
 uniform samplerCube uPointShadowMap;
+uniform samplerCube uPointTranslucentShadowMap;
 
 uniform vec4 uDiffuseColor;
 uniform vec4 uSpecularColor;
@@ -12,6 +15,10 @@ uniform float uShadowBias;
 uniform float uShadowNormalBias;
 uniform float uShadowFilterRadius;
 uniform float uPointShadowTexelScale;
+uniform float uTranslucentShadowEnabled;
+uniform float uTranslucentShadowDensity;
+
+const float kTranslucentMomentMinVariance = 1.0e-5;
 
 varying vec2 vBumpTexCoord;
 varying vec2 vDiffuseTexCoord;
@@ -28,6 +35,35 @@ vec3 SafeNormalize( vec3 value ) {
 	return value * inversesqrt( max( dot( value, value ), 1.0e-8 ) );
 }
 
+float ApproxErf( float x ) {
+	float s = sign( x );
+	float ax = abs( x );
+	float t = 1.0 / ( 1.0 + 0.3275911 * ax );
+	float y = 1.0 - ( ( ( ( ( 1.061405429 * t - 1.453152027 ) * t ) + 1.421413741 ) * t - 0.284496736 ) * t + 0.254829592 ) * t * exp( -ax * ax );
+	return s * y;
+}
+
+float NormalCdf( float x ) {
+	return 0.5 * ( 1.0 + ApproxErf( x * 0.70710678 ) );
+}
+
+float ResolveTranslucentShadowMoments( vec4 moments, float depth ) {
+	if ( uTranslucentShadowEnabled < 0.5 ) {
+		return 1.0;
+	}
+
+	float totalTau = max( moments.x, 0.0 );
+	if ( totalTau <= 1.0e-4 ) {
+		return 1.0;
+	}
+
+	float mean = moments.y / totalTau;
+	float variance = max( moments.z / totalTau - mean * mean, kTranslucentMomentMinVariance );
+	float sigma = sqrt( variance );
+	float tau = totalTau * clamp( NormalCdf( ( depth - mean ) / sigma ), 0.0, 1.0 );
+	return exp( -min( tau * max( uTranslucentShadowDensity, 0.0 ), 16.0 ) );
+}
+
 float ShadowReceiverBias() {
 	float lightCos = clamp( vShadowLightCos, 0.0, 1.0 );
 	float slopeBias = sqrt( max( 1.0 - lightCos * lightCos, 0.0 ) );
@@ -81,6 +117,20 @@ float SamplePointShadow() {
 	return shadow * ( 1.0 / 13.0 );
 }
 
+float SamplePointTranslucentShadow() {
+	if ( uTranslucentShadowEnabled < 0.5 || uPointShadowFar <= 0.0 ) {
+		return 1.0;
+	}
+
+	float depth = length( vPointShadowVector ) / uPointShadowFar;
+	if ( depth <= 0.0 || depth >= 1.0 ) {
+		return 1.0;
+	}
+
+	vec3 direction = SafeNormalize( vPointShadowVector );
+	return ResolveTranslucentShadowMoments( textureCube( uPointTranslucentShadowMap, direction ), depth );
+}
+
 void main() {
 	vec4 bumpSample = texture2D( uBumpMap, vBumpTexCoord );
 	vec3 localNormal = vec3( bumpSample.a, bumpSample.g, bumpSample.b ) * 2.0 - 1.0;
@@ -93,6 +143,7 @@ void main() {
 	light *= texture2DProj( uLightFalloffMap, vLightFalloffTexCoord ).rgb;
 	light *= texture2DProj( uLightProjectionMap, vLightProjectionTexCoord ).rgb;
 	light *= SamplePointShadow();
+	light *= SamplePointTranslucentShadow();
 
 	vec3 diffuse = texture2D( uDiffuseMap, vDiffuseTexCoord ).rgb * uDiffuseColor.rgb;
 

.install/openq4/glprogs/openq4_shadow_point_interaction.vs

@@ -1,3 +1,5 @@
+#version 110
+
 attribute vec2 attr_TexCoord0;
 attribute vec3 attr_Tangent;
 attribute vec3 attr_Bitangent;

.install/openq4/glprogs/openq4_shadow_point_translucent_caster.fs

@@ -0,0 +1,41 @@
+#version 110
+
+uniform float uPointShadowFar;
+uniform sampler2D uAlphaMap;
+uniform float uAlphaScale;
+uniform float uOpacitySourceMode;
+uniform float uTranslucentMinAlpha;
+
+varying vec3 vPointShadowVector;
+varying vec2 vAlphaTexCoord;
+varying float vVertexAlpha;
+
+float OpticalDepth( float alpha ) {
+	return -log( max( 1.0 - clamp( alpha, 0.0, 0.999 ), 1.0e-4 ) );
+}
+
+float StageOpacity() {
+	if ( uOpacitySourceMode > 1.5 ) {
+		return clamp( uAlphaScale * vVertexAlpha, 0.0, 0.999 );
+	}
+
+	vec4 sampleColor = texture2D( uAlphaMap, vAlphaTexCoord );
+	float opacity = sampleColor.a;
+	if ( uOpacitySourceMode > 0.5 ) {
+		opacity = max( opacity, dot( sampleColor.rgb, vec3( 0.2126, 0.7152, 0.0722 ) ) );
+	}
+
+	return clamp( opacity * uAlphaScale * vVertexAlpha, 0.0, 0.999 );
+}
+
+void main() {
+	float alpha = StageOpacity();
+	if ( alpha <= uTranslucentMinAlpha ) {
+		discard;
+	}
+
+	float tau = OpticalDepth( alpha );
+	float depth = clamp( length( vPointShadowVector ) / uPointShadowFar, 0.0, 1.0 );
+	float depth2 = depth * depth;
+	gl_FragColor = vec4( tau, tau * depth, tau * depth2, tau * depth2 * depth );
+}