Add stencil support for spatial materials

Author: apples

doc/classes/BaseMaterial3D.xml

@@ -378,6 +378,24 @@
 			The method for rendering the specular blob.
 			[b]Note:[/b] [member specular_mode] only applies to the specular blob. It does not affect specular reflections from the sky, screen-space reflections, [VoxelGI], SDFGI or [ReflectionProbe]s. To disable reflections from these sources as well, set [member metallic_specular] to [code]0.0[/code] instead.
 		</member>
+		<member name="stencil_color" type="Color" setter="set_stencil_effect_color" getter="get_stencil_effect_color" default="Color(0, 0, 0, 1)" experimental="May be affected by future rendering pipeline changes.">
+			The primary color of the stencil effect.
+		</member>
+		<member name="stencil_compare" type="int" setter="set_stencil_compare" getter="get_stencil_compare" enum="BaseMaterial3D.StencilCompare" default="0" experimental="May be affected by future rendering pipeline changes.">
+			The comparison operator to use for stencil masking operations. See [enum StencilCompare].
+		</member>
+		<member name="stencil_flags" type="int" setter="set_stencil_flags" getter="get_stencil_flags" default="0" experimental="May be affected by future rendering pipeline changes.">
+			The flags dictating how the stencil operation behaves. See [enum StencilFlags].
+		</member>
+		<member name="stencil_mode" type="int" setter="set_stencil_mode" getter="get_stencil_mode" enum="BaseMaterial3D.StencilMode" default="0" experimental="May be affected by future rendering pipeline changes.">
+			The stencil effect mode. See [enum StencilMode].
+		</member>
+		<member name="stencil_outline_thickness" type="float" setter="set_stencil_effect_outline_thickness" getter="get_stencil_effect_outline_thickness" default="0.01" experimental="May be affected by future rendering pipeline changes.">
+			The outline thickness for [constant STENCIL_MODE_OUTLINE].
+		</member>
+		<member name="stencil_reference" type="int" setter="set_stencil_reference" getter="get_stencil_reference" default="1" experimental="May be affected by future rendering pipeline changes.">
+			The stencil reference value (0-255). Typically a power of 2.
+		</member>
 		<member name="subsurf_scatter_enabled" type="bool" setter="set_feature" getter="get_feature" default="false">
 			If [code]true[/code], subsurface scattering is enabled. Emulates light that penetrates an object's surface, is scattered, and then emerges. Subsurface scattering quality is controlled by [member ProjectSettings.rendering/environment/subsurface_scattering/subsurface_scattering_quality].
 		</member>
@@ -828,5 +846,49 @@
 		<constant name="DISTANCE_FADE_OBJECT_DITHER" value="3" enum="DistanceFadeMode">
 			Smoothly fades the object out based on the object's distance from the camera using a dithering approach. Dithering discards pixels based on a set pattern to smoothly fade without enabling transparency. On certain hardware, this can be faster than [constant DISTANCE_FADE_PIXEL_ALPHA] and [constant DISTANCE_FADE_PIXEL_DITHER].
 		</constant>
+		<constant name="STENCIL_MODE_DISABLED" value="0" enum="StencilMode">
+			Disables stencil operations.
+		</constant>
+		<constant name="STENCIL_MODE_OUTLINE" value="1" enum="StencilMode">
+			Stencil preset which applies an outline to the object.
+			[b]Note:[/b] Requires a [member Material.next_pass] material which will be automatically applied. Any manual changes made to [member Material.next_pass] will be lost when the stencil properties are modified or the scene is reloaded. To safely apply a [member Material.next_pass] material on a material that uses stencil presets, use [member GeometryInstance3D.material_overlay] instead.
+		</constant>
+		<constant name="STENCIL_MODE_XRAY" value="2" enum="StencilMode">
+			Stencil preset which shows a silhouette of the object behind walls.
+			[b]Note:[/b] Requires a [member Material.next_pass] material which will be automatically applied. Any manual changes made to [member Material.next_pass] will be lost when the stencil properties are modified or the scene is reloaded. To safely apply a [member Material.next_pass] material on a material that uses stencil presets, use [member GeometryInstance3D.material_overlay] instead.
+		</constant>
+		<constant name="STENCIL_MODE_CUSTOM" value="3" enum="StencilMode">
+			Enables stencil operations without a preset.
+		</constant>
+		<constant name="STENCIL_FLAG_READ" value="1" enum="StencilFlags">
+			The material will only be rendered where it passes a stencil comparison with existing stencil buffer values. See [enum StencilCompare].
+		</constant>
+		<constant name="STENCIL_FLAG_WRITE" value="2" enum="StencilFlags">
+			The material will write the reference value to the stencil buffer where it passes the depth test.
+		</constant>
+		<constant name="STENCIL_FLAG_WRITE_DEPTH_FAIL" value="4" enum="StencilFlags">
+			The material will write the reference value to the stencil buffer where it fails the depth test.
+		</constant>
+		<constant name="STENCIL_COMPARE_ALWAYS" value="0" enum="StencilCompare">
+			Always passes the stencil test.
+		</constant>
+		<constant name="STENCIL_COMPARE_LESS" value="1" enum="StencilCompare">
+			Passes the stencil test when the reference value is less than the existing stencil value.
+		</constant>
+		<constant name="STENCIL_COMPARE_EQUAL" value="2" enum="StencilCompare">
+			Passes the stencil test when the reference value is equal to the existing stencil value.
+		</constant>
+		<constant name="STENCIL_COMPARE_LESS_OR_EQUAL" value="3" enum="StencilCompare">
+			Passes the stencil test when the reference value is less than or equal to the existing stencil value.
+		</constant>
+		<constant name="STENCIL_COMPARE_GREATER" value="4" enum="StencilCompare">
+			Passes the stencil test when the reference value is greater than the existing stencil value.
+		</constant>
+		<constant name="STENCIL_COMPARE_NOT_EQUAL" value="5" enum="StencilCompare">
+			Passes the stencil test when the reference value is not equal to the existing stencil value.
+		</constant>
+		<constant name="STENCIL_COMPARE_GREATER_OR_EQUAL" value="6" enum="StencilCompare">
+			Passes the stencil test when the reference value is greater than or equal to the existing stencil value.
+		</constant>
 	</constants>
 </class>

drivers/gles3/rasterizer_gles3.cpp

@@ -138,6 +138,10 @@ void RasterizerGLES3::clear_depth(float p_depth) {
 #endif // GLES_API_ENABLED
 }
 
+void RasterizerGLES3::clear_stencil(int32_t p_stencil) {
+	glClearStencil(p_stencil);
+}
+
 #ifdef CAN_DEBUG
 static void GLAPIENTRY _gl_debug_print(GLenum source, GLenum type, GLuint id, GLenum severity, GLsizei length, const GLchar *message, const GLvoid *userParam) {
 	// These are ultimately annoying, so removing for now.

drivers/gles3/rasterizer_gles3.h

@@ -116,6 +116,7 @@ class RasterizerGLES3 : public RendererCompositor {
 
 	static bool is_gles_over_gl() { return gles_over_gl; }
 	static void clear_depth(float p_depth);
+	static void clear_stencil(int32_t p_stencil);
 
 	static void make_current(bool p_gles_over_gl) {
 		gles_over_gl = p_gles_over_gl;

drivers/gles3/rasterizer_scene_gles3.cpp

@@ -224,6 +224,10 @@ void RasterizerSceneGLES3::_geometry_instance_add_surface_with_material(Geometry
 		flags |= GeometryInstanceSurface::FLAG_USES_DOUBLE_SIDED_SHADOWS;
 	}
 
+	if (p_material->shader_data->stencil_enabled) {
+		flags |= GeometryInstanceSurface::FLAG_USES_STENCIL;
+	}
+
 	if (has_alpha || has_read_screen_alpha || p_material->shader_data->depth_draw == GLES3::SceneShaderData::DEPTH_DRAW_DISABLED || p_material->shader_data->depth_test != GLES3::SceneShaderData::DEPTH_TEST_ENABLED) {
 		//material is only meant for alpha pass
 		flags |= GeometryInstanceSurface::FLAG_PASS_ALPHA;
@@ -237,6 +241,17 @@ void RasterizerSceneGLES3::_geometry_instance_add_surface_with_material(Geometry
 		flags |= GeometryInstanceSurface::FLAG_PASS_SHADOW;
 	}
 
+	if (p_material->shader_data->stencil_enabled) {
+		if (p_material->shader_data->stencil_flags & GLES3::SceneShaderData::STENCIL_FLAG_READ) {
+			// Stencil materials which read from the stencil buffer must be in the alpha pass.
+			// This is critical to preserve compatibility once we'll have the compositor.
+			if (!(flags & GeometryInstanceSurface::FLAG_PASS_ALPHA)) {
+				String shader_path = p_material->shader_data->path.is_empty() ? "" : "(" + p_material->shader_data->path + ")";
+				ERR_PRINT_ED(vformat("Attempting to use a shader %s that reads stencil but is not in the alpha queue. Ensure the material uses alpha blending or has depth_draw disabled or depth_test disabled.", shader_path));
+			}
+		}
+	}
+
 	GLES3::SceneMaterialData *material_shadow = nullptr;
 	void *surface_shadow = nullptr;
 	if (!p_material->shader_data->uses_particle_trails && !p_material->shader_data->writes_modelview_or_projection && !p_material->shader_data->uses_vertex && !p_material->shader_data->uses_discard && !p_material->shader_data->uses_depth_prepass_alpha && !p_material->shader_data->uses_alpha_clip && !p_material->shader_data->uses_world_coordinates && !p_material->shader_data->wireframe) {
@@ -1234,6 +1249,7 @@ void RasterizerSceneGLES3::_fill_render_list(RenderListType p_render_list, const
 		scene_state.used_screen_texture = false;
 		scene_state.used_normal_texture = false;
 		scene_state.used_depth_texture = false;
+		scene_state.used_opaque_stencil = false;
 	}
 
 	Plane near_plane;
@@ -1427,6 +1443,9 @@ void RasterizerSceneGLES3::_fill_render_list(RenderListType p_render_list, const
 				if (surf->flags & GeometryInstanceSurface::FLAG_USES_DEPTH_TEXTURE) {
 					scene_state.used_depth_texture = true;
 				}
+				if ((surf->flags & GeometryInstanceSurface::FLAG_USES_STENCIL) && !force_alpha && (surf->flags & (GeometryInstanceSurface::FLAG_PASS_DEPTH | GeometryInstanceSurface::FLAG_PASS_OPAQUE))) {
+					scene_state.used_opaque_stencil = true;
+				}
 
 			} else if (p_pass_mode == PASS_MODE_SHADOW) {
 				if (surf->flags & GeometryInstanceSurface::FLAG_PASS_SHADOW) {
@@ -2490,6 +2509,9 @@ void RasterizerSceneGLES3::render_scene(const Ref<RenderSceneBuffers> &p_render_
 	// Do depth prepass if it's explicitly enabled
 	bool use_depth_prepass = config->use_depth_prepass;
 
+	// Forcibly enable depth prepass if opaque stencil writes are used.
+	use_depth_prepass = use_depth_prepass || scene_state.used_opaque_stencil;
+
 	// Don't do depth prepass we are rendering overdraw
 	use_depth_prepass = use_depth_prepass && get_debug_draw_mode() != RS::VIEWPORT_DEBUG_DRAW_OVERDRAW;
 
@@ -2506,10 +2528,13 @@ void RasterizerSceneGLES3::render_scene(const Ref<RenderSceneBuffers> &p_render_
 		scene_state.enable_gl_blend(false);
 		scene_state.set_gl_depth_func(GL_GEQUAL);
 		scene_state.enable_gl_scissor_test(false);
+		scene_state.enable_gl_stencil_test(false);
+		scene_state.set_gl_stencil_write_mask(255);
 
 		glColorMask(0, 0, 0, 0);
 		RasterizerGLES3::clear_depth(0.0);
-		glClear(GL_DEPTH_BUFFER_BIT);
+		RasterizerGLES3::clear_stencil(0);
+		glClear(GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
 		// Some desktop GL implementations fall apart when using Multiview with GL_NONE.
 		GLuint db = p_camera_data->view_count > 1 ? GL_COLOR_ATTACHMENT0 : GL_NONE;
 		glDrawBuffers(1, &db);
@@ -2549,9 +2574,12 @@ void RasterizerSceneGLES3::render_scene(const Ref<RenderSceneBuffers> &p_render_
 		glDrawBuffers(1, &db);
 	}
 
+	scene_state.enable_gl_stencil_test(false);
+
 	if (!fb_cleared) {
 		RasterizerGLES3::clear_depth(0.0);
-		glClear(GL_DEPTH_BUFFER_BIT);
+		RasterizerGLES3::clear_stencil(0);
+		glClear(GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
 	}
 
 	// Need to clear framebuffer unless:
@@ -2631,6 +2659,7 @@ void RasterizerSceneGLES3::render_scene(const Ref<RenderSceneBuffers> &p_render_
 	_render_list_template<PASS_MODE_COLOR>(&render_list_params, &render_data, 0, render_list[RENDER_LIST_OPAQUE].elements.size());
 
 	scene_state.enable_gl_depth_draw(false);
+	scene_state.enable_gl_stencil_test(false);
 
 	if (draw_sky || draw_sky_fog_only) {
 		RENDER_TIMESTAMP("Render Sky");
@@ -2689,7 +2718,7 @@ void RasterizerSceneGLES3::render_scene(const Ref<RenderSceneBuffers> &p_render_
 			if (scene_state.used_depth_texture) {
 				glBlitFramebuffer(0, 0, size.x, size.y,
 						0, 0, size.x, size.y,
-						GL_DEPTH_BUFFER_BIT, GL_NEAREST);
+						GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT, GL_NEAREST);
 				glActiveTexture(GL_TEXTURE0 + config->max_texture_image_units - 7);
 				glBindTexture(GL_TEXTURE_2D, backbuffer_depth);
 			}
@@ -2707,6 +2736,8 @@ void RasterizerSceneGLES3::render_scene(const Ref<RenderSceneBuffers> &p_render_
 
 	_render_list_template<PASS_MODE_COLOR_TRANSPARENT>(&render_list_params_alpha, &render_data, 0, render_list[RENDER_LIST_ALPHA].elements.size(), true);
 
+	scene_state.enable_gl_stencil_test(false);
+
 	if (!flip_y) {
 		// Restore the default winding order.
 		glFrontFace(GL_CCW);
@@ -2836,7 +2867,7 @@ void RasterizerSceneGLES3::_render_post_processing(const RenderDataGLES3 *p_rend
 			// Copy depth buffer
 			glBindFramebuffer(GL_READ_FRAMEBUFFER, fbo_int);
 			glBindFramebuffer(GL_DRAW_FRAMEBUFFER, fbo_rt);
-			glBlitFramebuffer(0, 0, internal_size.x, internal_size.y, 0, 0, target_size.x, target_size.y, GL_DEPTH_BUFFER_BIT, GL_NEAREST);
+			glBlitFramebuffer(0, 0, internal_size.x, internal_size.y, 0, 0, target_size.x, target_size.y, GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT, GL_NEAREST);
 		}
 
 		glBindFramebuffer(GL_FRAMEBUFFER, fbo_rt);
@@ -2865,10 +2896,10 @@ void RasterizerSceneGLES3::_render_post_processing(const RenderDataGLES3 *p_rend
 
 			for (uint32_t v = 0; v < view_count; v++) {
 				glFramebufferTextureLayer(GL_READ_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, read_color, 0, v);
-				glFramebufferTextureLayer(GL_READ_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, read_depth, 0, v);
+				glFramebufferTextureLayer(GL_READ_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, read_depth, 0, v);
 				glFramebufferTextureLayer(GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, write_color, 0, v);
-				glFramebufferTextureLayer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, write_depth, 0, v);
-				glBlitFramebuffer(0, 0, internal_size.x, internal_size.y, 0, 0, internal_size.x, internal_size.y, GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT, GL_NEAREST);
+				glFramebufferTextureLayer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, write_depth, 0, v);
+				glBlitFramebuffer(0, 0, internal_size.x, internal_size.y, 0, 0, internal_size.x, internal_size.y, GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT, GL_NEAREST);
 			}
 		}
 
@@ -2910,10 +2941,10 @@ void RasterizerSceneGLES3::_render_post_processing(const RenderDataGLES3 *p_rend
 			glBindFramebuffer(GL_DRAW_FRAMEBUFFER, fbos[1]);
 
 			for (uint32_t v = 0; v < view_count; v++) {
-				glFramebufferTextureLayer(GL_READ_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, read_depth, 0, v);
-				glFramebufferTextureLayer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, write_depth, 0, v);
+				glFramebufferTextureLayer(GL_READ_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, read_depth, 0, v);
+				glFramebufferTextureLayer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, write_depth, 0, v);
 
-				glBlitFramebuffer(0, 0, internal_size.x, internal_size.y, 0, 0, target_size.x, target_size.y, GL_DEPTH_BUFFER_BIT, GL_NEAREST);
+				glBlitFramebuffer(0, 0, internal_size.x, internal_size.y, 0, 0, target_size.x, target_size.y, GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT, GL_NEAREST);
 			}
 		}
 
@@ -3052,6 +3083,47 @@ void RasterizerSceneGLES3::_render_list_template(RenderListParameters *p_params,
 				}
 			}
 
+			// Stencil.
+			if (shader->stencil_enabled) {
+				static const GLenum stencil_compare_table[GLES3::SceneShaderData::STENCIL_COMPARE_MAX] = {
+					GL_LESS,
+					GL_EQUAL,
+					GL_LEQUAL,
+					GL_GREATER,
+					GL_NOTEQUAL,
+					GL_GEQUAL,
+					GL_ALWAYS,
+				};
+
+				GLenum stencil_compare = stencil_compare_table[shader->stencil_compare];
+				GLuint stencil_compare_mask = 0;
+				GLuint stencil_write_mask = 0;
+				GLenum stencil_op_dpfail = GL_KEEP;
+				GLenum stencil_op_dppass = GL_KEEP;
+
+				if (shader->stencil_flags & GLES3::SceneShaderData::STENCIL_FLAG_READ) {
+					stencil_compare_mask = 255;
+				}
+
+				if (shader->stencil_flags & GLES3::SceneShaderData::STENCIL_FLAG_WRITE) {
+					stencil_op_dppass = GL_REPLACE;
+					stencil_write_mask = 255;
+				}
+
+				if (shader->stencil_flags & GLES3::SceneShaderData::STENCIL_FLAG_WRITE_DEPTH_FAIL) {
+					stencil_op_dpfail = GL_REPLACE;
+					stencil_write_mask = 255;
+				}
+
+				scene_state.enable_gl_stencil_test(true);
+				scene_state.set_gl_stencil_func(stencil_compare, shader->stencil_reference, stencil_compare_mask);
+				scene_state.set_gl_stencil_write_mask(stencil_write_mask);
+				scene_state.set_gl_stencil_op(GL_KEEP, stencil_op_dpfail, stencil_op_dppass);
+			} else {
+				scene_state.enable_gl_stencil_test(false);
+				scene_state.set_gl_stencil_write_mask(255);
+			}
+
 			if constexpr (p_pass_mode == PASS_MODE_COLOR || p_pass_mode == PASS_MODE_COLOR_TRANSPARENT) {
 				if (!uses_additive_lighting && pass == 1) {
 					// Don't render additive passes if not using additive lighting.

drivers/gles3/rasterizer_scene_gles3.h

@@ -247,6 +247,7 @@ class RasterizerSceneGLES3 : public RendererSceneRender {
 			FLAG_USES_DEPTH_TEXTURE = 4096,
 			FLAG_USES_NORMAL_TEXTURE = 8192,
 			FLAG_USES_DOUBLE_SIDED_SHADOWS = 16384,
+			FLAG_USES_STENCIL = 32768,
 		};
 
 		union {
@@ -488,6 +489,15 @@ class RasterizerSceneGLES3 : public RendererSceneRender {
 
 			glDepthFunc(GL_GEQUAL);
 			current_depth_function = GL_GEQUAL;
+
+			glDisable(GL_STENCIL_TEST);
+			current_stencil_test_enabled = false;
+			glStencilMask(255);
+			current_stencil_write_mask = 255;
+			glStencilFunc(GL_ALWAYS, 0, 255);
+			current_stencil_compare = GL_ALWAYS;
+			current_stencil_reference = 0;
+			current_stencil_compare_mask = 255;
 		}
 
 		void set_gl_cull_mode(RS::CullMode p_mode) {
@@ -552,10 +562,56 @@ class RasterizerSceneGLES3 : public RendererSceneRender {
 			}
 		}
 
+		void enable_gl_stencil_test(bool p_enabled) {
+			if (current_stencil_test_enabled != p_enabled) {
+				if (p_enabled) {
+					glEnable(GL_STENCIL_TEST);
+				} else {
+					glDisable(GL_STENCIL_TEST);
+				}
+				current_stencil_test_enabled = p_enabled;
+			}
+		}
+
+		void set_gl_stencil_func(GLenum p_compare, GLint p_reference, GLenum p_compare_mask) {
+			if (current_stencil_compare != p_compare || current_stencil_reference != p_reference || current_stencil_compare_mask != p_compare_mask) {
+				glStencilFunc(p_compare, p_reference, p_compare_mask);
+				current_stencil_compare = p_compare;
+				current_stencil_reference = p_reference;
+				current_stencil_compare_mask = p_compare_mask;
+			}
+		}
+
+		void set_gl_stencil_write_mask(GLuint p_mask) {
+			if (current_stencil_write_mask != p_mask) {
+				glStencilMask(p_mask);
+				current_stencil_write_mask = p_mask;
+			}
+		}
+
+		void set_gl_stencil_op(GLenum p_op_fail, GLenum p_op_dpfail, GLenum p_op_dppass) {
+			if (current_stencil_op_fail != p_op_fail || current_stencil_op_dpfail != p_op_dpfail || current_stencil_op_dppass != p_op_dppass) {
+				glStencilOp(p_op_fail, p_op_dpfail, p_op_dppass);
+				current_stencil_op_fail = p_op_fail;
+				current_stencil_op_dpfail = p_op_dpfail;
+				current_stencil_op_dppass = p_op_dppass;
+			}
+		}
+
+		GLenum current_stencil_compare = GL_ALWAYS;
+		GLuint current_stencil_compare_mask = 255;
+		GLuint current_stencil_write_mask = 255;
+		GLint current_stencil_reference = 0;
+		GLenum current_stencil_op_fail = GL_KEEP;
+		GLenum current_stencil_op_dpfail = GL_KEEP;
+		GLenum current_stencil_op_dppass = GL_KEEP;
+		bool current_stencil_test_enabled = false;
+
 		bool texscreen_copied = false;
 		bool used_screen_texture = false;
 		bool used_normal_texture = false;
 		bool used_depth_texture = false;
+		bool used_opaque_stencil = false;
 
 		LightData *omni_lights = nullptr;
 		LightData *spot_lights = nullptr;