Implement rasterization ray-query shadows and refine rendering pipelines:

- Add support for ray-query shadows in raster fragment shaders using TLAS.
- Introduce frustum and LOD culling for planar reflections and TLAS.
- Enhance reflection handling with adjustable intensity and refined tone mapping.
- Improve material and instance masking for ray-query shadows.
- Optimize descriptor binding updates, reducing redundancy.
- Refactor draw queue logic to separate opaque and blended entities for efficiency.
- Extend RayQuery uniforms to support soft shadows and reflection customization.
- Introduce rendering shadows with ray query to all pipelines.

Author: gpx1000

attachments/simple_engine/mesh_component.cpp

@@ -17,6 +17,90 @@
 #include "mesh_component.h"
 #include "model_loader.h"
 #include <cmath>
+#include <limits>
+
+// Helper to transform an AABB by a matrix
+static void transformAABBLocal(const glm::mat4 &M,
+                               const glm::vec3 &localMin,
+                               const glm::vec3 &localMax,
+                               glm::vec3       &outMin,
+                               glm::vec3       &outMax)
+{
+	const glm::vec3 c = 0.5f * (localMin + localMax);
+	const glm::vec3 e = 0.5f * (localMax - localMin);
+
+	const glm::vec3 worldCenter = glm::vec3(M * glm::vec4(c, 1.0f));
+	const glm::mat3 A            = glm::mat3(M);
+	const glm::mat3 AbsA         = glm::mat3(glm::abs(A[0]), glm::abs(A[1]), glm::abs(A[2]));
+	const glm::vec3 worldExtents = AbsA * e;
+
+	outMin = worldCenter - worldExtents;
+	outMax = worldCenter + worldExtents;
+}
+
+void MeshComponent::RecomputeMeshAABB()
+{
+	if (meshAABBValid)
+		return;
+
+	if (vertices.empty())
+	{
+		meshAABBMin   = glm::vec3(0.0f);
+		meshAABBMax   = glm::vec3(0.0f);
+		meshAABBValid = false;
+		return;
+	}
+	glm::vec3 minB = vertices[0].position;
+	glm::vec3 maxB = vertices[0].position;
+	for (const auto &v : vertices)
+	{
+		minB = glm::min(minB, v.position);
+		maxB = glm::max(maxB, v.position);
+	}
+	meshAABBMin   = minB;
+	meshAABBMax   = maxB;
+	meshAABBValid = true;
+}
+
+void MeshComponent::RecomputeLocalAABB()
+{
+	// First ensure base mesh AABB is up to date
+	RecomputeMeshAABB();
+
+	if (!meshAABBValid)
+	{
+		localAABBMin   = glm::vec3(0.0f);
+		localAABBMax   = glm::vec3(0.0f);
+		localAABBValid = false;
+		return;
+	}
+
+	if (instances.empty())
+	{
+		// No instances: local AABB is just the mesh AABB
+		localAABBMin   = meshAABBMin;
+		localAABBMax   = meshAABBMax;
+		localAABBValid = true;
+	}
+	else
+	{
+		// Union of all transformed instance AABBs
+		glm::vec3 fullMin(std::numeric_limits<float>::max());
+		glm::vec3 fullMax(-std::numeric_limits<float>::max());
+
+		for (const auto &inst : instances)
+		{
+			glm::vec3 instMin, instMax;
+			transformAABBLocal(inst.modelMatrix, meshAABBMin, meshAABBMax, instMin, instMax);
+			fullMin = glm::min(fullMin, instMin);
+			fullMax = glm::max(fullMax, instMax);
+		}
+
+		localAABBMin   = fullMin;
+		localAABBMax   = fullMax;
+		localAABBValid = true;
+	}
+}
 
 // Most of the MeshComponent class implementation is in the header file
 // This file is mainly for any methods that might need additional implementation

attachments/simple_engine/mesh_component.h

@@ -18,6 +18,7 @@
 
 #include <array>
 #include <glm/glm.hpp>
+#include <limits>
 #include <string>
 #include <vector>
 
@@ -259,11 +260,16 @@ class MeshComponent final : public Component
 	std::vector<Vertex>   vertices;
 	std::vector<uint32_t> indices;
 
-	// Cached local-space AABB
+	// Cached local-space AABB (encompassing all instances)
 	glm::vec3 localAABBMin{0.0f};
 	glm::vec3 localAABBMax{0.0f};
 	bool      localAABBValid = false;
 
+	// Cached base mesh AABB (vertices only)
+	glm::vec3 meshAABBMin{0.0f};
+	glm::vec3 meshAABBMax{0.0f};
+	bool      meshAABBValid = false;
+
 	// All PBR texture paths for this mesh
 	std::string texturePath;                         // Primary texture path (baseColor) - kept for backward compatibility
 	std::string baseColorTexturePath;                // Base color (albedo) texture
@@ -289,26 +295,8 @@ class MeshComponent final : public Component
 	{}
 
 	// Local AABB utilities
-	void RecomputeLocalAABB()
-	{
-		if (vertices.empty())
-		{
-			localAABBMin   = glm::vec3(0.0f);
-			localAABBMax   = glm::vec3(0.0f);
-			localAABBValid = false;
-			return;
-		}
-		glm::vec3 minB = vertices[0].position;
-		glm::vec3 maxB = vertices[0].position;
-		for (const auto &v : vertices)
-		{
-			minB = glm::min(minB, v.position);
-			maxB = glm::max(maxB, v.position);
-		}
-		localAABBMin   = minB;
-		localAABBMax   = maxB;
-		localAABBValid = true;
-	}
+	void RecomputeLocalAABB();
+	void RecomputeMeshAABB();
 	[[nodiscard]] bool HasLocalAABB() const
 	{
 		return localAABBValid;
@@ -321,14 +309,24 @@ class MeshComponent final : public Component
 	{
 		return localAABBMax;
 	}
+	[[nodiscard]] glm::vec3 GetBaseMeshAABBMin() const
+	{
+		return meshAABBMin;
+	}
+	[[nodiscard]] glm::vec3 GetBaseMeshAABBMax() const
+	{
+		return meshAABBMax;
+	}
 
 	/**
 	 * @brief Set the vertices of the mesh.
 	 * @param newVertices The new vertices.
 	 */
 	void SetVertices(const std::vector<Vertex> &newVertices)
 	{
-		vertices = newVertices;
+		vertices       = newVertices;
+		meshAABBValid  = false;
+		localAABBValid = false;
 		RecomputeLocalAABB();
 	}
 
@@ -447,6 +445,7 @@ class MeshComponent final : public Component
 	{
 		instances.emplace_back(transform, materialIndex);
 		isInstanced = instances.size() > 1;
+		RecomputeLocalAABB();
 	}
 
 	/**
@@ -457,6 +456,7 @@ class MeshComponent final : public Component
 	{
 		instances   = newInstances;
 		isInstanced = instances.size() > 1;
+		RecomputeLocalAABB();
 	}
 
 	/**
@@ -493,6 +493,7 @@ class MeshComponent final : public Component
 	{
 		instances.clear();
 		isInstanced = false;
+		RecomputeLocalAABB();
 	}
 
 	/**
@@ -506,6 +507,7 @@ class MeshComponent final : public Component
 		if (index < instances.size())
 		{
 			instances[index] = InstanceData(transform, materialIndex);
+			RecomputeLocalAABB();
 		}
 	}
 

attachments/simple_engine/renderer.h

@@ -93,6 +93,7 @@ struct LightData
 	alignas(16) glm::vec4 position;                // Light position (w component used for direction vs position)
 	alignas(16) glm::vec4 color;                   // Light color and intensity
 	alignas(16) glm::mat4 lightSpaceMatrix;        // Light space matrix for shadow mapping
+	alignas(16) glm::vec4 direction;               // Light direction (for directional/spotlights)
 	alignas(4) int lightType;                      // 0=Point, 1=Directional, 2=Spot, 3=Emissive
 	alignas(4) float range;                        // Light range
 	alignas(4) float innerConeAngle;               // For spotlights
@@ -169,18 +170,23 @@ struct RayQueryUniformBufferObject
 	alignas(4) int geometryInfoCount;
     alignas(4) int materialCount;
     alignas(4) int _pad0; // used for rayQueryMaxBounces
-    // Thick-glass controls (RQ-only)
-    alignas(4) int enableThickGlass;          // 0/1 toggle
-    alignas(4) float thicknessClamp;          // max thickness in meters
-    alignas(4) float absorptionScale;         // scales sigma_a
-    alignas(4) int _pad1;                     // reserved
-};
-
-static_assert(sizeof(RayQueryUniformBufferObject) == 272, "RayQueryUniformBufferObject size must match shader layout");
-static_assert(offsetof(RayQueryUniformBufferObject, model) == 0);
-static_assert(offsetof(RayQueryUniformBufferObject, view) == 64);
-static_assert(offsetof(RayQueryUniformBufferObject, proj) == 128);
-static_assert(offsetof(RayQueryUniformBufferObject, camPos) == 192);
+   	// Thick-glass controls (RQ-only)
+   	alignas(4) int enableThickGlass;          // 0/1 toggle
+   	alignas(4) float thicknessClamp;          // max thickness in meters
+   	alignas(4) float absorptionScale;         // scales sigma_a
+   	alignas(4) int _pad1;                     // Ray Query: enable hard shadows for direct lighting (0/1)
+   	// Ray Query soft shadows (area-light approximation)
+   	alignas(4) int   shadowSampleCount;        // 1 = hard shadows; >1 = multi-sample
+   	alignas(4) float shadowSoftness;           // 0 = hard; otherwise scales effective light radius (fraction of range)
+   	alignas(4) float reflectionIntensity;      // User control for glass reflection strength
+   	alignas(4) float _padShadow[2]{};
+      };
+
+   static_assert(sizeof(RayQueryUniformBufferObject) == 288, "RayQueryUniformBufferObject size must match shader layout");
+   static_assert(offsetof(RayQueryUniformBufferObject, model) == 0);
+   static_assert(offsetof(RayQueryUniformBufferObject, view) == 64);
+   static_assert(offsetof(RayQueryUniformBufferObject, proj) == 128);
+   static_assert(offsetof(RayQueryUniformBufferObject, camPos) == 192);
 static_assert(offsetof(RayQueryUniformBufferObject, exposure) == 208);
 static_assert(offsetof(RayQueryUniformBufferObject, gamma) == 212);
 static_assert(offsetof(RayQueryUniformBufferObject, scaleIBLAmbient) == 216);
@@ -195,6 +201,8 @@ static_assert(offsetof(RayQueryUniformBufferObject, enableThickGlass) == 252);
 static_assert(offsetof(RayQueryUniformBufferObject, thicknessClamp) == 256);
 static_assert(offsetof(RayQueryUniformBufferObject, absorptionScale) == 260);
 static_assert(offsetof(RayQueryUniformBufferObject, _pad1) == 264);
+static_assert(offsetof(RayQueryUniformBufferObject, shadowSampleCount) == 268);
+static_assert(offsetof(RayQueryUniformBufferObject, shadowSoftness) == 272);
 
 /**
  * @brief Structure for PBR material properties.
@@ -891,7 +899,7 @@ class Renderer
 	bool buildAccelerationStructures(const std::vector<Entity *> &entities);
 
 	// Refit/UPDATE the TLAS with latest entity transforms (no rebuild)
-	bool refitTopLevelAS(const std::vector<Entity *> &entities);
+	bool refitTopLevelAS(const std::vector<Entity *> &entities, CameraComponent *camera);
 
 	/**
 	 * @brief Update ray query descriptor sets with current resources.
@@ -1003,9 +1011,15 @@ class Renderer
 	float gamma               = 2.2f;        // Gamma correction value
 	float exposure            = 1.2f;        // HDR exposure value (default tuned to avoid washout)
 	float reflectionIntensity = 1.0f;        // User control for glass reflection strength
+	// Raster shadows (experimental): use ray queries in the raster PBR fragment shader.
+	// Wired through `UniformBufferObject.padding2` to avoid UBO layout churn.
+	bool enableRasterRayQueryShadows = false;
 
 	// Ray Query tuning
-	int rayQueryMaxBounces = 1;        // 0 = no secondary rays, 1 = one-bounce reflection/refraction
+	int  rayQueryMaxBounces = 1;            // 0 = no secondary rays, 1 = one-bounce reflection/refraction
+	bool enableRayQueryShadows = true;      // Hard shadows for Ray Query direct lighting (shadow rays)
+	int  rayQueryShadowSampleCount = 1;     // 1 = hard; >1 enables soft-shadow sampling in the shader
+	float rayQueryShadowSoftness = 0.0f;    // 0 = hard; otherwise scales effective light radius (fraction of range)
 	// Thick-glass controls (RQ-only)
 	bool  enableThickGlass             = true;
 	float thickGlassAbsorptionScale    = 1.0f;
@@ -1563,6 +1577,13 @@ class Renderer
 		std::vector<bool> pbrImagesWritten;          // size = MAX_FRAMES_IN_FLIGHT
 		std::vector<bool> basicImagesWritten;        // size = MAX_FRAMES_IN_FLIGHT
 
+		// Tracks whether the remaining required bindings in the PBR set 0 layout have
+		// been written at least once for each frame.
+		// This includes bindings like Forward+ tile buffers (7/8), reflection sampler (10),
+		// and TLAS (11). These bindings are required by the pipeline layout and must be
+		// valid before any draw that uses the PBR/glass pipelines.
+		std::vector<bool> pbrFixedBindingsWritten;        // size = MAX_FRAMES_IN_FLIGHT
+
 		// Cached material lookup/classification for raster rendering.
 		// Avoids per-frame string parsing of entity names ("_Material_") and repeated
 		// ModelLoader material lookups across culling, sorting, and draw loops.

attachments/simple_engine/renderer_pipelines.cpp

@@ -159,12 +159,19 @@ bool Renderer::createPBRDescriptorSetLayout()
 		        .descriptorType     = vk::DescriptorType::eCombinedImageSampler,
 		        .descriptorCount    = 1,
 		        .stageFlags         = vk::ShaderStageFlagBits::eFragment,
+		        .pImmutableSamplers = nullptr},
+		    // Binding 11: TLAS (ray-query shadows in raster fragment shader)
+		    vk::DescriptorSetLayoutBinding{
+		        .binding            = 11,
+		        .descriptorType     = vk::DescriptorType::eAccelerationStructureKHR,
+		        .descriptorCount    = 1,
+		        .stageFlags         = vk::ShaderStageFlagBits::eFragment,
 		        .pImmutableSamplers = nullptr}};
 
 		// Create a descriptor set layout
 		// Descriptor indexing: set per-binding flags for UPDATE_AFTER_BIND on UBO (0) and sampled images (1..5)
 		vk::DescriptorSetLayoutBindingFlagsCreateInfo bindingFlagsInfo{};
-		std::array<vk::DescriptorBindingFlags, 11>    bindingFlags{};
+		std::array<vk::DescriptorBindingFlags, 12>    bindingFlags{};
 		if (descriptorIndexingEnabled)
 		{
 			bindingFlags[0] = vk::DescriptorBindingFlagBits::eUpdateAfterBind | vk::DescriptorBindingFlagBits::eUpdateUnusedWhilePending;