Improve Watchdog functionality and mesh resource handling

- Added progress tracking to Watchdog for better diagnostics during long operations.
- Enhanced Vulkan extension support checks with reusable helper.
- Optimized fencing and timeline semaphore handling for resource uploads.
- Improved staging buffer usage and batch processing for entity resources.

The windows debug now loads in under a minute and displays ray query as default and at 80fps.  Forward+ causes raster pass to average around 30fps while normal PBR raster, phong and no culling are all around 50 fps.

Author: gpx1000

attachments/simple_engine/crash_reporter.h

@@ -16,6 +16,7 @@
  */
 #pragma once
 
+#include <atomic>
 #include <chrono>
 #include <cstring>
 #include <ctime>
@@ -25,10 +26,11 @@
 #include <mutex>
 #include <string>
 #include <thread>
+#include <unordered_map>
 
 #ifdef _WIN32
-#	include <dbghelp.h>
 #	include <windows.h>
+#	include <dbghelp.h>
 #	pragma comment(lib, "dbghelp.lib")
 #elif defined(__APPLE__) || defined(__linux__)
 #	include <execinfo.h>
@@ -118,18 +120,7 @@ class CrashReporter
 	 */
 	void HandleCrash(const std::string &message)
 	{
-		std::lock_guard<std::mutex> lock(mutex);
-
-		LOG_FATAL("CrashReporter", "Crash detected: " + message);
-
-		// Generate minidump
-		GenerateMinidump(message);
-
-		// Call registered callbacks
-		for (const auto &callback : crashCallbacks)
-		{
-			callback(message);
-		}
+		HandleCrashInternal(message, nullptr);
 	}
 
 	/**
@@ -161,7 +152,7 @@ class CrashReporter
 	 * @brief Generate a minidump.
 	 * @param message The crash message.
 	 */
-	void GenerateMinidump(const std::string &message)
+	void GenerateMinidump(const std::string &message, void *platformExceptionPointers = nullptr)
 	{
 		// Get current time for filename
 		auto now  = std::chrono::system_clock::now();
@@ -171,12 +162,37 @@ class CrashReporter
 
 		// Create minidump filename
 		std::string filename = minidumpDir + "/" + appName + "_" + timeStr + ".dmp";
+		std::string report   = minidumpDir + "/" + appName + "_" + timeStr + ".txt";
 
-		LOG_INFO("CrashReporter", "Generating minidump: " + filename);
+		// Also write a small sidecar text file so users can quickly see the exception code/address
+		// without needing a debugger.
+		try
+		{
+			std::ofstream rep(report, std::ios::out | std::ios::trunc);
+			rep << "Crash Report for " << appName << " " << appVersion << "\n";
+			rep << "Timestamp: " << timeStr << "\n";
+			rep << "Message: " << message << "\n";
+#ifdef _WIN32
+			if (platformExceptionPointers)
+			{
+				auto *exPtrs = reinterpret_cast<EXCEPTION_POINTERS *>(platformExceptionPointers);
+				if (exPtrs && exPtrs->ExceptionRecord)
+				{
+					rep << "ExceptionCode: 0x" << std::hex << exPtrs->ExceptionRecord->ExceptionCode << std::dec << "\n";
+					rep << "ExceptionAddress: " << exPtrs->ExceptionRecord->ExceptionAddress << "\n";
+					rep << "ExceptionFlags: 0x" << std::hex << exPtrs->ExceptionRecord->ExceptionFlags << std::dec << "\n";
+				}
+			}
+#endif
+		}
+		catch (...)
+		{
+		}
 
 // Generate minidump based on platform
 #ifdef _WIN32
 		// Windows implementation
+		EXCEPTION_POINTERS *exPtrs = reinterpret_cast<EXCEPTION_POINTERS *>(platformExceptionPointers);
 		HANDLE hFile = CreateFileA(
 		    filename.c_str(),
 		    GENERIC_WRITE,
@@ -188,19 +204,19 @@ class CrashReporter
 
 		if (hFile != INVALID_HANDLE_VALUE)
 		{
-			MINIDUMP_EXCEPTION_INFORMATION exInfo;
+			MINIDUMP_EXCEPTION_INFORMATION exInfo{};
 			exInfo.ThreadId          = GetCurrentThreadId();
-			exInfo.ExceptionPointers = NULL;        // Would be set in a real exception handler
+			exInfo.ExceptionPointers = exPtrs;
 			exInfo.ClientPointers    = FALSE;
 
-			MiniDumpWriteDump(
-			    GetCurrentProcess(),
-			    GetCurrentProcessId(),
-			    hFile,
-			    MiniDumpNormal,
-			    &exInfo,
-			    NULL,
-			    NULL);
+			MINIDUMP_EXCEPTION_INFORMATION *exInfoPtr = exPtrs ? &exInfo : nullptr;
+			MiniDumpWriteDump(GetCurrentProcess(),
+			                  GetCurrentProcessId(),
+			                  hFile,
+			                  MiniDumpNormal,
+			                  exInfoPtr,
+			                  NULL,
+			                  NULL);
 
 			CloseHandle(hFile);
 		}
@@ -232,7 +248,10 @@ class CrashReporter
 		}
 #endif
 
-		LOG_INFO("CrashReporter", "Minidump generated: " + filename);
+		// Best-effort stderr note (stdout/stderr redirection will capture this even if DebugSystem isn't initialized)
+		std::fprintf(stderr, "[CrashReporter] Wrote minidump: %s\n", filename.c_str());
+		std::fprintf(stderr, "[CrashReporter] Wrote report:  %s\n", report.c_str());
+		std::fflush(stderr);
 	}
 
   private:
@@ -259,6 +278,80 @@ class CrashReporter
 	// Crash callbacks
 	std::unordered_map<int, std::function<void(const std::string &)>> crashCallbacks;
 	int                                                               nextCallbackId = 0;
+	std::atomic<bool>                                                 handlingCrash{false};
+
+#ifdef _WIN32
+	static bool ShouldCaptureException(EXCEPTION_POINTERS *exInfo, bool unhandled)
+	{
+		if (unhandled)
+			return true;
+		if (!exInfo || !exInfo->ExceptionRecord)
+			return false;
+		const DWORD code  = exInfo->ExceptionRecord->ExceptionCode;
+		const DWORD flags = exInfo->ExceptionRecord->ExceptionFlags;
+		// Ignore common first-chance C++ exceptions and breakpoint exceptions.
+		if (code == 0xE06D7363u /* MSVC C++ EH */ || code == 0x80000003u /* breakpoint */)
+			return false;
+		// Capture likely-fatal errors and non-continuable exceptions.
+		if ((flags & EXCEPTION_NONCONTINUABLE) != 0)
+			return true;
+		switch (code)
+		{
+			case 0xC0000409u: // STATUS_STACK_BUFFER_OVERRUN
+			case 0xC0000005u: // STATUS_ACCESS_VIOLATION
+			case 0xC000001Du: // STATUS_ILLEGAL_INSTRUCTION
+			case 0xC00000FDu: // STATUS_STACK_OVERFLOW
+			case 0xC0000374u: // STATUS_HEAP_CORRUPTION
+				return true;
+			default:
+				return false;
+		}
+	}
+#endif
+
+#ifdef _WIN32
+	void *vectoredHandlerHandle = nullptr;
+#endif
+
+	void HandleCrashInternal(const std::string &message, void *platformExceptionPointers)
+	{
+		bool expected = false;
+		if (!handlingCrash.compare_exchange_strong(expected, true))
+		{
+			// Already handling a crash; avoid recursion.
+			return;
+		}
+		std::lock_guard<std::mutex> lock(mutex);
+
+		std::string msg = message;
+		(void) platformExceptionPointers;
+
+#ifdef _WIN32
+		if (platformExceptionPointers)
+		{
+			auto *exPtrs = reinterpret_cast<EXCEPTION_POINTERS *>(platformExceptionPointers);
+			if (exPtrs && exPtrs->ExceptionRecord)
+			{
+				const DWORD code = exPtrs->ExceptionRecord->ExceptionCode;
+				void *addr       = exPtrs->ExceptionRecord->ExceptionAddress;
+				char buf[128];
+				std::snprintf(buf, sizeof(buf), " (code=0x%08lX, addr=%p)", static_cast<unsigned long>(code), addr);
+				msg += buf;
+			}
+		}
+#endif
+
+		LOG_FATAL("CrashReporter", "Crash detected: " + msg);
+
+		// Generate minidump
+		GenerateMinidump(msg, platformExceptionPointers);
+
+		// Call registered callbacks
+		for (const auto &callback : crashCallbacks)
+		{
+			callback.second(msg);
+		}
+	}
 
 	/**
 	 * @brief Install platform-specific crash handlers.
@@ -267,8 +360,16 @@ class CrashReporter
 	{
 #ifdef _WIN32
 		// Windows implementation
+		// Vectored handler runs before SEH/unhandled filters and is more likely to fire for fast-fail style crashes.
+		vectoredHandlerHandle = AddVectoredExceptionHandler(1, [](EXCEPTION_POINTERS *exInfo) -> LONG {
+			if (CrashReporter::ShouldCaptureException(exInfo, /*unhandled=*/false))
+			{
+				CrashReporter::GetInstance().HandleCrashInternal("Vectored exception", exInfo);
+			}
+			return EXCEPTION_CONTINUE_SEARCH;
+		});
 		SetUnhandledExceptionFilter([](EXCEPTION_POINTERS *exInfo) -> LONG {
-			CrashReporter::GetInstance().HandleCrash("Unhandled exception");
+			CrashReporter::GetInstance().HandleCrashInternal("Unhandled exception", exInfo);
 			return EXCEPTION_EXECUTE_HANDLER;
 		});
 #else
@@ -303,6 +404,11 @@ class CrashReporter
 #ifdef _WIN32
 		// Windows implementation
 		SetUnhandledExceptionFilter(NULL);
+		if (vectoredHandlerHandle)
+		{
+			RemoveVectoredExceptionHandler(vectoredHandlerHandle);
+			vectoredHandlerHandle = nullptr;
+		}
 #else
 		// Unix implementation
 		signal(SIGSEGV, SIG_DFL);

attachments/simple_engine/imgui_system.cpp

@@ -165,13 +165,11 @@ void ImGuiSystem::NewFrame()
 
 	ImGui::NewFrame();
 
-	// Loading overlay: show only a fullscreen progress bar while the model
-	// itself is loading. Once the scene is ready and geometry is visible,
-	// we no longer block the view with a full-screen progress bar.
+	// Loading overlay: show a fullscreen progress bar while the initial scene is loading.
+	// The bar resets between phases (Textures -> Physics -> AS -> Finalizing) so users
+	// don't stare at a 100% bar while the engine is still doing work.
 	if (renderer)
 	{
-		const uint32_t scheduled    = renderer->GetTextureTasksScheduled();
-		const uint32_t completed    = renderer->GetTextureTasksCompleted();
 		const bool     modelLoading = renderer->IsLoading();
 		if (modelLoading)
 		{
@@ -202,11 +200,40 @@ void ImGuiSystem::NewFrame()
 				const float barY     = dispSize.y * 0.45f;
 				ImGui::SetCursorPos(ImVec2(barX, barY));
 				ImGui::BeginGroup();
-				float frac = (scheduled > 0) ? (float) completed / (float) scheduled : 0.0f;
+
+				// Phase-aware progress (resets between phases).
+				float frac = 0.0f;
+				auto  phase = renderer->GetLoadingPhase();
+				if (phase == Renderer::LoadingPhase::Textures)
+				{
+					const uint32_t scheduled = renderer->GetTextureTasksScheduled();
+					const uint32_t completed = renderer->GetTextureTasksCompleted();
+					frac = (scheduled > 0) ? (static_cast<float>(completed) / static_cast<float>(scheduled)) : 0.0f;
+				}
+				else if (phase == Renderer::LoadingPhase::AccelerationStructures)
+				{
+					frac = renderer->GetASBuildProgress();
+				}
+				else
+				{
+					frac = renderer->GetLoadingPhaseProgress();
+				}
 				ImGui::ProgressBar(frac, ImVec2(barWidth, 0.0f));
 				ImGui::Dummy(ImVec2(0.0f, 10.0f));
 				ImGui::SetCursorPosX(barX);
-				ImGui::Text("Loading scene...");
+				ImGui::Text("Loading: %s", renderer->GetLoadingPhaseName());
+				if (phase == Renderer::LoadingPhase::Textures)
+				{
+					const uint32_t scheduled = renderer->GetTextureTasksScheduled();
+					const uint32_t completed = renderer->GetTextureTasksCompleted();
+					ImGui::Text("Textures: %u/%u", completed, scheduled);
+				}
+				else if (phase == Renderer::LoadingPhase::AccelerationStructures)
+				{
+					const uint32_t done  = renderer->GetASBuildItemsDone();
+					const uint32_t total = renderer->GetASBuildItemsTotal();
+					ImGui::Text("%s (%u/%u, %.1fs)", renderer->GetASBuildStage(), done, total, renderer->GetASBuildElapsedSeconds());
+				}
 				ImGui::EndGroup();
 				ImGui::PopStyleVar();
 			}
@@ -226,6 +253,41 @@ void ImGuiSystem::NewFrame()
 		const uint32_t uploadTotal  = renderer->GetUploadJobsTotal();
 		const uint32_t uploadDone   = renderer->GetUploadJobsCompleted();
 		const bool     modelLoading = renderer->IsLoading();
+		const bool     showASBuild  = renderer->ShouldShowASBuildProgressInUI();
+
+		// Acceleration structure build can happen after initial load completes.
+		// If it takes a long time, show a compact progress window.
+		if (!modelLoading && showASBuild)
+		{
+			ImGuiIO     &io       = ImGui::GetIO();
+			const ImVec2 dispSize = io.DisplaySize;
+
+			const float  windowWidth  = std::min(320.0f, dispSize.x * 0.42f);
+			const float  windowHeight = 90.0f;
+			const ImVec2 winPos(dispSize.x - windowWidth - 10.0f, 10.0f);
+
+			ImGui::SetNextWindowPos(winPos, ImGuiCond_Always);
+			ImGui::SetNextWindowSize(ImVec2(windowWidth, windowHeight));
+			ImGuiWindowFlags flags = ImGuiWindowFlags_NoResize |
+			                         ImGuiWindowFlags_NoMove |
+			                         ImGuiWindowFlags_NoCollapse |
+			                         ImGuiWindowFlags_NoSavedSettings;
+
+			if (ImGui::Begin("##ASBuildStatus", nullptr, flags))
+			{
+				ImGui::Text("Building acceleration structures...");
+				const float asFrac = renderer->GetASBuildProgress();
+				ImGui::ProgressBar(asFrac, ImVec2(-1.0f, 0.0f));
+				const uint32_t done  = renderer->GetASBuildItemsDone();
+				const uint32_t total = renderer->GetASBuildItemsTotal();
+				ImGui::Text("%s (%u/%u, %.1fs)",
+				           renderer->GetASBuildStage(),
+				           done,
+				           total,
+				           renderer->GetASBuildElapsedSeconds());
+			}
+			ImGui::End();
+		}
 
 		if (!modelLoading && uploadTotal > 0 && uploadDone < uploadTotal)
 		{
@@ -234,7 +296,9 @@ void ImGuiSystem::NewFrame()
 
 			const float  windowWidth  = std::min(260.0f, dispSize.x * 0.35f);
 			const float  windowHeight = 120.0f;
-			const ImVec2 winPos(dispSize.x - windowWidth - 10.0f, 10.0f);
+			// If the AS build status window is visible, offset streaming window below it.
+			const float  yBase        = 10.0f + (showASBuild ? (90.0f + 10.0f) : 0.0f);
+			const ImVec2 winPos(dispSize.x - windowWidth - 10.0f, yBase);
 
 			ImGui::SetNextWindowPos(winPos, ImGuiCond_Always);
 			ImGui::SetNextWindowSize(ImVec2(windowWidth, windowHeight));

attachments/simple_engine/main.cpp

@@ -15,6 +15,7 @@
  * limitations under the License.
  */
 #include "camera_component.h"
+#include "crash_reporter.h"
 #include "engine.h"
 #include "scene_loading.h"
 #include "transform_component.h"
@@ -63,6 +64,7 @@ void SetupScene(Engine *engine)
 	if (auto *renderer = engine->GetRenderer())
 	{
 		renderer->SetLoading(true);
+		renderer->SetLoadingPhase(Renderer::LoadingPhase::Textures);
 	}
 	std::thread([engine] {
 		LoadGLTFModel(engine, "../Assets/bistro/bistro.gltf");
@@ -107,6 +109,10 @@ int main(int, char *[])
 {
 	try
 	{
+		// Enable minidump generation for Release-only crashes (e.g., stack cookie failures / fast-fail).
+		// Writes dumps under the current working directory (the build/run directory).
+		CrashReporter::GetInstance().Initialize("crashes", "SimpleEngine", "1.0.0");
+
 		// Create the engine
 		Engine engine;
 
@@ -122,11 +128,14 @@ int main(int, char *[])
 		// Run the engine
 		engine.Run();
 
+		CrashReporter::GetInstance().Cleanup();
+
 		return 0;
 	}
 	catch (const std::exception &e)
 	{
 		std::cerr << "Exception: " << e.what() << std::endl;
+		CrashReporter::GetInstance().Cleanup();
 		return 1;
 	}
 }