From 557387375fc0f3ed41a81ccc64fb2022a0c21163 Mon Sep 17 00:00:00 2001 From: Benjamin Doherty Date: Tue, 24 Feb 2026 10:02:12 -0800 Subject: [PATCH 1/3] Print command stream histogram before crashing --- .../private/backend/CommandBufferQueue.h | 4 +- .../include/private/backend/CommandStream.h | 37 +++++++++++++- filament/backend/src/CommandBufferQueue.cpp | 7 ++- filament/backend/src/CommandStream.cpp | 50 +++++++++++++++++++ filament/src/details/Engine.cpp | 4 +- 5 files changed, 97 insertions(+), 5 deletions(-) diff --git a/filament/backend/include/private/backend/CommandBufferQueue.h b/filament/backend/include/private/backend/CommandBufferQueue.h index a33778b93323..5405915c9cbb 100644 --- a/filament/backend/include/private/backend/CommandBufferQueue.h +++ b/filament/backend/include/private/backend/CommandBufferQueue.h @@ -27,6 +27,8 @@ #include #include +#include + namespace filament::backend { /* @@ -64,7 +66,7 @@ class CommandBufferQueue { // all commands buffers (Slices) written to this point are returned by waitForCommand(). This // call blocks until the CircularBuffer has at least mRequiredSize bytes available. - void flush(); + void flush(std::function const& debugPrintHistogram = nullptr); // returns from waitForCommands() immediately. void requestExit(); diff --git a/filament/backend/include/private/backend/CommandStream.h b/filament/backend/include/private/backend/CommandStream.h index 5b288ef7bd44..e6875ecc9478 100644 --- a/filament/backend/include/private/backend/CommandStream.h +++ b/filament/backend/include/private/backend/CommandStream.h @@ -56,6 +56,7 @@ namespace filament::backend { class CommandBase { static constexpr size_t FILAMENT_OBJECT_ALIGNMENT = alignof(std::max_align_t); + friend class CommandStream; protected: using Execute = Dispatcher::Execute; @@ -168,8 +169,8 @@ struct CommandType { class CustomCommand : public CommandBase { std::function mCommand; - static void execute(Driver&, CommandBase* base, intptr_t* next); public: + static void execute(Driver&, CommandBase* base, intptr_t* next); CustomCommand(CustomCommand&& rhs) = default; explicit CustomCommand(std::function cmd) @@ -179,11 +180,12 @@ class CustomCommand : public CommandBase { // ------------------------------------------------------------------------------------------------ class NoopCommand : public CommandBase { +public: intptr_t mNext; static void execute(Driver&, CommandBase* self, intptr_t* next) noexcept { *next = static_cast(self)->mNext; } -public: + constexpr explicit NoopCommand(void* next) noexcept : CommandBase(execute), mNext(intptr_t((char *)next - (char *)this)) { } }; @@ -219,6 +221,32 @@ class CommandStream { CircularBuffer const& getCircularBuffer() const noexcept { return mCurrentBuffer; } + using Execute = Dispatcher::Execute; + struct CommandInfo { + size_t size; + const char* name; + }; + std::unordered_map mCommands; + + void initializeLookup() { +#define DECL_DRIVER_API_SYNCHRONOUS(RetType, methodName, paramsDecl, params) +#define DECL_DRIVER_API(methodName, paramsDecl, params) \ + mCommands[mDispatcher.methodName##_] = { CommandBase::align(sizeof(COMMAND_TYPE(methodName))), \ + #methodName }; +#define DECL_DRIVER_API_RETURN(RetType, methodName, paramsDecl, params) \ + mCommands[mDispatcher.methodName##_] = { \ + CommandBase::align(sizeof(COMMAND_TYPE(methodName##R))), #methodName \ + }; + +#include "private/backend/DriverAPI.inc" + + mCommands[CustomCommand::execute] = { CommandBase::align(sizeof(CustomCommand)), + "CustomCommand" }; + + // NoopCommands have variable size. We will handle them specially using their mNext pointer. + mCommands[NoopCommand::execute] = { 0, "NoopCommand" }; + } + public: #define DECL_DRIVER_API(methodName, paramsDecl, params) \ inline void methodName(paramsDecl) noexcept { \ @@ -263,6 +291,11 @@ class CommandStream { void execute(void* buffer); + void debugIterateCommands(void* head, void* tail, + std::function const& callback); + + void debugPrintHistogram(void* head, void* tail); + /* * queueCommand() allows to queue a lambda function as a command. * This is much less efficient than using the Driver* API. diff --git a/filament/backend/src/CommandBufferQueue.cpp b/filament/backend/src/CommandBufferQueue.cpp index 5ea8aacd03a9..3a2f31b87eb5 100644 --- a/filament/backend/src/CommandBufferQueue.cpp +++ b/filament/backend/src/CommandBufferQueue.cpp @@ -79,7 +79,7 @@ bool CommandBufferQueue::isExitRequested() const { } -void CommandBufferQueue::flush() { +void CommandBufferQueue::flush(std::function const& debugPrintHistogram) { FILAMENT_TRACING_CALL(FILAMENT_TRACING_CATEGORY_FILAMENT); CircularBuffer& circularBuffer = mCircularBuffer; @@ -106,6 +106,11 @@ void CommandBufferQueue::flush() { std::unique_lock lock(mLock); // circular buffer is too small, we corrupted the stream + if (UTILS_VERY_UNLIKELY(used > mFreeSpace)) { + if (debugPrintHistogram) { + debugPrintHistogram(begin, end); + } + } FILAMENT_CHECK_POSTCONDITION(used <= mFreeSpace) << "Backend CommandStream overflow. Commands are corrupted and unrecoverable.\n" "Please increase minCommandBufferSizeMB inside the Config passed to Engine::create.\n" diff --git a/filament/backend/src/CommandStream.cpp b/filament/backend/src/CommandStream.cpp index 4266683f6204..0f9414ba6cd6 100644 --- a/filament/backend/src/CommandStream.cpp +++ b/filament/backend/src/CommandStream.cpp @@ -30,10 +30,12 @@ #include #include +#include #include #include #include #include +#include #ifdef __ANDROID__ #include @@ -83,6 +85,8 @@ CommandStream::CommandStream(Driver& driver, CircularBuffer& buffer) noexcept __system_property_get("debug.filament.perfcounters", property); mUsePerformanceCounter = bool(atoi(property)); #endif + + initializeLookup(); } void CommandStream::execute(void* buffer) { @@ -126,6 +130,52 @@ void CommandStream::execute(void* buffer) { } } +void CommandStream::debugIterateCommands(void* head, void* tail, + std::function const& callback) { + CommandBase* UTILS_RESTRICT base = static_cast(head); + auto p = base; + while (UTILS_LIKELY(p)) { + if (p >= tail) { + break; + } + Execute e = p->mExecute; + + if (e == NoopCommand::execute) { + NoopCommand* noop = static_cast(p); + size_t size = noop->mNext; + callback({ size, "NoopCommand" }); + p = reinterpret_cast(reinterpret_cast(p) + size); + continue; + } + + if (auto it = mCommands.find(e); it != mCommands.end()) { + size_t size = it->second.size; + callback(it->second); + p = reinterpret_cast(reinterpret_cast(p) + size); + } else { + LOG(ERROR) << "Cannot find command in lookup table"; + return; + } + } +} + +void CommandStream::debugPrintHistogram(void* head, void* tail) { + std::unordered_map histogram; + debugIterateCommands(head, tail, + [&](CommandInfo const& info) { histogram[std::string_view(info.name)]++; }); + + std::vector> sorted_histogram(histogram.begin(), + histogram.end()); + std::sort(sorted_histogram.begin(), sorted_histogram.end(), + [](auto const& a, auto const& b) { return a.second > b.second; }); + + LOG(INFO) << "Command stream histogram:"; + for (auto const& [name, count]: sorted_histogram) { + LOG(INFO) << name << ": " << count; + } + LOG(INFO) << ""; +} + void CommandStream::queueCommand(std::function command) { new(allocateCommand(CustomCommand::align(sizeof(CustomCommand)))) CustomCommand(std::move(command)); } diff --git a/filament/src/details/Engine.cpp b/filament/src/details/Engine.cpp index 4010eb30db17..ce402117efcc 100644 --- a/filament/src/details/Engine.cpp +++ b/filament/src/details/Engine.cpp @@ -920,7 +920,9 @@ int FEngine::loop() { void FEngine::flushCommandBuffer(CommandBufferQueue& commandBufferQueue) const { getDriver().purge(); - commandBufferQueue.flush(); + commandBufferQueue.flush([this](void* begin, void* end) { + const_cast(this)->getDriverApi().debugPrintHistogram(begin, end); + }); } const FMaterial* FEngine::getSkyboxMaterial() const noexcept { From 7d3b8eb7b92f2afc9f155432ef154ebd5e022e24 Mon Sep 17 00:00:00 2001 From: Benjamin Doherty Date: Tue, 24 Feb 2026 11:31:31 -0800 Subject: [PATCH 2/3] Print a compressed histogram --- .../include/private/backend/CommandStream.h | 10 ++++---- filament/backend/src/CommandStream.cpp | 23 ++++++++++++++++--- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/filament/backend/include/private/backend/CommandStream.h b/filament/backend/include/private/backend/CommandStream.h index e6875ecc9478..a7ef4b5891de 100644 --- a/filament/backend/include/private/backend/CommandStream.h +++ b/filament/backend/include/private/backend/CommandStream.h @@ -225,26 +225,28 @@ class CommandStream { struct CommandInfo { size_t size; const char* name; + int index; }; std::unordered_map mCommands; void initializeLookup() { + int currentIndex = 0; #define DECL_DRIVER_API_SYNCHRONOUS(RetType, methodName, paramsDecl, params) #define DECL_DRIVER_API(methodName, paramsDecl, params) \ mCommands[mDispatcher.methodName##_] = { CommandBase::align(sizeof(COMMAND_TYPE(methodName))), \ - #methodName }; + #methodName, currentIndex++ }; #define DECL_DRIVER_API_RETURN(RetType, methodName, paramsDecl, params) \ mCommands[mDispatcher.methodName##_] = { \ - CommandBase::align(sizeof(COMMAND_TYPE(methodName##R))), #methodName \ + CommandBase::align(sizeof(COMMAND_TYPE(methodName##R))), #methodName, currentIndex++ \ }; #include "private/backend/DriverAPI.inc" mCommands[CustomCommand::execute] = { CommandBase::align(sizeof(CustomCommand)), - "CustomCommand" }; + "CustomCommand", currentIndex++ }; // NoopCommands have variable size. We will handle them specially using their mNext pointer. - mCommands[NoopCommand::execute] = { 0, "NoopCommand" }; + mCommands[NoopCommand::execute] = { 0, "NoopCommand", currentIndex++ }; } public: diff --git a/filament/backend/src/CommandStream.cpp b/filament/backend/src/CommandStream.cpp index 0f9414ba6cd6..39c473293858 100644 --- a/filament/backend/src/CommandStream.cpp +++ b/filament/backend/src/CommandStream.cpp @@ -143,7 +143,8 @@ void CommandStream::debugIterateCommands(void* head, void* tail, if (e == NoopCommand::execute) { NoopCommand* noop = static_cast(p); size_t size = noop->mNext; - callback({ size, "NoopCommand" }); + int noopIndex = mCommands[NoopCommand::execute].index; + callback({ size, "NoopCommand", noopIndex }); p = reinterpret_cast(reinterpret_cast(p) + size); continue; } @@ -161,8 +162,11 @@ void CommandStream::debugIterateCommands(void* head, void* tail, void CommandStream::debugPrintHistogram(void* head, void* tail) { std::unordered_map histogram; - debugIterateCommands(head, tail, - [&](CommandInfo const& info) { histogram[std::string_view(info.name)]++; }); + std::unordered_map index_histogram; + debugIterateCommands(head, tail, [&](CommandInfo const& info) { + histogram[std::string_view(info.name)]++; + index_histogram[info.index]++; + }); std::vector> sorted_histogram(histogram.begin(), histogram.end()); @@ -173,6 +177,19 @@ void CommandStream::debugPrintHistogram(void* head, void* tail) { for (auto const& [name, count]: sorted_histogram) { LOG(INFO) << name << ": " << count; } + + std::vector> sorted_index_histogram(index_histogram.begin(), + index_histogram.end()); + std::sort(sorted_index_histogram.begin(), sorted_index_histogram.end(), + [](auto const& a, auto const& b) { return a.second > b.second; }); + + std::string short_histogram = ""; + for (size_t i = 0, n = sorted_index_histogram.size(); i < n; ++i) { + short_histogram += std::to_string(sorted_index_histogram[i].first) + ":" + + std::to_string(sorted_index_histogram[i].second); + short_histogram += (i < n - 1) ? ";" : "."; + } + LOG(INFO) << "CS hist: " << short_histogram; LOG(INFO) << ""; } From 2730fbc31b084b06037383762e48a05c5df3e247 Mon Sep 17 00:00:00 2001 From: Benjamin Doherty Date: Tue, 24 Feb 2026 12:41:24 -0800 Subject: [PATCH 3/3] Add FILAMENT_DEBUG_COMMANDS_HISTOGRAM compile flag --- filament/backend/include/private/backend/CommandStream.h | 4 ++++ filament/backend/include/private/backend/Driver.h | 5 +++++ filament/backend/src/CommandBufferQueue.cpp | 2 ++ filament/backend/src/CommandStream.cpp | 9 ++++++++- filament/src/details/Engine.cpp | 5 ++++- 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/filament/backend/include/private/backend/CommandStream.h b/filament/backend/include/private/backend/CommandStream.h index a7ef4b5891de..a6e58676eb9b 100644 --- a/filament/backend/include/private/backend/CommandStream.h +++ b/filament/backend/include/private/backend/CommandStream.h @@ -221,6 +221,7 @@ class CommandStream { CircularBuffer const& getCircularBuffer() const noexcept { return mCurrentBuffer; } +#if FILAMENT_DEBUG_COMMANDS_HISTOGRAM using Execute = Dispatcher::Execute; struct CommandInfo { size_t size; @@ -248,6 +249,7 @@ class CommandStream { // NoopCommands have variable size. We will handle them specially using their mNext pointer. mCommands[NoopCommand::execute] = { 0, "NoopCommand", currentIndex++ }; } +#endif public: #define DECL_DRIVER_API(methodName, paramsDecl, params) \ @@ -293,10 +295,12 @@ class CommandStream { void execute(void* buffer); +#if FILAMENT_DEBUG_COMMANDS_HISTOGRAM void debugIterateCommands(void* head, void* tail, std::function const& callback); void debugPrintHistogram(void* head, void* tail); +#endif /* * queueCommand() allows to queue a lambda function as a command. diff --git a/filament/backend/include/private/backend/Driver.h b/filament/backend/include/private/backend/Driver.h index ed2ff9b27c97..c2bfa19b94e9 100644 --- a/filament/backend/include/private/backend/Driver.h +++ b/filament/backend/include/private/backend/Driver.h @@ -48,6 +48,11 @@ #define FILAMENT_DEBUG_COMMANDS FILAMENT_DEBUG_COMMANDS_NONE +// Upon command stream overflow, print a histogram of commands +#ifndef FILAMENT_DEBUG_COMMANDS_HISTOGRAM +#define FILAMENT_DEBUG_COMMANDS_HISTOGRAM 0 +#endif + namespace filament::backend { class BufferDescriptor; diff --git a/filament/backend/src/CommandBufferQueue.cpp b/filament/backend/src/CommandBufferQueue.cpp index 3a2f31b87eb5..1ca8a6f04b81 100644 --- a/filament/backend/src/CommandBufferQueue.cpp +++ b/filament/backend/src/CommandBufferQueue.cpp @@ -106,11 +106,13 @@ void CommandBufferQueue::flush(std::function const& debugPri std::unique_lock lock(mLock); // circular buffer is too small, we corrupted the stream +#if FILAMENT_DEBUG_COMMANDS_HISTOGRAM if (UTILS_VERY_UNLIKELY(used > mFreeSpace)) { if (debugPrintHistogram) { debugPrintHistogram(begin, end); } } +#endif FILAMENT_CHECK_POSTCONDITION(used <= mFreeSpace) << "Backend CommandStream overflow. Commands are corrupted and unrecoverable.\n" "Please increase minCommandBufferSizeMB inside the Config passed to Engine::create.\n" diff --git a/filament/backend/src/CommandStream.cpp b/filament/backend/src/CommandStream.cpp index 39c473293858..e3787d55074f 100644 --- a/filament/backend/src/CommandStream.cpp +++ b/filament/backend/src/CommandStream.cpp @@ -30,12 +30,15 @@ #include #include +#if FILAMENT_DEBUG_COMMANDS_HISTOGRAM #include +#include +#endif + #include #include #include #include -#include #ifdef __ANDROID__ #include @@ -86,7 +89,9 @@ CommandStream::CommandStream(Driver& driver, CircularBuffer& buffer) noexcept mUsePerformanceCounter = bool(atoi(property)); #endif +#if FILAMENT_DEBUG_COMMANDS_HISTOGRAM initializeLookup(); +#endif } void CommandStream::execute(void* buffer) { @@ -130,6 +135,7 @@ void CommandStream::execute(void* buffer) { } } +#if FILAMENT_DEBUG_COMMANDS_HISTOGRAM void CommandStream::debugIterateCommands(void* head, void* tail, std::function const& callback) { CommandBase* UTILS_RESTRICT base = static_cast(head); @@ -192,6 +198,7 @@ void CommandStream::debugPrintHistogram(void* head, void* tail) { LOG(INFO) << "CS hist: " << short_histogram; LOG(INFO) << ""; } +#endif void CommandStream::queueCommand(std::function command) { new(allocateCommand(CustomCommand::align(sizeof(CustomCommand)))) CustomCommand(std::move(command)); diff --git a/filament/src/details/Engine.cpp b/filament/src/details/Engine.cpp index ce402117efcc..bbccc8407684 100644 --- a/filament/src/details/Engine.cpp +++ b/filament/src/details/Engine.cpp @@ -921,7 +921,10 @@ int FEngine::loop() { void FEngine::flushCommandBuffer(CommandBufferQueue& commandBufferQueue) const { getDriver().purge(); commandBufferQueue.flush([this](void* begin, void* end) { - const_cast(this)->getDriverApi().debugPrintHistogram(begin, end); + UTILS_UNUSED FEngine* engine = const_cast(this); +#if FILAMENT_DEBUG_COMMANDS_HISTOGRAM + engine->getDriverApi().debugPrintHistogram(begin, end); +#endif }); }