Merge pull request #61 from Simple-Robotics/video-writer-performance

Rework texture downloader, screenshot capture and video recording, fix `CommandBuffer::submitAndAcquireFence()` not setting the internal pointer to null, fix velocity arrow direction in `RobotDebugScene`

Author: ManifoldFR

CHANGELOG.md

@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- utils : add `FileDialogGui.h` (https://github.com/Simple-Robotics/candlewick/pull/61)
+
+### Changed
+
+- utils : avoid public inclusion of `libavutil/pixfmt` header for video recorder (https://github.com/Simple-Robotics/candlewick/pull/61)
+- core : `errors.h`: make terminate_with_message a template (with formatting arguments, etc) (https://github.com/Simple-Robotics/candlewick/pull/61)
+- utils: add VideoRecorder::close() API (to manually close recorder) (https://github.com/Simple-Robotics/candlewick/pull/61)
+- multibody/RobotDebug.cpp : fix velocity arrow direction (https://github.com/Simple-Robotics/candlewick/pull/61)
+
+### Fixed
+
+- core : fix `CommandBuffer::submitAndAcquireFence()` not setting the internal pointer to null
+
 ## [0.0.7] - 2025-05-17
 
 ### Added

examples/Ur5WithSystems.cpp

@@ -14,6 +14,7 @@
 #include "candlewick/multibody/RobotScene.h"
 #include "candlewick/multibody/RobotDebug.h"
 #include "candlewick/primitives/Primitives.h"
+#include "candlewick/utils/FileDialogGui.h"
 #include "candlewick/utils/WriteTextureToImage.h"
 
 #include <imgui.h>
@@ -184,11 +185,17 @@ void eventLoop(const Renderer &renderer) {
   }
 }
 
-Renderer createRenderer(Uint32 width, Uint32 height,
-                        SDL_GPUTextureFormat depth_stencil_format) {
-  return Renderer{Device{auto_detect_shader_format_subset(), true},
-                  Window(__FILE__, int(width), int(height), 0),
-                  depth_stencil_format};
+static void screenshot_button_callback(Renderer &renderer,
+                                       media::TransferBufferPool &pool,
+                                       const char *filename) {
+  const auto &device = renderer.device;
+  CommandBuffer command_buffer{device};
+  renderer.waitAndAcquireSwapchain(command_buffer);
+
+  SDL_Log("Saving screenshot at %s", filename);
+  media::writeToFile(command_buffer, device, pool, renderer.swapchain,
+                     renderer.getSwapchainTextureFormat(), wWidth, wHeight,
+                     filename);
 }
 
 int main(int argc, char **argv) {
@@ -206,8 +213,11 @@ int main(int argc, char **argv) {
     return 1;
 
   // D16_UNORM works on macOS, D24_UNORM and D32_FLOAT break the depth prepass
-  Renderer renderer =
-      createRenderer(wWidth, wHeight, SDL_GPU_TEXTUREFORMAT_D16_UNORM);
+  Renderer renderer{
+      Device{auto_detect_shader_format_subset(), true},
+      Window(__FILE__, wWidth, wHeight, 0),
+      SDL_GPU_TEXTUREFORMAT_D16_UNORM,
+  };
 
   entt::registry registry{};
 
@@ -313,6 +323,8 @@ int main(int argc, char **argv) {
 
   FrustumBoundsDebugSystem frustumBoundsDebug{registry, renderer};
 
+  const char *screenshot_filename = nullptr;
+
   GuiSystem gui_system{
       renderer, [&](const Renderer &r) {
         IMGUI_CHECKVERSION();
@@ -391,6 +403,13 @@ int main(int argc, char **argv) {
           ImGui::RadioButton("Heatmap", (int *)&depth_mode, 1);
         }
 
+        ImGui::SeparatorText("Screenshots");
+        static GuiFileSaveDialog scr_dialog;
+        scr_dialog.addFileDialog(renderer.window);
+        if (ImGui::Button("Take screenshot")) {
+          screenshot_filename = scr_dialog.filename.c_str();
+        }
+
         ImGui::SeparatorText("Robot model");
         ImGui::SetItemTooltip("Information about the displayed robot model.");
         multibody::guiAddPinocchioModelInfo(registry, model, geom_model);
@@ -414,44 +433,43 @@ int main(int argc, char **argv) {
 
   Uint32 frameNo = 0;
 
-  srand(42);
+  std::srand(42);
   Eigen::VectorXd q0 = pin::neutral(model);
   Eigen::VectorXd q1 = pin::randomConfiguration(model);
 
 #ifdef CANDLEWICK_WITH_FFMPEG_SUPPORT
   media::VideoRecorder recorder{NoInit};
+  media::TransferBufferPool transfer_buffer_pool{renderer.device};
   if (performRecording)
-    recorder = media::VideoRecorder{wWidth, wHeight, "ur5.mp4"};
+    recorder = media::VideoRecorder{wWidth,
+                                    wHeight,
+                                    "ur5.mp4",
+                                    {
+                                        .fps = 50,
+                                    }};
 #endif
 
-  auto record_callback = [&] {
-#ifdef CANDLEWICK_WITH_FFMPEG_SUPPORT
-    auto swapchain_format = renderer.getSwapchainTextureFormat();
-    media::videoWriteTextureToFrame(renderer.device, recorder,
-                                    renderer.swapchain, swapchain_format,
-                                    wWidth, wHeight);
-#endif
-  };
-
   AABB &worldSpaceBounds = robot_scene.worldSpaceBounds;
   worldSpaceBounds.update({-1.f, -1.f, 0.f}, {+1.f, +1.f, 1.f});
 
   frustumBoundsDebug.addBounds(worldSpaceBounds);
   frustumBoundsDebug.addFrustum(shadowPassInfo.cam);
 
   Eigen::VectorXd q = q0;
+  Eigen::VectorXd qn = q;
+  Eigen::VectorXd v{model.nv};
   const double dt = 1e-2;
 
   while (!quitRequested) {
     // logic
     eventLoop(renderer);
     double alpha = 0.5 * (1. + std::sin(frameNo * dt));
-    Eigen::VectorXd qn = q;
-    q = pin::interpolate(model, q0, q1, alpha);
-    Eigen::VectorXd v = pin::difference(model, q, qn) / dt;
-    pin::forwardKinematics(model, pin_data, q, v);
+    pin::interpolate(model, q0, q1, alpha, qn);
+    v = pin::difference(model, q, qn) / dt;
+    pin::forwardKinematics(model, pin_data, qn, v);
     pin::updateFramePlacements(model, pin_data);
     pin::updateGeometryPlacements(model, pin_data, geom_model, geom_data);
+    q = qn;
     debug_scene.update();
 
     // acquire command buffer and swapchain
@@ -494,7 +512,18 @@ int main(int argc, char **argv) {
     command_buffer.submit();
 
     if (performRecording) {
-      record_callback();
+#ifdef CANDLEWICK_WITH_FFMPEG_SUPPORT
+      CommandBuffer command_buffer = renderer.acquireCommandBuffer();
+      auto swapchain_format = renderer.getSwapchainTextureFormat();
+      media::videoWriteTextureToFrame(
+          command_buffer, renderer.device, transfer_buffer_pool, recorder,
+          renderer.swapchain, swapchain_format, wWidth, wHeight);
+#endif
+    }
+    if (screenshot_filename) {
+      screenshot_button_callback(renderer, transfer_buffer_pool,
+                                 screenshot_filename);
+      screenshot_filename = nullptr;
     }
     frameNo++;
   }
@@ -507,6 +536,10 @@ int main(int argc, char **argv) {
   robot_scene.release();
   debug_scene.release();
   gui_system.release();
+#ifdef CANDLEWICK_WITH_FFMPEG_SUPPORT
+  recorder.close();
+  transfer_buffer_pool.release();
+#endif
   renderer.destroy();
   SDL_Quit();
   return 0;

examples/lib/Common.cpp

@@ -1,5 +1,3 @@
-#include "candlewick/core/MeshLayout.h"
-#include "candlewick/core/Shader.h"
 #include "candlewick/core/math_types.h"
 #include "candlewick/primitives/Cube.h"
 #include "candlewick/utils/MeshTransforms.h"

src/CMakeLists.txt

@@ -39,6 +39,7 @@ add_library(
   candlewick/core/debug/Frustum.cpp
   candlewick/posteffects/ScreenSpaceShadows.cpp
   candlewick/posteffects/SSAO.cpp
+  candlewick/utils/FileDialogGui.cpp
   candlewick/utils/LoadMesh.cpp
   candlewick/utils/LoadMaterial.cpp
   candlewick/utils/MeshData.cpp
@@ -56,7 +57,7 @@ add_library(
   candlewick/primitives/Internal.cpp
   candlewick/primitives/Plane.cpp
   candlewick/primitives/Sphere.cpp
-  candlewick/third-party/sbti_lib.cpp
+  candlewick/third-party/stbi_lib.cpp
   # imgui sources
   ${IMGUI_SRC}
 )

src/candlewick/core/CommandBuffer.cpp

@@ -21,4 +21,15 @@ CommandBuffer &CommandBuffer::operator=(CommandBuffer &&other) noexcept {
   return *this;
 }
 
+bool CommandBuffer::cancel() noexcept {
+  bool ret = SDL_CancelGPUCommandBuffer(_cmdBuf);
+  _cmdBuf = nullptr;
+  if (!ret) {
+    SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                "Failed to cancel command buffer: %s", SDL_GetError());
+    return false;
+  }
+  return true;
+}
+
 } // namespace candlewick

src/candlewick/core/CommandBuffer.h

@@ -46,23 +46,28 @@ class CommandBuffer {
     return true;
   }
 
-  bool cancel() noexcept {
-    if (!(active() && SDL_CancelGPUCommandBuffer(_cmdBuf)))
-      return false;
+  SDL_GPUFence *submitAndAcquireFence() noexcept {
+    SDL_GPUFence *fence = SDL_SubmitGPUCommandBufferAndAcquireFence(_cmdBuf);
     _cmdBuf = nullptr;
-    return true;
+    return fence;
   }
 
+  /// \brief Cancel the command buffer, returning the bool value from the
+  /// wrapped SDL API.
+  bool cancel() noexcept;
+
+  /// \brief Check if the command buffer is still active.
+  ///
+  /// For this wrapper class, it means the internal pointer is non-null.
   bool active() const noexcept { return _cmdBuf; }
 
   ~CommandBuffer() noexcept {
     if (active()) {
       SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
                   "CommandBuffer object is being destroyed while still active! "
                   "It will be cancelled.");
-      SDL_CancelGPUCommandBuffer(_cmdBuf);
-      _cmdBuf = nullptr;
-      assert(false);
+      [[maybe_unused]] bool ret = cancel();
+      assert(ret);
     }
   }
 
@@ -99,10 +104,6 @@ class CommandBuffer {
     SDL_PushGPUFragmentUniformData(_cmdBuf, slot_index, data, length);
     return *this;
   }
-
-  SDL_GPUFence *submitAndAcquireFence() {
-    return SDL_SubmitGPUCommandBufferAndAcquireFence(_cmdBuf);
-  }
 };
 
 } // namespace candlewick

src/candlewick/core/DepthAndShadowPass.cpp

@@ -37,8 +37,8 @@ DepthPass::DepthPass(const Device &device, const MeshLayout &layout,
   };
   pipeline = SDL_CreateGPUGraphicsPipeline(device, &pipeline_desc);
   if (!pipeline)
-    terminate_with_message(
-        std::format("Failed to create graphics pipeline: %s.", SDL_GetError()));
+    terminate_with_message("Failed to create graphics pipeline: %s.",
+                           SDL_GetError());
 }
 
 DepthPass::DepthPass(const Device &device, const MeshLayout &layout,

src/candlewick/core/errors.h

@@ -47,13 +47,18 @@ namespace detail {
 
 } // namespace detail
 
+template <typename... Ts>
 [[noreturn]]
-inline void terminate_with_message(
-    std::string_view msg,
-    std::source_location location = std::source_location::current()) {
+void terminate_with_message(std::source_location location, std::string_view fmt,
+                            const Ts &...args) {
   throw std::runtime_error(
-      detail::error_message_format(location.function_name(), "{:s}", msg)
-          .c_str());
+      detail::error_message_format(location.function_name(), fmt, args...));
+}
+
+template <typename... Ts>
+[[noreturn]]
+void terminate_with_message(std::string_view fmt, const Ts &...args) {
+  terminate_with_message(std::source_location::current(), fmt, args...);
 }
 
 [[noreturn]]

src/candlewick/multibody/RobotDebug.cpp

@@ -61,7 +61,7 @@ void RobotDebugSystem::updateFrameVelocities(entt::registry &reg) {
     // the arrow mesh is posed z-up by default.
     // we need to rotate towards where the velocity is pointing,
     // then transform to the frame space.
-    quatf.setFromTwoVectors(Float3::UnitZ(), -v);
+    quatf.setFromTwoVectors(Float3::UnitZ(), v);
     Mat3f R2 = quatf.toRotationMatrix() * scaleMatrix;
     auto R = tr.topLeftCorner<3, 3>();
     R.applyOnTheRight(R2);

src/candlewick/multibody/RobotScene.cpp

@@ -226,8 +226,8 @@ void RobotScene::initCompositePipeline(const MeshLayout &layout) {
 
   pipelines.wboitComposite = SDL_CreateGPUGraphicsPipeline(device(), &desc);
   if (!pipelines.wboitComposite) {
-    terminate_with_message(
-        std::format("Failed to create WBOIT pipeline: %s", SDL_GetError()));
+    terminate_with_message("Failed to create WBOIT pipeline: %s",
+                           SDL_GetError());
   }
 }
 
@@ -279,7 +279,8 @@ void RobotScene::loadModels(const pin::GeometryModel &geom_model,
                                    m_config.shadow_config);
         shadows_configured = true;
       }
-      this->initCompositePipeline(layout);
+      if (!pipelines.wboitComposite)
+        this->initCompositePipeline(layout);
     }
   }
   m_initialized = true;