Merge pull request ceph#61264 from irq0/wip/breakpad

chardan · web-flow · commit 56117217933c · 2025-06-02T09:17:43.000-07:00
Breakpad Crash Handler
diff --git a/.gitmodules b/.gitmodules
@@ -79,3 +79,10 @@
 	path = src/nvmeof/gateway
 	url = https://github.com/ceph/ceph-nvmeof.git
 	fetchRecurseSubmodules = false
+	shallow = true
+[submodule "src/breakpad"]
+	path = src/breakpad
+	url = https://chromium.googlesource.com/breakpad/breakpad
+[submodule "src/lss"]
+	path = src/lss
+	url = https://chromium.googlesource.com/linux-syscall-support
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -542,6 +542,18 @@ if(WITH_CATCH2)
   message("-- Enabled Catch2 support")
 endif()
 
+if(WIN32)
+    set(WITH_BREAKPAD_DEFAULT OFF)
+else()
+    set(WITH_BREAKPAD_DEFAULT ON)
+endif()
+
+option(WITH_BREAKPAD "Build with Google Breakpad crash reporter"  ${WITH_BREAKPAD_DEFAULT})
+if(WITH_BREAKPAD)
+  set(HAVE_BREAKPAD ON)
+  message("-- Enabled Google Breakpad crash reporter")
+endif()
+
 #option for RGW
 option(WITH_RADOSGW "RADOS Gateway is enabled" ON)
 option(WITH_RADOSGW_BEAST_OPENSSL "RADOS Gateway's Beast frontend uses OpenSSL" ON)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -325,6 +325,47 @@ if (WITH_BLKIN)
   add_subdirectory(blkin/blkin-lib)
 endif(WITH_BLKIN)
 
+## breakpad
+if(WITH_BREAKPAD)
+  set(breakpad_SOURCE_DIR ${CMAKE_SOURCE_DIR}/src/breakpad)
+  set(lss_SOURCE_DIR ${CMAKE_SOURCE_DIR}/src/lss)
+
+  add_custom_target(breakpad_lss_symlink)
+  add_custom_command(
+    TARGET breakpad_lss_symlink
+    COMMAND ${CMAKE_COMMAND} -E create_symlink ${lss_SOURCE_DIR} ${breakpad_SOURCE_DIR}/src/third_party/lss
+    COMMENT "Creating symbolic link lss -> breakpad third party"
+  )
+  ExternalProject_Add(
+    breakpad_project
+    SOURCE_DIR "${breakpad_SOURCE_DIR}"
+    CONFIGURE_COMMAND cd "${breakpad_SOURCE_DIR}"
+        COMMAND "${breakpad_SOURCE_DIR}/configure"
+	    "CFLAGS=${CMAKE_C_FLAGS} -Wno-array-bounds  -Wno-maybe-uninitialized"
+	    "CXXFLAGS=${CMAKE_CXX_FLAGS} -Wno-array-bounds  -Wno-maybe-uninitialized"
+            "LDFLAGS=${CMAKE_EXE_LINKER_FLAGS} -Wno-array-bounds  -Wno-maybe-uninitialized"
+    BUILD_COMMAND
+    /bin/sh -cx "cd ${breakpad_SOURCE_DIR} && make"
+    INSTALL_COMMAND ""
+    UPDATE_DISCONNECTED ON
+    BUILD_IN_SOURCE ON
+    DEPENDS breakpad_lss_symlink
+    BUILD_BYPRODUCTS "${breakpad_SOURCE_DIR}/src/libbreakpad.a;${breakpad_SOURCE_DIR}/src/client/linux/libbreakpad_client.a"
+  )
+
+  add_library(libbreakpad STATIC IMPORTED GLOBAL)
+  set_property(TARGET libbreakpad PROPERTY IMPORTED_LOCATION ${breakpad_SOURCE_DIR}/src/libbreakpad.a)
+  add_library(libbreakpad_client STATIC IMPORTED GLOBAL)
+  set_property(TARGET libbreakpad_client PROPERTY IMPORTED_LOCATION ${breakpad_SOURCE_DIR}/src/client/linux/libbreakpad_client.a)
+
+  include_directories(SYSTEM "${breakpad_SOURCE_DIR}/src")
+  add_dependencies(libbreakpad breakpad_project)
+  add_dependencies(libbreakpad_client breakpad_project)
+
+  add_library(breakpad INTERFACE)
+  target_link_libraries(breakpad INTERFACE libbreakpad libbreakpad_client)
+endif(WITH_BREAKPAD)
+
 if(WITH_JAEGER)
   find_package(thrift 0.13.0 REQUIRED)
   include(BuildOpentelemetry)
@@ -481,6 +522,11 @@ if(WITH_JAEGER)
   target_link_libraries(common-objs jaeger_base)
 endif()
 
+if(WITH_BREAKPAD)
+  add_dependencies(common-objs breakpad_project)
+  target_link_libraries(common-objs breakpad)
+endif()
+
 CHECK_C_COMPILER_FLAG("-fvar-tracking-assignments" HAS_VTA)
 add_subdirectory(auth)
 add_subdirectory(common)
@@ -542,6 +588,10 @@ if(WITH_JAEGER)
   list(APPEND ceph_common_deps jaeger_base)
 endif()
 
+if(WITH_BREAKPAD)
+  list(APPEND ceph_common_deps breakpad)
+endif()
+
 if(WIN32)
   list(APPEND ceph_common_deps ws2_32 mswsock iphlpapi bcrypt)
   list(APPEND ceph_common_deps dlfcn_win32)
@@ -578,6 +628,10 @@ if(WITH_JAEGER)
 add_dependencies(common jaeger_base)
 endif()
 
+if(WITH_BREAKPAD)
+add_dependencies(common breakpad_project)
+endif()
+
 if (WIN32)
   # Statically building ceph-common on Windows fails. We're temporarily
   # reverting this: 22fefb2338cfc4fcb03ece3cbf77aa964a7f17f2
@@ -597,6 +651,10 @@ if(WITH_JAEGER)
 add_dependencies(ceph-common jaeger_base)
 endif()
 
+if(WITH_BREAKPAD)
+add_dependencies(ceph-common breakpad_project)
+endif()
+
 # appease dpkg-shlibdeps
 set_target_properties(ceph-common PROPERTIES
   SOVERSION 2
diff --git a/src/breakpad b/src/breakpad
@@ -0,0 +1 @@
+Subproject commit 41b6533e5f3dd7f0320ef58608ee32e8e4f132fb
diff --git a/src/common/ceph_context.h b/src/common/ceph_context.h
@@ -34,6 +34,9 @@
 #include "common/cmdparse.h"
 #include "common/code_environment.h"
 #include "msg/msg_types.h"
+#ifdef HAVE_BREAKPAD
+#include "breakpad/src/client/linux/handler/exception_handler.h"
+#endif
 #ifdef WITH_CRIMSON
 #include "crimson/common/config_proxy.h"
 #include "crimson/common/perf_counters_collection.h"
@@ -143,6 +146,9 @@ class CephContext {
 
   ConfigProxy _conf;
   ceph::logging::Log *_log;
+#ifdef HAVE_BREAKPAD
+  std::unique_ptr<google_breakpad::ExceptionHandler> _ex_handler;
+#endif
 
   /* init ceph::crypto */
   void init_crypto();
diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
@@ -433,6 +433,14 @@ options:
   flags:
   - startup
   with_legacy: true
+- name: breakpad
+  type: bool
+  level: advanced
+  desc: setup breakpad crash handler
+  default: false
+  flags:
+  - startup
+  with_legacy: true
 - name: crash_dir
   type: str
   level: advanced
diff --git a/src/crimson/CMakeLists.txt b/src/crimson/CMakeLists.txt
@@ -137,6 +137,10 @@ if(WITH_JAEGER)
   list(APPEND crimson_common_public_deps jaeger_base)
 endif()
 
+if(WITH_BREAKPAD)
+  list(APPEND crimson_common_public_deps breakpad)
+endif()
+
 if(NOT WITH_SYSTEM_BOOST)
   list(APPEND crimson_common_deps ${ZLIB_LIBRARIES})
 endif()
diff --git a/src/global/global_init.cc b/src/global/global_init.cc
@@ -26,6 +26,10 @@
 #include "extblkdev/ExtBlkDevPlugin.h"
 #include "global/global_context.h"
 #include "global/global_init.h"
+#ifdef HAVE_BREAKPAD
+#include <client/linux/handler/minidump_descriptor.h>
+#include <google_breakpad/common/minidump_format.h>
+#endif
 #include "global/pidfile.h"
 #include "global/signal_handler.h"
 #include "include/compat.h"
@@ -178,6 +182,17 @@ void global_pre_init(
   g_conf().complain_about_parse_error(g_ceph_context);
 }
 
+#ifdef HAVE_BREAKPAD
+static bool dumpCallback(
+    const google_breakpad::MinidumpDescriptor& descriptor, void* context,
+    bool succeeded) {
+  char buf[1024];
+  snprintf(buf, sizeof(buf), "minidump created in path %s", descriptor.path());
+  dout_emergency(buf);
+  return succeeded;
+}
+#endif
+
 boost::intrusive_ptr<CephContext>
 global_init(const std::map<std::string,std::string> *defaults,
 	    std::vector < const char* >& args,
@@ -215,6 +230,20 @@ global_init(const std::map<std::string,std::string> *defaults,
   if (g_conf()->fatal_signal_handlers) {
     install_standard_sighandlers();
   }
+
+#ifdef HAVE_BREAKPAD
+  if (g_conf()->breakpad) {
+    google_breakpad::MinidumpDescriptor descriptor(g_conf()->crash_dir);
+    g_ceph_context->_ex_handler.reset(
+	new google_breakpad::ExceptionHandler(descriptor, nullptr, dumpCallback, nullptr, true, -1));
+  }
+#else
+  if (g_conf()->breakpad) {
+    cerr << "breakpad crash reporting requested, but disabled at build time"
+         << std::endl;
+  }
+#endif
+
   ceph::register_assert_context(g_ceph_context);
 
   if (g_conf()->log_flush_on_exit)
diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake
@@ -405,4 +405,7 @@
 /* libexec directory path */
 #cmakedefine CMAKE_INSTALL_LIBEXECDIR "@CMAKE_INSTALL_LIBEXECDIR@"
 
+/* Define if breakpad is available */
+#cmakedefine HAVE_BREAKPAD
+
 #endif /* CONFIG_H */
diff --git a/src/lss b/src/lss
@@ -0,0 +1 @@
+Subproject commit ed31caa60f20a4f6569883b2d752ef7522de51e0
diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
@@ -365,6 +365,11 @@ if(WITH_JAEGER)
   target_link_libraries(rgw_common PUBLIC jaeger_base)
 endif()
 
+if(WITH_BREAKPAD)
+  add_dependencies(rgw_common breakpad_project)
+  target_link_libraries(rgw_common PUBLIC breakpad)
+endif()
+
 if(WITH_RADOSGW_DBSTORE)
   target_link_libraries(rgw_common PRIVATE global dbstore)
 endif()
diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt
@@ -1026,6 +1026,15 @@ target_link_libraries(test_nvmeof_mon_encoding
   mon ceph-common global-static
   )
 
+
+if(WITH_BREAKPAD)
+# unittest_ceph_breakpad
+add_executable(unittest_ceph_breakpad
+  ceph_breakpad.cc)
+add_ceph_unittest(unittest_ceph_breakpad)
+target_link_libraries(unittest_ceph_breakpad ceph-common global)
+endif()
+
 if(NOT WIN32)
 # unittest_ceph_assert
 add_executable(unittest_ceph_assert
diff --git a/src/test/ceph_breakpad.cc b/src/test/ceph_breakpad.cc
@@ -0,0 +1,114 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2025 Clyso GmbH
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <common/Thread.h>
+#include <common/ceph_time.h>
+#include <include/uuid.h>
+
+#include <csignal>
+#include <exception>
+#include <filesystem>
+#include <fstream>
+#include <include/expected.hpp>
+#include <ios>
+
+#include "common/ceph_argparse.h"
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "gtest/gtest.h"
+#include "include/ceph_assert.h"
+#include <google_breakpad/common/minidump_format.h>
+
+class BreakpadDeathTest : public ::testing::Test {
+  const std::filesystem::path crash_dir{
+      g_conf().get_val<std::string>("crash_dir")};
+
+  void SetUp() override {
+    std::filesystem::create_directories(crash_dir);
+    std::cout << "using crash dir: " << crash_dir << std::endl;
+  }
+
+  void TearDown() override { std::filesystem::remove_all(crash_dir);}
+
+ public:
+  std::optional<std::filesystem::path> minidump() {
+    for (auto const& dir_entry :
+         std::filesystem::directory_iterator{crash_dir}) {
+      if (dir_entry.is_regular_file() &&
+          dir_entry.path().extension() == ".dmp") {
+        return dir_entry.path();
+      }
+    }
+    return std::nullopt;
+  }
+
+  void check_minidump() {
+    const auto md = minidump();
+    ASSERT_TRUE(md.has_value());
+    EXPECT_TRUE(std::filesystem::exists(md.value())) << md.value();
+
+    std::ifstream in(md.value(), std::ios::binary);
+    std::array<char, sizeof(MDRawHeader)> buf{0};
+    in.read(buf.data(), sizeof(MDRawHeader));
+
+    const auto* header = reinterpret_cast<MDRawHeader*>(buf.data());
+    ASSERT_TRUE(header);
+    EXPECT_EQ(header->signature, MD_HEADER_SIGNATURE);
+    EXPECT_EQ(header->version, MD_HEADER_VERSION);
+    EXPECT_GT(header->stream_count, 0);
+    EXPECT_GT(header->time_date_stamp, 0);
+  }
+
+  std::string expected_minidump_message() {
+    return "minidump created in path " + crash_dir.string();
+  }
+};
+
+TEST_F(BreakpadDeathTest, CephAbortCreatesMinidump) {
+  ASSERT_DEATH(ceph_abort(), expected_minidump_message());
+  check_minidump();
+}
+
+TEST_F(BreakpadDeathTest, AbortCreatesMinidump) {
+  ASSERT_DEATH(abort(), expected_minidump_message());
+  check_minidump();
+}
+
+TEST_F(BreakpadDeathTest, SegfaultCreatesMinidump) {
+  EXPECT_EXIT(
+      std::raise(SIGSEGV), testing::KilledBySignal(SIGSEGV),
+      expected_minidump_message());
+  check_minidump();
+}
+
+TEST_F(BreakpadDeathTest, TerminateCreatesMinidump) {
+  ASSERT_DEATH(std::terminate(), expected_minidump_message());
+  check_minidump();
+}
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  GTEST_FLAG_SET(death_test_style, "threadsafe");
+
+  const auto dir =
+      std::filesystem::temp_directory_path() / "ceph_breakpad_test";
+  std::map<std::string, std::string> defaults = {
+      {"breakpad", "true"}, {"crash_dir", dir.string()}};
+  auto args = argv_to_vec(argc, argv);
+  auto cct = global_init(
+      &defaults, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+      CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  return RUN_ALL_TESTS();
+}
