feat(crashtracking): capture unhandled exception with the crashtracker (#5321)

gyuheon0h · web-flow · commit 40f46ee95369 · 2026-02-10T09:57:30.000-05:00
* Remove VS Code config files from tracking * Gitignore weird files that keep popping up (will pop this commit later) Signal based crash report (crash done, need to do ping) Revert "Gitignore weird files that keep popping up (will pop this commit later)" This reverts commit aeb3017. Revert "Remove VS Code config files from tracking" This reverts commit 2b30b86. Use locations array Clean Lazy logging Fix memory leak * Crash ping Fmt fmt Do work on ruby side, fix sus calls Remove noisy log Update symbol name Check result, build message in ruby unit test and test cleanup Inline + no order dependency + cleanup Number of frames logic on ruby side frame processing in helper Restore accidentally deleted comment Update tags on fork Fmt Fix potential mem leak move to core clean Extract into helper Fix more potential leaks Fmt Remove comment from Ruby exception crash reporting context Removed comment about Ruby exception crash reporting tests. Respond to oleg -(rescuing all exceptions) Flip negation No more do-while, crash vs exception naming, test sleep fix, minor refactoring Tag builder helper func, move all logic into ct component, move builder into build function * Comprehensive testing * Timestamp is handled by libdatadog Trigger CI rbs file Trigger CI CI debug We need to explicitly check, not depend on order Be explicit with typing Trigger CI Incomplete stack Clarity in tests * Final cleanups
diff --git a/ext/libdatadog_api/crashtracker.c b/ext/libdatadog_api/crashtracker.c
@@ -7,6 +7,8 @@
 static VALUE _native_start_or_update_on_fork(int argc, VALUE *argv, DDTRACE_UNUSED VALUE _self);
 static VALUE _native_stop(DDTRACE_UNUSED VALUE _self);
 
+void crashtracker_report_exception_init(VALUE crashtracker_class);
+
 static bool first_init = true;
 
 // Used to report Ruby VM crashes.
@@ -18,6 +20,9 @@ void crashtracker_init(VALUE core_module) {
 
   rb_define_singleton_method(crashtracker_class, "_native_start_or_update_on_fork", _native_start_or_update_on_fork, -1);
   rb_define_singleton_method(crashtracker_class, "_native_stop", _native_stop, 0);
+
+  // Initialize Ruby non-signal-crash reporting
+  crashtracker_report_exception_init(crashtracker_class);
 }
 
 static VALUE _native_start_or_update_on_fork(int argc, VALUE *argv, DDTRACE_UNUSED VALUE _self) {
diff --git a/ext/libdatadog_api/crashtracker_report_exception.c b/ext/libdatadog_api/crashtracker_report_exception.c
@@ -0,0 +1,205 @@
+#include <datadog/common.h>
+#include <datadog/crashtracker.h>
+#include <ruby.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <string.h>
+
+#include "datadog_ruby_common.h"
+
+static VALUE _native_report_ruby_exception(VALUE _self, VALUE agent_base_url,
+                                          VALUE message, VALUE frames_data,
+                                          VALUE tags_as_array, VALUE library_version);
+
+static bool process_crash_frames(VALUE frames_data, ddog_crasht_Handle_StackTrace *stack_trace);
+static bool build_and_send_crash_report(ddog_crasht_Metadata metadata,
+                                        ddog_Endpoint *endpoint,
+                                        VALUE message,
+                                        VALUE frames_data);
+
+void crashtracker_report_exception_init(VALUE crashtracker_class) {
+  rb_define_singleton_method(crashtracker_class, "_native_report_ruby_exception",
+                            _native_report_ruby_exception, 5);
+}
+
+static VALUE _native_report_ruby_exception(DDTRACE_UNUSED VALUE _self, VALUE agent_base_url,
+                                          VALUE message, VALUE frames_data,
+                                          VALUE tags_as_array, VALUE library_version) {
+  ENFORCE_TYPE(agent_base_url, T_STRING);
+  ENFORCE_TYPE(message, T_STRING);
+  ENFORCE_TYPE(frames_data, T_ARRAY);
+  ENFORCE_TYPE(tags_as_array, T_ARRAY);
+  ENFORCE_TYPE(library_version, T_STRING);
+
+  ddog_Endpoint *endpoint = ddog_endpoint_from_url(char_slice_from_ruby_string(agent_base_url));
+  if (!endpoint) return Qfalse;
+
+  ddog_Vec_Tag tags = convert_tags(tags_as_array);
+
+  ddog_crasht_Metadata metadata = {
+    .library_name = DDOG_CHARSLICE_C("dd-trace-rb"),
+    .library_version = char_slice_from_ruby_string(library_version),
+    .family = DDOG_CHARSLICE_C("ruby"),
+    .tags = &tags,
+  };
+
+  bool success = build_and_send_crash_report(metadata, endpoint, message, frames_data);
+  ddog_Vec_Tag_drop(tags);
+  ddog_endpoint_drop(endpoint);
+
+  return success ? Qtrue : Qfalse;
+}
+
+static bool process_crash_frames(VALUE frames_data, ddog_crasht_Handle_StackTrace *stack_trace) {
+  size_t frame_count = RARRAY_LEN(frames_data);
+
+  // Return false and early so we can mark the stack as incomplete
+  // libdatadog's definition of an incomplete stack is that it has no frames
+  // or that report generation died in the middle of unwinding frames
+  if (frame_count == 0) {
+    return false;
+  }
+
+  for (size_t i = 0; i < frame_count; i++) {
+    VALUE frame_array = RARRAY_AREF(frames_data, i);
+
+    // ruby should guarantee [String, String, Integer]
+    if (!RB_TYPE_P(frame_array, T_ARRAY) || RARRAY_LEN(frame_array) != 3) {
+      // Malformed data from Ruby; this is a bug, bail out
+      return false;
+    }
+
+    VALUE file_val = RARRAY_AREF(frame_array, 0);
+    VALUE func_val = RARRAY_AREF(frame_array, 1);
+    VALUE line_val = RARRAY_AREF(frame_array, 2);
+
+    // validate types; Ruby should guarantee these
+    if (!RB_TYPE_P(file_val, T_STRING) || !RB_TYPE_P(func_val, T_STRING) || !RB_TYPE_P(line_val, T_FIXNUM)) {
+      // Type mismatch from Ruby; this is a bug, bail out
+      return false;
+    }
+
+    ddog_crasht_StackFrame_NewResult frame_result = ddog_crasht_StackFrame_new();
+    if (frame_result.tag != DDOG_CRASHT_STACK_FRAME_NEW_RESULT_OK) {
+      return false;
+    }
+
+    ddog_crasht_Handle_StackFrame *frame = &frame_result.ok;
+
+    if (ddog_crasht_StackFrame_with_file(frame, char_slice_from_ruby_string(file_val)).tag != DDOG_VOID_RESULT_OK) {
+      ddog_crasht_StackFrame_drop(frame);
+      return false;
+    }
+    if (ddog_crasht_StackFrame_with_function(frame, char_slice_from_ruby_string(func_val)).tag != DDOG_VOID_RESULT_OK) {
+      ddog_crasht_StackFrame_drop(frame);
+      return false;
+    }
+
+    uint32_t line = (uint32_t)FIX2INT(line_val);
+    if (line > 0) {
+      if (ddog_crasht_StackFrame_with_line(frame, line).tag != DDOG_VOID_RESULT_OK) {
+        ddog_crasht_StackFrame_drop(frame);
+        return false;
+      }
+    }
+
+    if (ddog_crasht_StackTrace_push_frame(stack_trace, frame, true).tag != DDOG_VOID_RESULT_OK) {
+      ddog_crasht_StackFrame_drop(frame);
+      return false;
+    }
+  }
+
+  return true;
+}
+
+static bool build_and_send_crash_report(ddog_crasht_Metadata metadata,
+                                        ddog_Endpoint *endpoint,
+                                        VALUE message,
+                                        VALUE frames_data) {
+  ddog_crasht_Handle_StackTrace *stack_trace = NULL;
+
+  ddog_crasht_CrashInfoBuilder_NewResult builder_result = ddog_crasht_CrashInfoBuilder_new();
+  if (builder_result.tag != DDOG_CRASHT_CRASH_INFO_BUILDER_NEW_RESULT_OK) {
+    return false;
+  }
+
+  ddog_crasht_Handle_CrashInfoBuilder *builder = &builder_result.ok;
+
+  // Setup builder metadata and configuration
+  if (ddog_crasht_CrashInfoBuilder_with_metadata(builder, metadata).tag != DDOG_VOID_RESULT_OK) {
+    ddog_crasht_CrashInfoBuilder_drop(builder);
+    return false;
+  }
+
+  if (ddog_crasht_CrashInfoBuilder_with_kind(builder, DDOG_CRASHT_ERROR_KIND_UNHANDLED_EXCEPTION).tag != DDOG_VOID_RESULT_OK) {
+    ddog_crasht_CrashInfoBuilder_drop(builder);
+    return false;
+  }
+
+  // Send ping first
+  if (ddog_crasht_CrashInfoBuilder_upload_ping_to_endpoint(builder, endpoint).tag != DDOG_VOID_RESULT_OK) {
+    ddog_crasht_CrashInfoBuilder_drop(builder);
+    return false;
+  }
+
+  ddog_crasht_ProcInfo proc_info = { .pid = (uint32_t)getpid() };
+  if (ddog_crasht_CrashInfoBuilder_with_proc_info(builder, proc_info).tag != DDOG_VOID_RESULT_OK) {
+    ddog_crasht_CrashInfoBuilder_drop(builder);
+    return false;
+  }
+
+  if (ddog_crasht_CrashInfoBuilder_with_os_info_this_machine(builder).tag != DDOG_VOID_RESULT_OK) {
+    ddog_crasht_CrashInfoBuilder_drop(builder);
+    return false;
+  }
+
+  if (ddog_crasht_CrashInfoBuilder_with_message(builder, char_slice_from_ruby_string(message)).tag != DDOG_VOID_RESULT_OK) {
+    ddog_crasht_CrashInfoBuilder_drop(builder);
+    return false;
+  }
+
+  // Create and populate stack trace
+  ddog_crasht_StackTrace_NewResult stack_result = ddog_crasht_StackTrace_new();
+  if (stack_result.tag != DDOG_CRASHT_STACK_TRACE_NEW_RESULT_OK) {
+    ddog_crasht_CrashInfoBuilder_drop(builder);
+    return false;
+  }
+
+  stack_trace = &stack_result.ok;
+
+  bool frames_processed_successfully = process_crash_frames(frames_data, stack_trace);
+
+  // Only mark as complete if we successfully processed all frames
+  if (frames_processed_successfully) {
+    if (ddog_crasht_StackTrace_set_complete(stack_trace).tag != DDOG_VOID_RESULT_OK) {
+      ddog_crasht_StackTrace_drop(stack_trace);
+      ddog_crasht_CrashInfoBuilder_drop(builder);
+      return false;
+    }
+  }
+  // If frames processing failed, we still include the stack trace (which may be empty or partial)
+  // but don't mark it as complete, indicating it's incomplete
+
+  if (ddog_crasht_CrashInfoBuilder_with_stack(builder, stack_trace).tag != DDOG_VOID_RESULT_OK) {
+    ddog_crasht_StackTrace_drop(stack_trace);
+    ddog_crasht_CrashInfoBuilder_drop(builder);
+    return false;
+  }
+
+  // Builder takes ownership of stack_trace, so we don't need to clean it up anymore
+  stack_trace = NULL;
+
+  // Build and upload crash info
+  ddog_crasht_CrashInfo_NewResult crash_info_result = ddog_crasht_CrashInfoBuilder_build(builder);
+  if (crash_info_result.tag != DDOG_CRASHT_RESULT_HANDLE_CRASH_INFO_OK_HANDLE_CRASH_INFO) {
+    ddog_crasht_CrashInfoBuilder_drop(builder);
+    return false;
+  }
+
+  ddog_crasht_Handle_CrashInfo *crash_info = &crash_info_result.ok;
+  ddog_VoidResult upload_result = ddog_crasht_CrashInfo_upload_to_endpoint(crash_info, endpoint);
+  bool success = (upload_result.tag == DDOG_VOID_RESULT_OK);
+
+  ddog_crasht_CrashInfo_drop(crash_info);
+  return success;
+}
diff --git a/lib/datadog/core.rb b/lib/datadog/core.rb
@@ -29,9 +29,14 @@ module Core
   # Ensures the Datadog components have a chance to gracefully
   # shut down and cleanup before terminating the process.
   at_exit do
-    if Interrupt === $! # rubocop:disable Style/SpecialGlobalVars is process terminating due to a ctrl+c or similar?
+    exception = $! # rubocop:disable Style/SpecialGlobalVars
+
+    if Interrupt === exception # is process terminating due to a ctrl+c or similar?
       Datadog.send(:handle_interrupt_shutdown!)
     else
+      # Report unhandled exception to crash tracker before shutdown
+      Datadog::Core::Crashtracking::Component.report_unhandled_exception(exception)
+
       Datadog.shutdown!
     end
   end
diff --git a/lib/datadog/core/crashtracking/component.rb b/lib/datadog/core/crashtracking/component.rb
@@ -17,7 +17,7 @@ module Crashtracking
       # Methods prefixed with _native_ are implemented in `crashtracker.c`
       class Component
         def self.build(settings, agent_settings, logger:)
-          tags = TagBuilder.call(settings)
+          tags = latest_tags(settings)
           agent_base_url = agent_settings.url
 
           ld_library_path = ::Libdatadog.ld_library_path
@@ -41,6 +41,32 @@ def self.build(settings, agent_settings, logger:)
           ).tap(&:start)
         end
 
+        # Reports unhandled exceptions to the crash tracker if available and appropriate.
+        # This is called from the at_exit hook to report unhandled exceptions.
+        def self.report_unhandled_exception(exception)
+          return unless exception && !exception.is_a?(SystemExit) && !exception.is_a?(NoMemoryError)
+
+          begin
+            crashtracker = Datadog.send(:components, allow_initialization: false)&.crashtracker
+            return unless crashtracker
+
+            crashtracker.report_unhandled_exception(exception)
+          rescue => e
+            # Unhandled exception report triggering means that the application is already in a bad state
+            # We don't want to swallow non-StandardError exceptions here; we would rather just let the
+            # application crash
+            Datadog.logger.debug("Crashtracker failed to report unhandled exception: #{e.message}")
+          end
+        end
+
+        # Gets the latest tags from the current configuration.
+        #
+        # We always fetch fresh tags because:
+        # After forking, we need the latest tags, not the parent's tags, such as the pid or runtime-id
+        def self.latest_tags(settings)
+          TagBuilder.call(settings)
+        end
+
         def initialize(tags:, agent_base_url:, ld_library_path:, path_to_crashtracking_receiver_binary:, logger:)
           @tags = tags
           @agent_base_url = agent_base_url
@@ -54,9 +80,52 @@ def start
         end
 
         def update_on_fork(settings: Datadog.configuration)
-          # Here we pick up the latest settings, so that we pick up any tags that change after forking
-          # such as the pid or runtime-id
-          start_or_update_on_fork(action: :update_on_fork, tags: TagBuilder.call(settings))
+          start_or_update_on_fork(action: :update_on_fork, tags: self.class.latest_tags(settings))
+        end
+
+        def report_unhandled_exception(exception, settings: Datadog.configuration)
+          # Maximum number of stack frames to include in exception crash reports
+          # This is the same number used for profiling and signal-based crashtracking
+          max_exception_stack_frames = 400
+
+          current_tags = self.class.latest_tags(settings)
+          # extract all frame data upfront; c expects exactly 3 elements, proper types, no nils
+          # limit to max_exception_stack_frames frames
+          all_backtrace_locations = exception.backtrace_locations || []
+          was_truncated = all_backtrace_locations.length > max_exception_stack_frames
+
+          backtrace_slice = all_backtrace_locations[0...max_exception_stack_frames] || []
+          # @type var frames_data: Array[[String, String, Integer]]
+          frames_data = backtrace_slice.map do |loc|
+            file = loc.path
+            file = '<unknown>' if file.nil? || file.empty? || !file.is_a?(String)
+
+            function = loc.label
+            function = '<unknown>' if function.nil? || function.empty? || !function.is_a?(String)
+
+            line = loc.lineno
+            line = 0 if line.nil? || line < 0 || !line.is_a?(Integer)
+
+            [file, function, line]
+          end
+
+          # Add truncation indicator frame if we had to cut off frames
+          if was_truncated
+            truncated_count = all_backtrace_locations.length - max_exception_stack_frames
+            frames_data << ['<truncated>', "<truncated #{truncated_count} more frames>", 0]
+          end
+
+          message = "Unhandled #{exception.class}: #{exception.message || "<no message>"}"
+
+          success = self.class._native_report_ruby_exception(
+            agent_base_url,
+            message,
+            frames_data,
+            current_tags.to_a,
+            Datadog::VERSION::STRING
+          )
+
+          logger.debug('Crashtracker failed to report unhandled exception to crash tracker') unless success
         end
 
         def stop
diff --git a/sig/datadog/core/crashtracking/component.rbs b/sig/datadog/core/crashtracking/component.rbs
@@ -8,6 +8,10 @@ module Datadog
           logger: untyped
         ) -> Datadog::Core::Crashtracking::Component?
 
+        def self.report_unhandled_exception: (::Exception?) -> void
+
+        def self.latest_tags: (Datadog::Core::Configuration::Settings) -> ::Hash[::String, ::String]
+
         def initialize: (
           tags: ::Hash[::String, ::String],
           agent_base_url: ::String,
@@ -32,6 +36,8 @@ module Datadog
 
         def start_or_update_on_fork: (action: :start | :update_on_fork, tags: ::Hash[::String, ::String]) -> void
 
+        def report_unhandled_exception: (::Exception, ?settings: Datadog::Core::Configuration::Settings) -> void
+
         def self._native_start_or_update_on_fork: (
           action: :start | :update_on_fork,
           agent_base_url: ::String,
@@ -42,6 +48,14 @@ module Datadog
         ) -> void
 
         def self._native_stop: -> void
+
+        def self._native_report_ruby_exception: (
+          ::String agent_base_url,
+          ::String message,
+          ::Array[[::String, ::String, ::Integer]] frames_data,
+          ::Array[[::String, ::String]] tags_as_array,
+          ::String library_version
+        ) -> bool
       end
     end
   end
diff --git a/spec/datadog/core/crashtracking/component_spec.rb b/spec/datadog/core/crashtracking/component_spec.rb
