Dump with debugger before killing stuck worker

Author: nobu

tool/lib/dump.gdb

@@ -0,0 +1,17 @@
+set height 0
+set width 0
+set confirm off
+
+echo \n>>> Threads\n\n
+info threads
+
+echo \n>>> Machine level backtrace\n\n
+thread apply all info stack full
+
+echo \n>>> Dump Ruby level backtrace (if possible)\n\n
+call rb_vmdebug_stack_dump_all_threads()
+call fflush(stderr)
+
+echo ">>> Finish\n"
+detach
+quit

tool/lib/dump.lldb

@@ -0,0 +1,13 @@
+script print("\n>>> Threads\n\n")
+thread list
+
+script print("\n>>> Machine level backtrace\n\n")
+thread backtrace all
+
+script print("\n>>> Dump Ruby level backtrace (if possible)\n\n")
+call rb_vmdebug_stack_dump_all_threads()
+call fflush(stderr)
+
+script print(">>> Finish\n")
+detach
+quit

tool/lib/envutil.rb

@@ -79,6 +79,70 @@ def timeout(sec, klass = nil, message = nil, &blk)
   end
   module_function :timeout
 
+  class Debugger
+    @list = []
+
+    attr_accessor :name
+
+    def self.register(name, &block)
+      @list << new(name, &block)
+    end
+
+    def initialize(name, &block)
+      @name = name
+      instance_eval(&block)
+    end
+
+    def usable?; false; end
+
+    def start(pid, *args) end
+
+    def dump(pid, timeout: 60, reprieve: timeout&.div(4))
+      dpid = start(pid, *command_file(File.join(__dir__, "dump.#{name}")))
+    rescue Errno::ENOENT
+      return
+    else
+      return unless dpid
+      [[timeout, :TERM], [reprieve, :KILL]].find do |t, sig|
+        return EnvUtil.timeout(t) {Process.wait(dpid)}
+      rescue Timeout::Error
+        Process.kill(sig, dpid)
+      end
+      true
+    end
+
+    # sudo -n: --non-interactive
+    PRECOMMAND = (%[sudo -n] if /darwin/ =~ RUBY_PLATFORM)
+
+    def spawn(*args, **opts)
+      super(*PRECOMMAND, *args, **opts)
+    end
+
+    register("gdb") do
+      class << self
+        def usable?; system(*%w[gdb --batch --quiet --nx -ex exit]); end
+        def start(pid, *args)
+          spawn(*%w[gdb --batch --quiet --pid #{pid}], *args)
+        end
+        def command_file(file) "--command=#{file}"; end
+      end
+    end
+
+    register("lldb") do
+      class << self
+        def usable?; system(*%w[lldb -Q --no-lldbinit -o exit]); end
+        def start(pid, *args)
+          spawn(*%w[lldb --batch -Q --attach-pid #{pid}])
+        end
+        def command_file(file) ["--source", file]; end
+      end
+    end
+
+    def self.search
+      @debugger ||= @list.find(&:usable?)
+    end
+  end
+
   def terminate(pid, signal = :TERM, pgroup = nil, reprieve = 1)
     reprieve = apply_timeout_scale(reprieve) if reprieve
 
@@ -94,17 +158,10 @@ def terminate(pid, signal = :TERM, pgroup = nil, reprieve = 1)
       pgroup = pid
     end
 
-    lldb = true if /darwin/ =~ RUBY_PLATFORM
-
     while signal = signals.shift
 
-      if lldb and [:ABRT, :KILL].include?(signal)
-        lldb = false
-        # sudo -n: --non-interactive
-        # lldb -p: attach
-        #      -o: run command
-        system(*%W[sudo -n lldb -p #{pid} --batch -o bt\ all -o call\ rb_vmdebug_stack_dump_all_threads() -o quit])
-        true
+      if (dbg = Debugger.search) and [:ABRT, :KILL].include?(signal)
+        dbg.dump(pid)
       end
 
       begin

tool/lib/test/unit.rb

@@ -421,6 +421,7 @@ def quit(reason = :normal)
         end
 
         def kill
+          EnvUtil::Debugger.search&.dump(@pid)
           signal = RUBY_PLATFORM =~ /mswin|mingw/ ? :KILL : :SEGV
           Process.kill(signal, @pid)
           warn "worker #{to_s} does not respond; #{signal} is sent"