Unwind past Go runtime.morestack (#184)

In Go, [`runtime.morestack`](https://github.com/golang/go/blob/7b60d06739/src/runtime/asm_amd64.s#L680-L680) is called by goroutines that have exhausted their stack space and need to create more. This can be a significant bottleneck in some programs. As can be seen from the above link, the function in question clears the frame pointer before calling into `newstack`, making it impossible for us to unwind further. Luckily, before doing so, it stashes the necessary registers on the current goroutine.

This PR introduces a new unwinding command specifically for unwinding past `runtime.morestack`. It uses the infrastructure already developed for Go Custom Labels to find the current goroutine, then grabs the appropriate values. Currently, this only works in Go 1.25 and does not work for kernel stacks; making it more general will be done in a [later follow-up](parca-dev/parca-agent#3124).

Author: umanwizard

nativeunwind/elfunwindinfo/elfgopclntab.go

@@ -36,6 +36,8 @@ var goFunctionsStopDelta = map[string]*sdtypes.UnwindInfo{
 	// signal return frame
 	"runtime.sigreturn":            &sdtypes.UnwindInfoSignal,
 	"runtime.sigreturn__sigaction": &sdtypes.UnwindInfoSignal,
+
+	"runtime.morestack": &sdtypes.UnwindInfoGoMorestack,
 }
 
 const (

nativeunwind/stackdeltatypes/stackdeltatypes.go

@@ -60,6 +60,13 @@ var UnwindInfoLR = UnwindInfo{
 	FPOpcode: support.UnwindOpcodeBaseLR,
 }
 
+// UnwindInfoGoMorestack contains the description to unwind past the Go
+// "runtime.morestack" function.
+var UnwindInfoGoMorestack = UnwindInfo{
+	Opcode: support.UnwindOpcodeCommand,
+	Param: support.UnwindCommandGoMorestack,
+}
+
 // StackDelta defines the start address for the delta interval, along with
 // the unwind information.
 type StackDelta struct {

support/ebpf/go_support.h

@@ -0,0 +1,47 @@
+#ifndef OPTI_GOROUTINE_H
+#define OPTI_GOROUTINE_H
+
+#include "bpfdefs.h"
+#include "tsd.h"
+#include "types.h"
+
+static EBPF_INLINE void *get_go_m_ptr(struct GoLabelsOffsets *offs, UNUSED UnwindState *state)
+{
+  u64 g_addr     = 0;
+  void *tls_base = NULL;
+  if (tsd_get_base(&tls_base) < 0) {
+    DEBUG_PRINT("cl mptr new: failed to get tsd base; can't read m_ptr");
+    return NULL;
+  }
+  DEBUG_PRINT(
+    "cl mptr new: read tsd_base at 0x%lx, g offset: %d", (unsigned long)tls_base, offs->tls_offset);
+
+  if (offs->tls_offset == 0 || tls_base == 0) {
+#if defined(__aarch64__)
+    // On aarch64 for !iscgo programs the g is only stored in r28 register.
+    g_addr = state->r28;
+#elif defined(__x86_64__)
+    DEBUG_PRINT("cl mptr new: TLS offset for g pointer missing for amd64");
+    return NULL;
+#endif
+  }
+
+  if (g_addr == 0) {
+    if (bpf_probe_read_user(&g_addr, sizeof(void *), (void *)((s64)tls_base + offs->tls_offset))) {
+      DEBUG_PRINT("cl mptr new: failed to read g_addr, tls_base(%lx)", (unsigned long)tls_base);
+      return NULL;
+    }
+  }
+
+  DEBUG_PRINT(
+    "cl mptr new: reading m_ptr_addr at 0x%lx + 0x%x", (unsigned long)g_addr, offs->m_offset);
+  void *m_ptr_addr;
+  if (bpf_probe_read_user(&m_ptr_addr, sizeof(void *), (void *)(g_addr + offs->m_offset))) {
+    DEBUG_PRINT("cl: failed m_ptr_addr");
+    return NULL;
+  }
+  DEBUG_PRINT("cl mptr new: returning 0x%lx", (unsigned long)m_ptr_addr);
+  return m_ptr_addr;
+}
+
+#endif

support/ebpf/interpreter_dispatcher.ebpf.c

@@ -3,6 +3,7 @@
 // perf event and will call the appropriate tracer for a given process
 
 #include "bpfdefs.h"
+#include "go_support.h"
 #include "kernel.h"
 #include "tracemgmt.h"
 #include "tsd.h"
@@ -151,65 +152,28 @@ get_m_ptr_legacy(struct GoCustomLabelsOffsets *offs, UNUSED UnwindState *state)
   void *tls_base    = NULL;
   res               = tsd_get_base(&tls_base);
   if (res < 0) {
-    DEBUG_PRINT("cl: failed to get tsd base; can't read m_ptr");
+    DEBUG_PRINT("cl mptr legacy: failed to get tsd base; can't read m_ptr");
     return NULL;
   }
 
   res = bpf_probe_read_user(&g_addr, sizeof(void *), (void *)((u64)tls_base + g_addr_offset));
   if (res < 0) {
-    DEBUG_PRINT("cl: failed to read g_addr, tls_base(%lx)", (unsigned long)tls_base);
+    DEBUG_PRINT("cl mptr legacy: failed to read g_addr, tls_base(%lx)", (unsigned long)tls_base);
     return NULL;
   }
 #elif defined(__aarch64__)
   g_addr = state->r28;
 #endif
 
-  DEBUG_PRINT("cl: reading m_ptr_addr at 0x%lx + 0x%x", g_addr, offs->m_offset);
+  DEBUG_PRINT("cl mptr legacy: reading m_ptr_addr at 0x%lx + 0x%x", g_addr, offs->m_offset);
   void *m_ptr_addr;
   res = bpf_probe_read_user(&m_ptr_addr, sizeof(void *), (void *)(g_addr + offs->m_offset));
   if (res < 0) {
-    DEBUG_PRINT("cl: failed m_ptr_addr");
+    DEBUG_PRINT("cl mptr legacy: failed m_ptr_addr");
     return NULL;
   }
 
-  return m_ptr_addr;
-}
-
-static EBPF_INLINE void *get_m_ptr(struct GoLabelsOffsets *offs, UNUSED UnwindState *state)
-{
-  u64 g_addr     = 0;
-  void *tls_base = NULL;
-  if (tsd_get_base(&tls_base) < 0) {
-    DEBUG_PRINT("cl: failed to get tsd base; can't read m_ptr");
-    return NULL;
-  }
-  DEBUG_PRINT(
-    "cl: read tsd_base at 0x%lx, g offset: %d", (unsigned long)tls_base, offs->tls_offset);
-
-  if (offs->tls_offset == 0) {
-#if defined(__aarch64__)
-    // On aarch64 for !iscgo programs the g is only stored in r28 register.
-    g_addr = state->r28;
-#elif defined(__x86_64__)
-    DEBUG_PRINT("cl: TLS offset for g pointer missing for amd64");
-    return NULL;
-#endif
-  }
-
-  if (g_addr == 0) {
-    if (bpf_probe_read_user(&g_addr, sizeof(void *), (void *)((s64)tls_base + offs->tls_offset))) {
-      DEBUG_PRINT("cl: failed to read g_addr, tls_base(%lx)", (unsigned long)tls_base);
-      return NULL;
-    }
-  }
-
-  DEBUG_PRINT("cl: reading m_ptr_addr at 0x%lx + 0x%x", (unsigned long)g_addr, offs->m_offset);
-  void *m_ptr_addr;
-  if (bpf_probe_read_user(&m_ptr_addr, sizeof(void *), (void *)(g_addr + offs->m_offset))) {
-    DEBUG_PRINT("cl: failed m_ptr_addr");
-    return NULL;
-  }
-  DEBUG_PRINT("cl: m_ptr_addr 0x%lx", (unsigned long)m_ptr_addr);
+  DEBUG_PRINT("cl mptr legacy: returning 0x%llx", (u64)m_ptr_addr);
   return m_ptr_addr;
 }
 
@@ -244,7 +208,7 @@ static EBPF_INLINE void maybe_add_go_custom_labels(struct pt_regs *ctx, PerCPURe
     return;
   }
 
-  void *m_ptr_addr = get_m_ptr(offsets, &record->state);
+  void *m_ptr_addr = get_go_m_ptr(offsets, &record->state);
   if (!m_ptr_addr) {
     return;
   }

support/ebpf/native_stack_trace.ebpf.c

@@ -357,9 +357,10 @@ static EBPF_INLINE u64 unwind_register_address(UnwindState *state, u64 cfa, u8 o
 // is marked with UNWIND_COMMAND_STOP which marks entry points (main function,
 // thread spawn function, signal handlers, ...).
 #if defined(__x86_64__)
-static EBPF_INLINE ErrorCode unwind_one_frame(UnwindState *state, bool *stop)
+static EBPF_INLINE ErrorCode unwind_one_frame(PerCPURecord *record, bool *stop)
 {
-  *stop = false;
+  UnwindState *state = &record->state;
+  *stop              = false;
 
   u32 unwindInfo = 0;
   u64 rt_regs[18];
@@ -408,6 +409,11 @@ static EBPF_INLINE ErrorCode unwind_one_frame(UnwindState *state, bool *stop)
         goto err_native_pc_read;
       }
       goto frame_ok;
+    case UNWIND_COMMAND_GO_MORESTACK:
+      if (!unwinder_unwind_go_morestack(record)) {
+        goto err_native_pc_read;
+      }
+      goto frame_ok;
     default: return ERR_UNREACHABLE;
     }
   } else {
@@ -451,9 +457,10 @@ static EBPF_INLINE ErrorCode unwind_one_frame(UnwindState *state, bool *stop)
   return ERR_OK;
 }
 #elif defined(__aarch64__)
-static EBPF_INLINE ErrorCode unwind_one_frame(struct UnwindState *state, bool *stop)
+static EBPF_INLINE ErrorCode unwind_one_frame(struct PerCPURecord *record, bool *stop)
 {
-  *stop = false;
+  UnwindState *state = &record->state;
+  *stop              = false;
 
   u32 unwindInfo = 0;
   int addrDiff   = 0;
@@ -497,6 +504,11 @@ static EBPF_INLINE ErrorCode unwind_one_frame(struct UnwindState *state, bool *s
         goto err_native_pc_read;
       }
       goto frame_ok;
+    case UNWIND_COMMAND_GO_MORESTACK:
+      if (!unwinder_unwind_go_morestack(record)) {
+        goto err_native_pc_read;
+      }
+      goto frame_ok;
     default: return ERR_UNREACHABLE;
     }
   }
@@ -618,7 +630,7 @@ static EBPF_INLINE int unwind_native(struct pt_regs *ctx)
 
     // Unwind the native frame using stack deltas. Stop if no next frame.
     bool stop;
-    error = unwind_one_frame(&record->state, &stop);
+    error = unwind_one_frame(record, &stop);
     if (error || stop) {
       break;
     }

support/ebpf/stackdeltatypes.h

@@ -26,6 +26,8 @@
 #define UNWIND_COMMAND_SIGNAL        3
 // Unwind using standard frame pointer
 #define UNWIND_COMMAND_FRAME_POINTER 4
+// Unwind past the Go runtime.morestack function
+#define UNWIND_COMMAND_GO_MORESTACK  5
 
 // If opcode has UNWIND_OPCODEF_DEREF set, the lowest bits of 'param' are used
 // as second adder as post-deref operation. This contains the mask for that.

support/ebpf/tracemgmt.h

@@ -7,6 +7,7 @@
 #include "errors.h"
 #include "extmaps.h"
 #include "frametypes.h"
+#include "go_support.h"
 #include "types.h"
 
 #if defined(TESTING_COREDUMP)
@@ -32,17 +33,6 @@
 
 #endif // TESTING_COREDUMP
 
-// increment_metric increments the value of the given metricID by 1
-static inline EBPF_INLINE void increment_metric(u32 metricID)
-{
-  u64 *count = bpf_map_lookup_elem(&metrics, &metricID);
-  if (count) {
-    ++*count;
-  } else {
-    DEBUG_PRINT("Failed to lookup metrics map for metricID %d", metricID);
-  }
-}
-
 // Send immediate notifications for event triggers to Go.
 // Notifications for GENERIC_PID and TRACES_FOR_SYMBOLIZATION will be
 // automatically inhibited until HA resets the type.
@@ -316,6 +306,44 @@ static inline EBPF_INLINE bool unwinder_unwind_frame_pointer(UnwindState *state)
   return true;
 }
 
+static inline EBPF_INLINE bool unwinder_unwind_go_morestack(PerCPURecord *record)
+{
+  GoLabelsOffsets *offs = bpf_map_lookup_elem(&go_labels_procs, &record->trace.pid);
+  if (!offs) {
+    DEBUG_PRINT("morestack: failed to read go labels offsets");
+    return false;
+  }
+  void *mptr = get_go_m_ptr(offs, &record->state);
+  DEBUG_PRINT("morestack: curg offset: %d, mptr: %llx\n", offs->curg, (u64)mptr);
+
+  size_t curg_ptr_addr;
+  if (bpf_probe_read_user(&curg_ptr_addr, sizeof(void *), (void *)((u64)mptr + offs->curg))) {
+    DEBUG_PRINT("morestack: failed to read value for m_ptr->curg");
+    return false;
+  }
+
+  DEBUG_PRINT("morestack: curg is %lx\n", curg_ptr_addr);
+
+  // Valid since go 1.25:
+  // https://github.com/golang/go/blob/7b60d06739/src/runtime/runtime2.go#L303-L322
+  // On previous versions, there was an extra "ret" value, so "bp" is one spot later.
+  // TODO - make this work on earlier versions.
+  unsigned long regs[6];
+  if (bpf_probe_read_user(regs, sizeof(regs), (void *)(curg_ptr_addr + 56 /* XXX */))) {
+    DEBUG_PRINT("morestack: failed to read regs");
+    return false;
+  }
+  record->state.sp = regs[0];
+  record->state.pc = regs[1];
+  record->state.fp = regs[5];
+  DEBUG_PRINT(
+    "morestack: success, sp is %llx, pc is %llx, fp is %llx",
+    record->state.sp,
+    record->state.pc,
+    record->state.fp);
+  return true;
+}
+
 // Push the file ID, line number and frame type into FrameList with a user-defined
 // maximum stack size.
 //

support/ebpf/tracer.ebpf.amd64

@@ -2,6 +2,8 @@
 #define OPTI_TSD_H
 
 #include "bpfdefs.h"
+#include "types.h"
+#include "util.h"
 
 // tsd_read reads from the Thread Specific Data location associated with the provided key.
 static inline EBPF_INLINE int