66// Copyright 2022 The Parca Authors
77
88#include <common.h>
9- #include <vmlinux.h>
109#include <hash.h>
10+ #include <vmlinux.h>
1111
1212#include <bpf/bpf_core_read.h>
1313#include <bpf/bpf_endian.h>
@@ -97,7 +97,15 @@ struct unwinder_stats_t {
9797 u64 error_catchall ;
9898 u64 error_should_never_happen ;
9999 u64 error_pc_not_covered ;
100- u64 error_jit ;
100+ u64 error_pc_not_covered_jit ;
101+ u64 error_jit_unupdated_mapping ;
102+ u64 error_jit_mixed_mode_disabled ; // JIT error because mixed-mode unwinding is disabled
103+ u64 error_jit_unwinding_machinery ;
104+ u64 success_jit_frame ;
105+ u64 success_jit_to_dwarf ;
106+ u64 success_dwarf_to_jit ;
107+ u64 success_dwarf_reach_bottom ;
108+ u64 success_jit_reach_bottom ;
101109};
102110
103111const volatile struct unwinder_config_t unwinder_config = {};
@@ -157,7 +165,7 @@ typedef struct {
157165
158166typedef struct {
159167 int pid ;
160- int tgid ;
168+ int tid ;
161169 int user_stack_id ;
162170 int kernel_stack_id ;
163171 int user_stack_id_dwarf ;
@@ -267,7 +275,15 @@ DEFINE_COUNTER(error_unsupported_cfa_register);
267275DEFINE_COUNTER (error_catchall );
268276DEFINE_COUNTER (error_should_never_happen );
269277DEFINE_COUNTER (error_pc_not_covered );
270- DEFINE_COUNTER (error_jit );
278+ DEFINE_COUNTER (error_pc_not_covered_jit );
279+ DEFINE_COUNTER (error_jit_unwinding_machinery );
280+ DEFINE_COUNTER (error_jit_unupdated_mapping );
281+ DEFINE_COUNTER (error_jit_mixed_mode_disabled );
282+ DEFINE_COUNTER (success_jit_frame );
283+ DEFINE_COUNTER (success_jit_to_dwarf );
284+ DEFINE_COUNTER (success_dwarf_to_jit );
285+ DEFINE_COUNTER (success_dwarf_reach_bottom );
286+ DEFINE_COUNTER (success_jit_reach_bottom );
271287
272288static void unwind_print_stats () {
273289 // Do not use the LOG macro, always print the stats.
@@ -285,9 +301,17 @@ static void unwind_print_stats() {
285301 bpf_printk ("\tunsup_cfa_reg=%lu" , unwinder_stats -> error_unsupported_cfa_register );
286302 bpf_printk ("\tcatchall=%lu" , unwinder_stats -> error_catchall );
287303 bpf_printk ("\tnever=%lu" , unwinder_stats -> error_should_never_happen );
288- bpf_printk ("\tunsup_jit=%lu" , unwinder_stats -> error_jit );
304+ bpf_printk ("\terror_jit_unwinding_machinery=%lu" , unwinder_stats -> error_jit_unwinding_machinery );
305+ bpf_printk ("\tunsup_jit=%lu" , unwinder_stats -> error_jit_unupdated_mapping );
306+ bpf_printk ("\tunsup_jit_mixed_mode_disabled=%lu" , unwinder_stats -> error_jit_mixed_mode_disabled );
307+ bpf_printk ("\tjit_frame=%lu" , unwinder_stats -> success_jit_frame );
308+ bpf_printk ("\tjit_to_dwarf_switch=%lu" , unwinder_stats -> success_jit_to_dwarf );
309+ bpf_printk ("\tdwarf_to_jit_switch=%lu" , unwinder_stats -> success_dwarf_to_jit );
310+ bpf_printk ("\treached_bottom_frame_dwarf=%lu" , unwinder_stats -> success_dwarf_reach_bottom );
311+ bpf_printk ("\treached_bottom_frame_jit=%lu" , unwinder_stats -> success_jit_reach_bottom );
289312 bpf_printk ("\ttotal_counter=%lu" , unwinder_stats -> total );
290313 bpf_printk ("\t(not_covered=%lu)" , unwinder_stats -> error_pc_not_covered );
314+ bpf_printk ("\t(not_covered_jit=%lu)" , unwinder_stats -> error_pc_not_covered_jit );
291315 bpf_printk ("" );
292316}
293317
@@ -608,11 +632,16 @@ static __always_inline void add_stack(struct bpf_perf_event_data *ctx, u64 pid_t
608632 // mean different things in kernel and user space.
609633 //
610634 // - What we call PIDs in userspace, are TGIDs in kernel space.
611- // - What we call threads IDs in user space, are PIDs in kernel space.
635+ // - What we call **thread IDs** in user space, are PIDs in kernel space.
636+ // In other words, the process ID in the lower 32 bits (kernel's view of the PID,
637+ // which in user space is usually presented as the thread ID),
638+ // and the thread group ID in the upper 32 bits
639+ // (what user space often thinks of as the PID).
640+
612641 int user_pid = pid_tgid >> 32 ;
613642 int user_tgid = pid_tgid ;
614643 stack_key .pid = user_pid ;
615- stack_key .tgid = user_tgid ;
644+ stack_key .tid = user_tgid ;
616645
617646 if (method == STACK_WALKING_METHOD_DWARF ) {
618647 int stack_hash = MurmurHash2 ((u32 * )unwind_state -> stack .addresses , MAX_STACK_DEPTH * sizeof (u64 ) / sizeof (u32 ), 0 );
@@ -664,6 +693,8 @@ int walk_user_stacktrace_impl(struct bpf_perf_event_data *ctx) {
664693 bool reached_bottom_of_stack = false;
665694 u64 zero = 0 ;
666695
696+ bool dwarf_to_jit = false;
697+
667698 unwind_state_t * unwind_state = bpf_map_lookup_elem (& heap , & zero );
668699 if (unwind_state == NULL ) {
669700 LOG ("unwind_state is NULL, should not happen" );
@@ -686,12 +717,17 @@ int walk_user_stacktrace_impl(struct bpf_perf_event_data *ctx) {
686717 if (unwind_table_result == FIND_UNWIND_JITTED ) {
687718 if (!unwinder_config .mixed_stack_enabled ) {
688719 LOG ("JIT section, stopping. Please enable mixed-mode unwinding with the --dwarf-unwinding-mixed=true to profile JITed stacks." );
720+ bump_unwind_error_jit_mixed_mode_disabled ();
689721 return 1 ;
690722 }
691723
692724 LOG ("[debug] Unwinding JITed stacks" );
693725
694726 unwind_state -> unwinding_jit = true;
727+ if (dwarf_to_jit ) {
728+ dwarf_to_jit = false;
729+ bump_unwind_success_dwarf_to_jit ();
730+ }
695731
696732 u64 next_fp = 0 ;
697733 u64 ra = 0 ;
@@ -733,6 +769,7 @@ int walk_user_stacktrace_impl(struct bpf_perf_event_data *ctx) {
733769
734770 if (next_fp == 0 ) {
735771 LOG ("[info] found bottom frame while walking JITed section" );
772+ bump_unwind_success_jit_reach_bottom ();
736773 return 1 ;
737774 }
738775
@@ -746,6 +783,7 @@ int walk_user_stacktrace_impl(struct bpf_perf_event_data *ctx) {
746783 if (len >= 0 && len < MAX_STACK_DEPTH ) {
747784 unwind_state -> stack .addresses [len ] = ra ;
748785 unwind_state -> stack .len ++ ;
786+ bump_unwind_success_jit_frame ();
749787 }
750788
751789 continue ;
@@ -799,17 +837,17 @@ int walk_user_stacktrace_impl(struct bpf_perf_event_data *ctx) {
799837 if (found_cfa_type == CFA_TYPE_END_OF_FDE_MARKER ) {
800838 LOG ("[info] PC %llx not contained in the unwind info, found marker" , unwind_state -> ip );
801839 reached_bottom_of_stack = true;
840+ bump_unwind_success_dwarf_reach_bottom (); // assuming we only have unwind tables for DWARF frames, not FP or JIT frames
802841 break ;
803842 }
804843
805844 if (found_rbp_type == RBP_TYPE_UNDEFINED_RETURN_ADDRESS ) {
806845 LOG ("[info] null return address, end of stack" , unwind_state -> ip );
807846 reached_bottom_of_stack = true;
847+ bump_unwind_success_dwarf_reach_bottom ();
808848 break ;
809849 }
810850
811- LOG ("[debug] Switching to mixed-mode unwinding" );
812-
813851 // Add address to stack.
814852 u64 len = unwind_state -> stack .len ;
815853 // Appease the verifier.
@@ -827,7 +865,8 @@ int walk_user_stacktrace_impl(struct bpf_perf_event_data *ctx) {
827865 }
828866
829867 // Set unwind_state->unwinding_jit to false once we have checked for switch from JITed unwinding to DWARF unwinding
830- if (unwind_state -> unwinding_jit ) {
868+ if (unwind_state -> unwinding_jit ) {
869+ bump_unwind_success_jit_to_dwarf ();
831870 LOG ("[debug] Switched to mixed-mode DWARF unwinding" );
832871 }
833872 unwind_state -> unwinding_jit = false;
@@ -898,7 +937,7 @@ int walk_user_stacktrace_impl(struct bpf_perf_event_data *ctx) {
898937 LOG ("[warn] mapping not added yet" );
899938 request_refresh_process_info (ctx , user_pid );
900939
901- bump_unwind_error_jit ();
940+ bump_unwind_error_jit_unupdated_mapping ();
902941 return 1 ;
903942 }
904943
@@ -952,6 +991,8 @@ int walk_user_stacktrace_impl(struct bpf_perf_event_data *ctx) {
952991 LOG ("======= reached main! =======" );
953992 add_stack (ctx , pid_tgid , STACK_WALKING_METHOD_DWARF , unwind_state );
954993 bump_unwind_success_dwarf ();
994+ // success_dwarf_to_jit keeps track of transition from DWARF unwinding to JIT unwinding
995+ dwarf_to_jit = true;
955996 } else {
956997 int user_pid = pid_tgid ;
957998 process_info_t * proc_info = bpf_map_lookup_elem (& process_info , & user_pid );
@@ -961,9 +1002,10 @@ int walk_user_stacktrace_impl(struct bpf_perf_event_data *ctx) {
9611002 }
9621003
9631004 if (proc_info -> is_jit_compiler ) {
964- LOG ("[warn] mapping not added yet rbp %llx" , unwind_state -> bp );
1005+ LOG ("[warn] mapping not added yet to BPF maps, rbp %llx" , unwind_state -> bp );
9651006 request_refresh_process_info (ctx , user_pid );
966- bump_unwind_error_jit ();
1007+ bump_unwind_error_jit_unupdated_mapping (); // rbp != 0 and we are expecting unwind info which is absent and not expecting JITed stacks and therefore are
1008+ // not symbolising JITed stacks here but maybe it's a JIT stack
9671009 return 1 ;
9681010 }
9691011
@@ -1063,7 +1105,6 @@ int profile_cpu(struct bpf_perf_event_data *ctx) {
10631105 return 0 ;
10641106 }
10651107
1066-
10671108 // 1. If we have unwind information for a process, use it.
10681109 if (has_unwind_information (user_pid )) {
10691110 bump_samples ();
@@ -1083,18 +1124,18 @@ int profile_cpu(struct bpf_perf_event_data *ctx) {
10831124 bump_unwind_error_pc_not_covered ();
10841125 return 1 ;
10851126 } else if (unwind_table_result == FIND_UNWIND_JITTED ) {
1086-
1087-
10881127 if (!unwinder_config .mixed_stack_enabled ) {
1089- LOG ("[warn] IP 0x%llx not covered, JIT (but disabled)!." , unwind_state -> ip );
1090- bump_unwind_error_jit ();
1128+ LOG ("[warn] IP 0x%llx not covered, JIT (but mixed-mode unwinding disabled)!." , unwind_state -> ip );
1129+ bump_unwind_error_pc_not_covered_jit ();
1130+ bump_unwind_error_jit_mixed_mode_disabled ();
10911131 return 1 ;
10921132 }
10931133 } else if (proc_info -> is_jit_compiler ) {
10941134 LOG ("[warn] IP 0x%llx not covered, may be JIT!." , unwind_state -> ip );
10951135 request_refresh_process_info (ctx , user_pid );
1096- // We assume this failed because of a new JIT segment.
1097- bump_unwind_error_jit ();
1136+ bump_unwind_error_pc_not_covered_jit ();
1137+ // We assume this failed because of a new JIT segment so we refresh mappings to find JIT segment in updated mappings
1138+ bump_unwind_error_jit_unupdated_mapping ();
10981139 return 1 ;
10991140 }
11001141 }
0 commit comments