@@ -52,35 +52,65 @@ bool process_bucket(PerCPURecord *record, void *label_buckets, int j) {
5252 return false;
5353}
5454
55- // Go processes store the current goroutine in thread local store. From there
56- // this reads the g (aka goroutine) struct, then the m (the actual operating
57- // system thread) of that goroutine, and finally curg (current goroutine). This
58- // chain is necessary because getg().m.curg points to the current user g
59- // assigned to the thread (curg == getg() when not on the system stack). curg
60- // may be nil if there is no user g, such as when running in the scheduler. If
61- // curg is nil, then g is either a system stack (called g0) or a signal handler
62- // g (gsignal). Neither one will ever have label.
6355static inline __attribute__((__always_inline__ ))
64- bool get_go_custom_labels (struct pt_regs * ctx , PerCPURecord * record , GoCustomLabelsOffsets * offs ) {
65- long res ;
56+ void process_slice_pair (PerCPURecord * record , struct GoSlice * labels_slice , int i ) {
57+ CustomLabelsArray * out = & record -> trace .custom_labels ;
58+ if (out -> len >= MAX_CUSTOM_LABELS )
59+ return ;
6660
67- size_t curg_ptr_addr ;
68- res = bpf_probe_read_user (& curg_ptr_addr , sizeof (void * ), (void * )(record -> customLabelsState .go_m_ptr + offs -> curg ));
61+ CustomLabel * lbl = & out -> labels [out -> len ];
62+ void * str_addr = (char * )labels_slice -> array + i * sizeof (struct GoString ) * 2 ;
63+ long res = bpf_probe_read_user (& record -> labels , sizeof (struct GoString )* 2 , str_addr );
6964 if (res < 0 ) {
70- DEBUG_PRINT ("cl: failed to read value for m_ptr->curg: %ld" , res );
71- return false;
65+ DEBUG_PRINT ("cl: failed to read strings from labels slice (%lx): %ld" , (unsigned long )str_addr , res );
66+ return ;
67+ }
68+ unsigned klen = MIN (record -> labels [0 ].len , CUSTOM_LABEL_MAX_KEY_LEN - 1 );
69+ res = bpf_probe_read_user (lbl -> key , klen , record -> labels [0 ].str );
70+ if (res ) {
71+ DEBUG_PRINT ("cl: failed to read key for custom label (%lx): %ld" , (unsigned long ) record -> labels [0 ].str , res );
72+ return ;
73+ }
74+ unsigned vlen = MIN (record -> labels [1 ].len , CUSTOM_LABEL_MAX_VAL_LEN - 1 );
75+ res = bpf_probe_read_user (lbl -> val , vlen , record -> labels [1 ].str );
76+ if (res ) {
77+ DEBUG_PRINT ("cl: failed to read key for custom label (%lx): %ld" , (unsigned long ) record -> labels [1 ].str , res );
78+ return ;
7279 }
80+ out -> len ++ ;
81+ }
7382
74- void * labels_map_ptr_ptr ;
75- res = bpf_probe_read_user (& labels_map_ptr_ptr , sizeof (void * ), (void * )(curg_ptr_addr + offs -> labels ));
83+ static inline __attribute__((__always_inline__ ))
84+ bool get_go_custom_labels_from_slice (struct pt_regs * ctx , PerCPURecord * record , void * labels_slice_ptr ) {
85+ //https://github.com/golang/go/blob/80e2e474/src/runtime/pprof/label.go#L20
86+ struct GoSlice labels_slice ;
87+ long res = bpf_probe_read_user (& labels_slice , sizeof (struct GoSlice ), labels_slice_ptr );
7688 if (res < 0 ) {
77- DEBUG_PRINT ("cl: failed to read value for curg->labels (%lx->%lx): %ld" , (unsigned long )curg_ptr_addr ,
78- (unsigned long ) offs -> labels , res );
89+ DEBUG_PRINT ("cl: failed to read value for labels slice (%lx): %ld" , (unsigned long )labels_slice_ptr , res );
7990 return false;
8091 }
8192
93+ u64 label_count = MIN (MAX_CUSTOM_LABELS , labels_slice .len );
94+ switch (label_count ) {
95+ case 10 : process_slice_pair (record , & labels_slice , 9 );
96+ case 9 : process_slice_pair (record , & labels_slice , 8 );
97+ case 8 : process_slice_pair (record , & labels_slice , 7 );
98+ case 7 : process_slice_pair (record , & labels_slice , 6 );
99+ case 6 : process_slice_pair (record , & labels_slice , 5 );
100+ case 5 : process_slice_pair (record , & labels_slice , 4 );
101+ case 4 : process_slice_pair (record , & labels_slice , 3 );
102+ case 3 : process_slice_pair (record , & labels_slice , 2 );
103+ case 2 : process_slice_pair (record , & labels_slice , 1 );
104+ case 1 : process_slice_pair (record , & labels_slice , 0 );
105+ }
106+
107+ return true;
108+ }
109+
110+ static inline __attribute__((__always_inline__ ))
111+ bool get_go_custom_labels_from_map (struct pt_regs * ctx , PerCPURecord * record , void * labels_map_ptr_ptr , GoCustomLabelsOffsets * offs ) {
82112 void * labels_map_ptr ;
83- res = bpf_probe_read (& labels_map_ptr , sizeof (labels_map_ptr ), labels_map_ptr_ptr );
113+ long res = bpf_probe_read (& labels_map_ptr , sizeof (labels_map_ptr ), labels_map_ptr_ptr );
84114 if (res < 0 ) {
85115 DEBUG_PRINT ("cl: failed to read value for labels_map_ptr (%lx): %ld" , (unsigned long )labels_map_ptr_ptr , res );
86116 return false;
@@ -114,10 +144,6 @@ bool get_go_custom_labels(struct pt_regs *ctx, PerCPURecord *record, GoCustomLab
114144 // and we can't support as many buckets.
115145 u64 bucket_count = MIN (MAX_CUSTOM_LABELS , 1 << log_2_bucket_count );
116146 switch (bucket_count ) {
117- case 14 : if (process_bucket (record , label_buckets , 13 )) return true;
118- case 13 : if (process_bucket (record , label_buckets , 12 )) return true;
119- case 12 : if (process_bucket (record , label_buckets , 11 )) return true;
120- case 11 : if (process_bucket (record , label_buckets , 10 )) return true;
121147 case 10 : if (process_bucket (record , label_buckets , 9 )) return true;
122148 case 9 : if (process_bucket (record , label_buckets , 8 )) return true;
123149 case 8 : if (process_bucket (record , label_buckets , 7 )) return true;
@@ -133,6 +159,41 @@ bool get_go_custom_labels(struct pt_regs *ctx, PerCPURecord *record, GoCustomLab
133159 return false;
134160}
135161
162+ // Go processes store the current goroutine in thread local store. From there
163+ // this reads the g (aka goroutine) struct, then the m (the actual operating
164+ // system thread) of that goroutine, and finally curg (current goroutine). This
165+ // chain is necessary because getg().m.curg points to the current user g
166+ // assigned to the thread (curg == getg() when not on the system stack). curg
167+ // may be nil if there is no user g, such as when running in the scheduler. If
168+ // curg is nil, then g is either a system stack (called g0) or a signal handler
169+ // g (gsignal). Neither one will ever have label.
170+ static inline __attribute__((__always_inline__ ))
171+ bool get_go_custom_labels (struct pt_regs * ctx , PerCPURecord * record , GoCustomLabelsOffsets * offs ) {
172+ long res ;
173+
174+ size_t curg_ptr_addr ;
175+ res = bpf_probe_read_user (& curg_ptr_addr , sizeof (void * ), (void * )(record -> customLabelsState .go_m_ptr + offs -> curg ));
176+ if (res < 0 ) {
177+ DEBUG_PRINT ("cl: failed to read value for m_ptr->curg: %ld" , res );
178+ return false;
179+ }
180+
181+ void * labels_ptr ;
182+ res = bpf_probe_read_user (& labels_ptr , sizeof (void * ), (void * )(curg_ptr_addr + offs -> labels ));
183+ if (res < 0 ) {
184+ DEBUG_PRINT ("cl: failed to read value for curg->labels (%lx->%lx): %ld" , (unsigned long )curg_ptr_addr ,
185+ (unsigned long ) offs -> labels , res );
186+ return false;
187+ }
188+
189+ if (offs -> hmap_buckets == 0 ) {
190+ // go 1.24+ labels is a slice
191+ return get_go_custom_labels_from_slice (ctx , record , labels_ptr );
192+ }
193+
194+ // go 1.23- labels is a map
195+ return get_go_custom_labels_from_map (ctx , record , labels_ptr , offs );
196+ }
136197
137198SEC ("perf_event/go_labels" )
138199int go_labels (struct pt_regs * ctx ) {
0 commit comments