Skip to content

Commit 8f9cb1e

Browse files
anakryikoanakryiko
committed
snooper: demonstrate task_work and stack unwinding
WITH task_work_schedule_signal(): $ sudo ./snooper `pidof app` Snooping on tasks for PID 5331... Task: app_thread (PID=5331, TID=5332) No kernel stack User stack: 00007fbc700b90c2: random @ 0x42080+0x42 00007fbc700b8e7d: rand @ 0x41e70+0xd 00000000004012df: func_b @ 0x40129a+0x45 0000000000401283: func_a @ 0x4011f2+0x91 0000000000401476: thread_func @ 0x4013f6+0x80 00007fbc7010b39d: <no-symbol> 00007fbc7019049c: <no-symbol> Task: app (PID=5331, TID=5331) Kernel stack: ffffffff813e7e49: hrtimer_nanosleep @ 0xffffffff813e7db0+0x99 ffffffff813f1c73: common_nsleep @ 0xffffffff813f1c30+0x43 ffffffff813f2e0d: __x64_sys_clock_nanosleep @ 0xffffffff813f2d50+0xbd ffffffff8241a09a: do_syscall_64 @ 0xffffffff8241a030+0x6a ffffffff810000b0: entry_SYSCALL_64_after_hwframe @ 0xffffffff81000065+0x4b User stack: 00007fbc70158733: clock_nanosleep @ 0xe16d0+0x63 00007fbc70164827: nanosleep @ 0xed810+0x17 00007fbc70176f41: sleep @ 0xfff00+0x41 0000000000401622: main @ 0x40153c+0xe6 00007fbc7009ce08: <no-symbol> 00007fbc7009cecc: __libc_start_main @ 0x25e40+0x8c 0000000000401105: _start @ 0x4010e0+0x25 WITHOUT task_work_schedule_signal() CANNOT get correct stack trace after multiple tries: $ sudo ./snooper `pidof app` Snooping on tasks for PID 5004... Task: app_thread (PID=5004, TID=5005) No kernel stack User stack (frame pointer unwinding): 00000000004011fd: func_mux @ 0x4011f2+0xb 15c15fd673415a00: <no-symbol> $ sudo ./snooper `pidof app` Snooping on tasks for PID 5004... Task: app_thread (PID=5004, TID=5005) No kernel stack User stack (frame pointer unwinding): 00007fcbaf2490a8: random @ 0x42080+0x28 000062cbaf248e7d: <no-symbol> 0000000000401269: func_mux @ 0x4011f2+0x77 00000000004012a6: func_a @ 0x40126e+0x38 000000000040124b: func_mux @ 0x4011f2+0x59 15c15fd673415a00: <no-symbol> Note that 15c15fd673415a00 is NOT a valid address. Signed-off-by: Andrii Nakryiko <[email protected]>
1 parent 986b289 commit 8f9cb1e

File tree

5 files changed

+399
-1
lines changed

5 files changed

+399
-1
lines changed

‎examples/c/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,5 @@
1616
/lsm
1717
/cmake-build-debug/
1818
/cmake-build-release/
19+
/snooper
20+
compile_commands.json

‎examples/c/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ CARGO ?= $(shell which cargo)
3131
ifeq ($(strip $(CARGO)),)
3232
BZS_APPS :=
3333
else
34-
BZS_APPS := profile
34+
BZS_APPS := profile snooper
3535
APPS += $(BZS_APPS)
3636
# Required by libblazesym
3737
ALL_LDFLAGS += -lrt -ldl -lpthread -lm

‎examples/c/snooper.bpf.c‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2+
/* Copyright (c) 2025 Meta Platforms, Inc. */
3+
#include "vmlinux.h"
4+
#include <bpf/bpf_helpers.h>
5+
#include <bpf/bpf_tracing.h>
6+
#include <bpf/bpf_core_read.h>
7+
8+
#include "snooper.h"
9+
10+
char LICENSE[] SEC("license") = "Dual BSD/GPL";
11+
12+
struct task_state {
13+
struct task_event event;
14+
struct bpf_task_work tw;
15+
};
16+
17+
struct {
18+
__uint(type, BPF_MAP_TYPE_HASH);
19+
__uint(max_entries, 4096);
20+
__type(key, __u32);
21+
__type(value, struct task_state);
22+
} task_states SEC(".maps");
23+
24+
struct {
25+
__uint(type, BPF_MAP_TYPE_RINGBUF);
26+
__uint(max_entries, 1024 * 1024);
27+
} rb SEC(".maps");
28+
29+
/*
30+
* Frame pointer-based user stack unwinding.
31+
*
32+
* On x86_64 with frame pointers enabled (-fno-omit-frame-pointer):
33+
* [rbp + 0] = saved rbp (previous frame pointer)
34+
* [rbp + 8] = return address
35+
*
36+
* We walk the chain of frame pointers to collect return addresses.
37+
*/
38+
static int unwind_user_stack(struct task_struct *task, __u64 *stack, int max_depth)
39+
{
40+
struct pt_regs *regs;
41+
struct frame {
42+
__u64 next_fp; /* saved frame pointer (rbp) */
43+
__u64 ret_addr; /* return address */
44+
} frame;
45+
__u64 fp;
46+
unsigned i = 0;
47+
48+
regs = bpf_core_cast((void *)bpf_task_pt_regs(task), struct pt_regs);
49+
if (!(regs->cs & 3))
50+
return 0; /* not in user space mode */
51+
52+
stack[0] = regs->ip;
53+
54+
fp = regs->bp;
55+
bpf_for(i, 1, MAX_STACK_DEPTH) {
56+
/* read the frame, [fp] = next_fp, [fp+8] = ret_addr */
57+
if (bpf_copy_from_user_task(&frame, sizeof(frame), (void *)fp, task, 0))
58+
break;
59+
60+
barrier_var(i);
61+
if (i < MAX_STACK_DEPTH)
62+
stack[i] = frame.ret_addr;
63+
64+
fp = frame.next_fp;
65+
}
66+
67+
return i * sizeof(__u64);
68+
}
69+
70+
static int task_work_cb(struct bpf_map *map, void *key, void *value)
71+
{
72+
struct task_struct *task = bpf_get_current_task_btf();
73+
struct task_state *state = value;
74+
struct task_event *event = &state->event;
75+
__u32 tid = task->pid;
76+
77+
if (event->tid != task->pid) {
78+
bpf_printk("MISMATCHED PID %d != expected %d", task->pid, event->tid);
79+
goto cleanup;
80+
}
81+
82+
event->ustack_sz = unwind_user_stack(task, event->ustack, MAX_STACK_DEPTH);
83+
84+
bpf_ringbuf_output(&rb, event, sizeof(*event), 0);
85+
86+
cleanup:
87+
bpf_map_delete_elem(&task_states, key);
88+
return 0;
89+
}
90+
91+
/*
92+
* THIS DOESN'T CURRENTLY WORK:
93+
* static struct task_state empty_state;
94+
*
95+
* Verifier will complain:
96+
* bpf_task_work cannot be accessed directly by load/store
97+
*/
98+
static char empty_state[sizeof(struct task_state)];
99+
100+
SEC("iter.s/task")
101+
int snoop_tasks(struct bpf_iter__task *ctx)
102+
{
103+
struct seq_file *seq = ctx->meta->seq;
104+
struct task_struct *task = ctx->task;
105+
struct task_state *state;
106+
struct task_event *event;
107+
__u32 tid;
108+
int err;
109+
110+
if (!task)
111+
return 0;
112+
113+
tid = task->pid;
114+
115+
err = bpf_map_update_elem(&task_states, &tid, &empty_state, BPF_NOEXIST);
116+
if (err) {
117+
bpf_printk("Unexpected error adding task state for %d (%s): %d", tid, task->comm, err);
118+
return 0;
119+
}
120+
state = bpf_map_lookup_elem(&task_states, &tid);
121+
if (!state) {
122+
bpf_printk("Unexpected error fetching task state for %d (%s): %d", tid, task->comm, err);
123+
return 0;
124+
}
125+
126+
event = &state->event;
127+
event->pid = task->tgid;
128+
event->tid = task->pid;
129+
bpf_probe_read_kernel_str(event->comm, TASK_COMM_LEN, task->comm);
130+
131+
event->kstack_sz = bpf_get_task_stack(task, event->kstack, sizeof(event->kstack), 0);
132+
133+
err = bpf_task_work_schedule_signal_impl(task, &state->tw, &task_states, task_work_cb, NULL);
134+
if (err) {
135+
bpf_printk("Unexpected error scheduling task work %d (%s): %d", tid, task->comm, err);
136+
bpf_map_delete_elem(&task_states, &tid);
137+
return 0;
138+
}
139+
140+
return 0;
141+
}

‎examples/c/snooper.c‎

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2+
/* Copyright (c) 2025 Meta Platforms, Inc. */
3+
#include <assert.h>
4+
#include <stdint.h>
5+
#include <stdio.h>
6+
#include <stdlib.h>
7+
#include <string.h>
8+
#include <errno.h>
9+
#include <unistd.h>
10+
#include <signal.h>
11+
#include <bpf/libbpf.h>
12+
#include <bpf/bpf.h>
13+
14+
#include "snooper.skel.h"
15+
#include "snooper.h"
16+
#include "blazesym.h"
17+
18+
static struct blaze_symbolizer *symbolizer;
19+
static volatile bool exiting = false;
20+
21+
static void sig_handler(int sig)
22+
{
23+
exiting = true;
24+
}
25+
26+
static void print_frame(const char *name, uintptr_t input_addr, uintptr_t addr,
27+
uint64_t offset, const blaze_symbolize_code_info* code_info)
28+
{
29+
if (input_addr != 0) {
30+
printf(" %016lx: %s @ 0x%lx+0x%lx", input_addr, name, addr, offset);
31+
if (code_info != NULL && code_info->dir != NULL && code_info->file != NULL) {
32+
printf(" %s/%s:%u\n", code_info->dir, code_info->file, code_info->line);
33+
} else if (code_info != NULL && code_info->file != NULL) {
34+
printf(" %s:%u\n", code_info->file, code_info->line);
35+
} else {
36+
printf("\n");
37+
}
38+
} else {
39+
printf(" %16s %s", "", name);
40+
if (code_info != NULL && code_info->dir != NULL && code_info->file != NULL) {
41+
printf("@ %s/%s:%u [inlined]\n", code_info->dir, code_info->file, code_info->line);
42+
} else if (code_info != NULL && code_info->file != NULL) {
43+
printf("@ %s:%u [inlined]\n", code_info->file, code_info->line);
44+
} else {
45+
printf("[inlined]\n");
46+
}
47+
}
48+
}
49+
50+
static void show_stack_trace(__u64 *stack, int stack_sz, pid_t pid)
51+
{
52+
const struct blaze_symbolize_inlined_fn* inlined;
53+
const struct blaze_syms *syms;
54+
const struct blaze_sym *sym;
55+
int i, j;
56+
57+
assert(sizeof(uintptr_t) == sizeof(uint64_t));
58+
59+
if (pid) {
60+
struct blaze_symbolize_src_process src = {
61+
.type_size = sizeof(src),
62+
.pid = pid,
63+
};
64+
65+
syms = blaze_symbolize_process_abs_addrs(symbolizer, &src,
66+
(const uintptr_t *)stack, stack_sz);
67+
} else {
68+
struct blaze_symbolize_src_kernel src = {
69+
.type_size = sizeof(src),
70+
};
71+
72+
syms = blaze_symbolize_kernel_abs_addrs(symbolizer, &src,
73+
(const uintptr_t *)stack, stack_sz);
74+
}
75+
76+
if (!syms) {
77+
printf(" failed to symbolize addresses: %s\n", blaze_err_str(blaze_err_last()));
78+
return;
79+
}
80+
81+
for (i = 0; i < stack_sz; i++) {
82+
if (!syms || syms->cnt <= i || syms->syms[i].name == NULL) {
83+
printf(" %016llx: <no-symbol>\n", stack[i]);
84+
continue;
85+
}
86+
87+
sym = &syms->syms[i];
88+
print_frame(sym->name, stack[i], sym->addr, sym->offset, &sym->code_info);
89+
90+
for (j = 0; j < sym->inlined_cnt; j++) {
91+
inlined = &sym->inlined[j];
92+
print_frame(inlined->name, 0, 0, 0, &inlined->code_info);
93+
}
94+
}
95+
96+
blaze_syms_free(syms);
97+
}
98+
99+
/* Ringbuf callback for task events */
100+
static int handle_event(void *ctx, void *data, size_t size)
101+
{
102+
struct task_event *event = data;
103+
104+
printf("Task: %s (PID=%d, TID=%d)\n", event->comm, event->pid, event->tid);
105+
106+
/* Show kernel stack trace */
107+
if (event->kstack_sz > 0) {
108+
printf(" Kernel stack:\n");
109+
show_stack_trace(event->kstack, event->kstack_sz / sizeof(__u64), 0);
110+
} else if (event->kstack_sz < 0) {
111+
printf(" Kernel stack error: %d\n", event->kstack_sz);
112+
} else {
113+
printf(" No kernel stack\n");
114+
}
115+
116+
/* Show user stack trace */
117+
if (event->ustack_sz > 0) {
118+
printf(" User stack:\n");
119+
show_stack_trace(event->ustack, event->ustack_sz / sizeof(__u64), event->pid);
120+
} else if (event->ustack_sz < 0) {
121+
printf(" User stack error: %d\n", event->ustack_sz);
122+
} else {
123+
printf(" No user stack\n");
124+
}
125+
126+
printf("\n");
127+
return 0;
128+
}
129+
130+
static void show_help(const char *progname)
131+
{
132+
printf("Usage: %s <PID>\n", progname);
133+
printf(" PID Process ID to filter tasks (required)\n");
134+
}
135+
136+
int main(int argc, char **argv)
137+
{
138+
struct ring_buffer *rb = NULL;
139+
struct snooper_bpf *skel = NULL;
140+
LIBBPF_OPTS(bpf_iter_attach_opts, opts);
141+
union bpf_iter_link_info linfo;
142+
pid_t pid_filter = 0;
143+
int iter_fd = -1;
144+
int err = 0;
145+
char dummy;
146+
147+
if (argc < 2) {
148+
show_help(argv[0]);
149+
return 1;
150+
}
151+
152+
errno = 0;
153+
pid_filter = (pid_t)strtol(argv[1], NULL, 10);
154+
err = -errno;
155+
if (err != 0 || pid_filter <= 0) {
156+
fprintf(stderr, "Failed to parse PID '%s'\n", argv[1]);
157+
show_help(argv[0]);
158+
return 1;
159+
}
160+
161+
signal(SIGINT, sig_handler);
162+
signal(SIGTERM, sig_handler);
163+
164+
skel = snooper_bpf__open_and_load();
165+
if (!skel) {
166+
fprintf(stderr, "Failed to open and load BPF skeleton\n");
167+
err = -1;
168+
goto cleanup;
169+
}
170+
171+
symbolizer = blaze_symbolizer_new();
172+
if (!symbolizer) {
173+
fprintf(stderr, "Failed to create symbolizer\n");
174+
err = -1;
175+
goto cleanup;
176+
}
177+
178+
rb = ring_buffer__new(bpf_map__fd(skel->maps.rb), handle_event, NULL, NULL);
179+
if (!rb) {
180+
fprintf(stderr, "Failed to create ring buffer\n");
181+
err = -1;
182+
goto cleanup;
183+
}
184+
185+
memset(&linfo, 0, sizeof(linfo));
186+
linfo.task.pid = pid_filter;
187+
opts.link_info = &linfo;
188+
opts.link_info_len = sizeof(linfo);
189+
190+
skel->links.snoop_tasks = bpf_program__attach_iter(skel->progs.snoop_tasks, &opts);
191+
if (!skel->links.snoop_tasks) {
192+
err = -errno;
193+
fprintf(stderr, "Failed to attach BPF iterator\n");
194+
goto cleanup;
195+
}
196+
197+
iter_fd = bpf_iter_create(bpf_link__fd(skel->links.snoop_tasks));
198+
if (iter_fd < 0) {
199+
err = -errno;
200+
fprintf(stderr, "Failed to create iterator\n");
201+
goto cleanup;
202+
}
203+
204+
printf("Snooping on tasks for PID %d...\n\n", pid_filter);
205+
206+
/* trigger task iterator program */
207+
while (read(iter_fd, &dummy, sizeof(dummy)) > 0) {
208+
/* nothing */
209+
}
210+
211+
while (!exiting) {
212+
err = ring_buffer__poll(rb, 100 /* timeout */);
213+
if (err < 0 && err != -EINTR) {
214+
fprintf(stderr, "Error polling ring buffer: %d\n", err);
215+
break;
216+
}
217+
if (err == 0)
218+
break;
219+
}
220+
221+
cleanup:
222+
if (iter_fd >= 0)
223+
close(iter_fd);
224+
ring_buffer__free(rb);
225+
snooper_bpf__destroy(skel);
226+
blaze_symbolizer_free(symbolizer);
227+
228+
return err < 0 ? -err : 0;
229+
}

0 commit comments

Comments
 (0)