Skip to content

Pass external callchain entry to get_perf_callchain #6240

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: bpf-net_base
Choose a base branch
from
+107 −42
Open

Pass external callchain entry to get_perf_callchain #6240

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3b5554d
perf: Refactor get_perf_callchain
Oct 28, 2025
9b46ee0
bpf: Hold the perf callchain entry until used completely
Oct 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion include/linux/perf_event.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ struct perf_callchain_entry_ctx {
u32 nr;
short contexts;
bool contexts_maxed;
bool add_mark;
};

typedef unsigned long (*perf_copy_f)(void *dst, const void *src,
Expand Down Expand Up @@ -1718,9 +1719,17 @@ DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);

extern void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
extern void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);

extern void __init_perf_callchain_ctx(struct perf_callchain_entry_ctx *ctx,
struct perf_callchain_entry *entry,
u32 max_stack, bool add_mark);

extern void __get_perf_callchain_kernel(struct perf_callchain_entry_ctx *ctx, struct pt_regs *regs);
extern void __get_perf_callchain_user(struct perf_callchain_entry_ctx *ctx, struct pt_regs *regs);

extern struct perf_callchain_entry *
get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
u32 max_stack, bool crosstask, bool add_mark);
u32 max_stack, bool crosstask);
extern int get_callchain_buffers(int max_stack);
extern void put_callchain_buffers(void);
extern struct perf_callchain_entry *get_callchain_entry(int *rctx);
Expand Down
61 changes: 48 additions & 13 deletions kernel/bpf/stackmap.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,13 +188,12 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
}

static struct perf_callchain_entry *
get_callchain_entry_for_task(struct task_struct *task, u32 max_depth)
get_callchain_entry_for_task(int *rctx, struct task_struct *task, u32 max_depth)
{
#ifdef CONFIG_STACKTRACE
struct perf_callchain_entry *entry;
int rctx;

entry = get_callchain_entry(&rctx);
entry = get_callchain_entry(rctx);

if (!entry)
return NULL;
Expand All @@ -216,8 +215,6 @@ get_callchain_entry_for_task(struct task_struct *task, u32 max_depth)
to[i] = (u64)(from[i]);
}

put_callchain_entry(rctx);

return entry;
#else /* CONFIG_STACKTRACE */
return NULL;
Expand Down Expand Up @@ -297,6 +294,31 @@ static long __bpf_get_stackid(struct bpf_map *map,
return id;
}

static struct perf_callchain_entry *
bpf_get_perf_callchain(int *rctx, struct pt_regs *regs, bool kernel, bool user,
int max_stack, bool crosstask)
{
struct perf_callchain_entry_ctx ctx;
struct perf_callchain_entry *entry;

entry = get_callchain_entry(rctx);
if (unlikely(!entry))
return NULL;

__init_perf_callchain_ctx(&ctx, entry, max_stack, false);
if (kernel)
__get_perf_callchain_kernel(&ctx, regs);
if (user && !crosstask)
__get_perf_callchain_user(&ctx, regs);

return entry;
}

static void bpf_put_callchain_entry(int rctx)
{
put_callchain_entry(rctx);
}

BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
u64, flags)
{
Expand All @@ -305,6 +327,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
bool user = flags & BPF_F_USER_STACK;
struct perf_callchain_entry *trace;
bool kernel = !user;
int rctx, ret;

if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
Expand All @@ -314,14 +337,15 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
if (max_depth > sysctl_perf_event_max_stack)
max_depth = sysctl_perf_event_max_stack;

trace = get_perf_callchain(regs, kernel, user, max_depth,
false, false);

trace = bpf_get_perf_callchain(&rctx, regs, kernel, user, max_depth, false);
if (unlikely(!trace))
/* couldn't fetch the stack trace */
return -EFAULT;

return __bpf_get_stackid(map, trace, flags);
ret = __bpf_get_stackid(map, trace, flags);
bpf_put_callchain_entry(rctx);

return ret;
}

const struct bpf_func_proto bpf_get_stackid_proto = {
Expand Down Expand Up @@ -415,6 +439,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
bool kernel = !user;
int err = -EINVAL;
u64 *ips;
int rctx;

if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
BPF_F_USER_BUILD_ID)))
Expand Down Expand Up @@ -449,17 +474,24 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
if (trace_in)
trace = trace_in;
else if (kernel && task)
trace = get_callchain_entry_for_task(task, max_depth);
trace = get_callchain_entry_for_task(&rctx, task, max_depth);
else
trace = get_perf_callchain(regs, kernel, user, max_depth,
crosstask, false);
trace = bpf_get_perf_callchain(&rctx, regs, kernel, user, max_depth, crosstask);

if (unlikely(!trace) || trace->nr < skip) {
if (unlikely(!trace)) {
if (may_fault)
rcu_read_unlock();
goto err_fault;
}

if (trace->nr < skip) {
if (may_fault)
rcu_read_unlock();
if (!trace_in)
bpf_put_callchain_entry(rctx);
goto err_fault;
}

trace_nr = trace->nr - skip;
trace_nr = (trace_nr <= num_elem) ? trace_nr : num_elem;
copy_len = trace_nr * elem_size;
Expand All @@ -479,6 +511,9 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
if (may_fault)
rcu_read_unlock();

if (!trace_in)
bpf_put_callchain_entry(rctx);

if (user_build_id)
stack_map_get_build_id_offset(buf, trace_nr, user, may_fault);

Expand Down
75 changes: 48 additions & 27 deletions kernel/events/callchain.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,13 +216,54 @@ static void fixup_uretprobe_trampoline_entries(struct perf_callchain_entry *entr
#endif
}

void __init_perf_callchain_ctx(struct perf_callchain_entry_ctx *ctx,
struct perf_callchain_entry *entry,
u32 max_stack, bool add_mark)

{
ctx->entry = entry;
ctx->max_stack = max_stack;
ctx->nr = entry->nr = 0;
ctx->contexts = 0;
ctx->contexts_maxed = false;
ctx->add_mark = add_mark;
}

void __get_perf_callchain_kernel(struct perf_callchain_entry_ctx *ctx, struct pt_regs *regs)
{
if (user_mode(regs))
return;

if (ctx->add_mark)
perf_callchain_store_context(ctx, PERF_CONTEXT_KERNEL);
perf_callchain_kernel(ctx, regs);
}

void __get_perf_callchain_user(struct perf_callchain_entry_ctx *ctx, struct pt_regs *regs)
{
int start_entry_idx;

if (!user_mode(regs)) {
if (current->flags & (PF_KTHREAD | PF_USER_WORKER))
return;
regs = task_pt_regs(current);
}

if (ctx->add_mark)
perf_callchain_store_context(ctx, PERF_CONTEXT_USER);

start_entry_idx = ctx->nr;
perf_callchain_user(ctx, regs);
fixup_uretprobe_trampoline_entries(ctx->entry, start_entry_idx);
}

struct perf_callchain_entry *
get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
u32 max_stack, bool crosstask, bool add_mark)
u32 max_stack, bool crosstask)
{
struct perf_callchain_entry *entry;
struct perf_callchain_entry_ctx ctx;
int rctx, start_entry_idx;
int rctx;

/* crosstask is not supported for user stacks */
if (crosstask && user && !kernel)
Expand All @@ -232,34 +273,14 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
if (!entry)
return NULL;

ctx.entry = entry;
ctx.max_stack = max_stack;
ctx.nr = entry->nr = 0;
ctx.contexts = 0;
ctx.contexts_maxed = false;
__init_perf_callchain_ctx(&ctx, entry, max_stack, true);

if (kernel && !user_mode(regs)) {
if (add_mark)
perf_callchain_store_context(&ctx, PERF_CONTEXT_KERNEL);
perf_callchain_kernel(&ctx, regs);
}

if (user && !crosstask) {
if (!user_mode(regs)) {
if (current->flags & (PF_KTHREAD | PF_USER_WORKER))
goto exit_put;
regs = task_pt_regs(current);
}
if (kernel)
__get_perf_callchain_kernel(&ctx, regs);

if (add_mark)
perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);

start_entry_idx = entry->nr;
perf_callchain_user(&ctx, regs);
fixup_uretprobe_trampoline_entries(entry, start_entry_idx);
}
if (user && !crosstask)
__get_perf_callchain_user(&ctx, regs);

exit_put:
put_callchain_entry(rctx);

return entry;
Expand Down
2 changes: 1 addition & 1 deletion kernel/events/core.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8218,7 +8218,7 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs)
return &__empty_callchain;

callchain = get_perf_callchain(regs, kernel, user,
max_stack, crosstask, true);
max_stack, crosstask);
return callchain ?: &__empty_callchain;
}

Expand Down
Loading