Skip to content

bpf: introduce and use rcu_read_lock_dont_migrate #5816

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kernel-patches-daemon-bpf-rc wants to merge 7 commits into from
Closed

bpf: introduce and use rcu_read_lock_dont_migrate #5816

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b9b43aa
rcu: add rcu_read_lock_dont_migrate()
image-dragon Aug 21, 2025
1501ca9
bpf: use rcu_read_lock_dont_migrate() for bpf_cgrp_storage_free()
image-dragon Aug 21, 2025
c089316
bpf: use rcu_read_lock_dont_migrate() for bpf_inode_storage_free()
image-dragon Aug 21, 2025
e009246
bpf: use rcu_read_lock_dont_migrate() for bpf_iter_run_prog()
image-dragon Aug 21, 2025
0caba26
bpf: use rcu_read_lock_dont_migrate() for bpf_task_storage_free()
image-dragon Aug 21, 2025
dc7e9d6
bpf: use rcu_read_lock_dont_migrate() for bpf_prog_run_array_cg()
image-dragon Aug 21, 2025
c66620b
bpf: use rcu_read_lock_dont_migrate() for trampoline.c
image-dragon Aug 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions include/linux/rcupdate.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -962,6 +962,20 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
preempt_enable_notrace();
}

static __always_inline void rcu_read_lock_dont_migrate(void)
{
if (IS_ENABLED(CONFIG_PREEMPT_RCU))
migrate_disable();
rcu_read_lock();
}

static inline void rcu_read_unlock_migrate(void)
{
rcu_read_unlock();
if (IS_ENABLED(CONFIG_PREEMPT_RCU))
migrate_enable();
}

/**
* RCU_INIT_POINTER() - initialize an RCU protected pointer
* @p: The pointer to be initialized.
Expand Down
6 changes: 2 additions & 4 deletions kernel/bpf/bpf_cgrp_storage.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,7 @@ void bpf_cgrp_storage_free(struct cgroup *cgroup)
{
struct bpf_local_storage *local_storage;

migrate_disable();
rcu_read_lock();
rcu_read_lock_dont_migrate();
local_storage = rcu_dereference(cgroup->bpf_cgrp_storage);
if (!local_storage)
goto out;
Expand All @@ -55,8 +54,7 @@ void bpf_cgrp_storage_free(struct cgroup *cgroup)
bpf_local_storage_destroy(local_storage);
bpf_cgrp_storage_unlock();
out:
rcu_read_unlock();
migrate_enable();
rcu_read_unlock_migrate();
}

static struct bpf_local_storage_data *
Expand Down
6 changes: 2 additions & 4 deletions kernel/bpf/bpf_inode_storage.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,17 +62,15 @@ void bpf_inode_storage_free(struct inode *inode)
if (!bsb)
return;

migrate_disable();
rcu_read_lock();
rcu_read_lock_dont_migrate();

local_storage = rcu_dereference(bsb->storage);
if (!local_storage)
goto out;

bpf_local_storage_destroy(local_storage);
out:
rcu_read_unlock();
migrate_enable();
rcu_read_unlock_migrate();
}

static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
Expand Down
6 changes: 2 additions & 4 deletions kernel/bpf/bpf_iter.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -705,13 +705,11 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
migrate_enable();
rcu_read_unlock_trace();
} else {
rcu_read_lock();
migrate_disable();
rcu_read_lock_dont_migrate();
old_run_ctx = bpf_set_run_ctx(&run_ctx);
ret = bpf_prog_run(prog, ctx);
bpf_reset_run_ctx(old_run_ctx);
migrate_enable();
rcu_read_unlock();
rcu_read_unlock_migrate();
}

/* bpf program can only return 0 or 1:
Expand Down
6 changes: 2 additions & 4 deletions kernel/bpf/bpf_task_storage.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,7 @@ void bpf_task_storage_free(struct task_struct *task)
{
struct bpf_local_storage *local_storage;

migrate_disable();
rcu_read_lock();
rcu_read_lock_dont_migrate();

local_storage = rcu_dereference(task->bpf_storage);
if (!local_storage)
Expand All @@ -81,8 +80,7 @@ void bpf_task_storage_free(struct task_struct *task)
bpf_local_storage_destroy(local_storage);
bpf_task_storage_unlock();
out:
rcu_read_unlock();
migrate_enable();
rcu_read_unlock_migrate();
}

static void *bpf_pid_task_storage_lookup_elem(struct bpf_map *map, void *key)
Expand Down
6 changes: 2 additions & 4 deletions kernel/bpf/cgroup.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,7 @@ bpf_prog_run_array_cg(const struct cgroup_bpf *cgrp,
u32 func_ret;

run_ctx.retval = retval;
migrate_disable();
rcu_read_lock();
rcu_read_lock_dont_migrate();
array = rcu_dereference(cgrp->effective[atype]);
item = &array->items[0];
old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
Expand All @@ -88,8 +87,7 @@ bpf_prog_run_array_cg(const struct cgroup_bpf *cgrp,
item++;
}
bpf_reset_run_ctx(old_run_ctx);
rcu_read_unlock();
migrate_enable();
rcu_read_unlock_migrate();
return run_ctx.retval;
}

Expand Down
18 changes: 6 additions & 12 deletions kernel/bpf/trampoline.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -899,8 +899,7 @@ static __always_inline u64 notrace bpf_prog_start_time(void)
static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)
__acquires(RCU)
{
rcu_read_lock();
migrate_disable();
rcu_read_lock_dont_migrate();

run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);

Expand Down Expand Up @@ -949,8 +948,7 @@ static void notrace __bpf_prog_exit_recur(struct bpf_prog *prog, u64 start,

update_prog_stats(prog, start);
this_cpu_dec(*(prog->active));
migrate_enable();
rcu_read_unlock();
rcu_read_unlock_migrate();
}

static u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
Expand All @@ -960,8 +958,7 @@ static u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
/* Runtime stats are exported via actual BPF_LSM_CGROUP
* programs, not the shims.
*/
rcu_read_lock();
migrate_disable();
rcu_read_lock_dont_migrate();

run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);

Expand All @@ -974,8 +971,7 @@ static void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,
{
bpf_reset_run_ctx(run_ctx->saved_run_ctx);

migrate_enable();
rcu_read_unlock();
rcu_read_unlock_migrate();
}

u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
Expand Down Expand Up @@ -1033,8 +1029,7 @@ static u64 notrace __bpf_prog_enter(struct bpf_prog *prog,
struct bpf_tramp_run_ctx *run_ctx)
__acquires(RCU)
{
rcu_read_lock();
migrate_disable();
rcu_read_lock_dont_migrate();

run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);

Expand All @@ -1048,8 +1043,7 @@ static void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start,
bpf_reset_run_ctx(run_ctx->saved_run_ctx);

update_prog_stats(prog, start);
migrate_enable();
rcu_read_unlock();
rcu_read_unlock_migrate();
}

void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr)
Expand Down
Loading