Skip to content

bpf: cpumap/devmap: fix per-CPU bulk queue races on PREEMPT_RT #11047

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kernel-patches-daemon-bpf wants to merge 2 commits into
base: bpf_base
Choose a base branch
from
+36 −6
Open

bpf: cpumap/devmap: fix per-CPU bulk queue races on PREEMPT_RT #11047

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
07cf8b5
bpf: cpumap: fix race in bq_flush_to_queue on PREEMPT_RT
mrpre Feb 13, 2026
c17ff9e
bpf: devmap: fix race in bq_xmit_all on PREEMPT_RT
mrpre Feb 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions kernel/bpf/cpumap.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include <linux/sched.h>
#include <linux/workqueue.h>
#include <linux/kthread.h>
#include <linux/local_lock.h>
#include <linux/completion.h>
#include <trace/events/xdp.h>
#include <linux/btf_ids.h>
Expand All @@ -52,6 +53,7 @@ struct xdp_bulk_queue {
struct list_head flush_node;
struct bpf_cpu_map_entry *obj;
unsigned int count;
local_lock_t bq_lock;
};

/* Struct for every remote "destination" CPU in map */
Expand Down Expand Up @@ -451,6 +453,7 @@ __cpu_map_entry_alloc(struct bpf_map *map, struct bpf_cpumap_val *value,
for_each_possible_cpu(i) {
bq = per_cpu_ptr(rcpu->bulkq, i);
bq->obj = rcpu;
local_lock_init(&bq->bq_lock);
}

/* Alloc queue */
Expand Down Expand Up @@ -722,6 +725,8 @@ static void bq_flush_to_queue(struct xdp_bulk_queue *bq)
struct ptr_ring *q;
int i;

lockdep_assert_held(&bq->bq_lock);

if (unlikely(!bq->count))
return;

Expand Down Expand Up @@ -749,11 +754,15 @@ static void bq_flush_to_queue(struct xdp_bulk_queue *bq)
}

/* Runs under RCU-read-side, plus in softirq under NAPI protection.
* Thus, safe percpu variable access.
* Thus, safe percpu variable access. PREEMPT_RT relies on
* local_lock_nested_bh() to serialise access to the per-CPU bq.
*/
static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf)
{
struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);
struct xdp_bulk_queue *bq;

local_lock_nested_bh(&rcpu->bulkq->bq_lock);
bq = this_cpu_ptr(rcpu->bulkq);

if (unlikely(bq->count == CPU_MAP_BULK_SIZE))
bq_flush_to_queue(bq);
Expand All @@ -774,6 +783,8 @@ static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf)

list_add(&bq->flush_node, flush_list);
}

local_unlock_nested_bh(&rcpu->bulkq->bq_lock);
}

int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,
Expand Down Expand Up @@ -810,7 +821,9 @@ void __cpu_map_flush(struct list_head *flush_list)
struct xdp_bulk_queue *bq, *tmp;

list_for_each_entry_safe(bq, tmp, flush_list, flush_node) {
local_lock_nested_bh(&bq->obj->bulkq->bq_lock);
bq_flush_to_queue(bq);
local_unlock_nested_bh(&bq->obj->bulkq->bq_lock);

/* If already running, costs spin_lock_irqsave + smb_mb */
wake_up_process(bq->obj->kthread);
Expand Down
25 changes: 21 additions & 4 deletions kernel/bpf/devmap.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
* types of devmap; only the lookup and insertion is different.
*/
#include <linux/bpf.h>
#include <linux/local_lock.h>
#include <net/xdp.h>
#include <linux/filter.h>
#include <trace/events/xdp.h>
Expand All @@ -60,6 +61,7 @@ struct xdp_dev_bulk_queue {
struct net_device *dev_rx;
struct bpf_prog *xdp_prog;
unsigned int count;
local_lock_t bq_lock;
};

struct bpf_dtab_netdev {
Expand Down Expand Up @@ -381,6 +383,8 @@ static void bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags)
int to_send = cnt;
int i;

lockdep_assert_held(&bq->bq_lock);

if (unlikely(!cnt))
return;

Expand Down Expand Up @@ -425,10 +429,12 @@ void __dev_flush(struct list_head *flush_list)
struct xdp_dev_bulk_queue *bq, *tmp;

list_for_each_entry_safe(bq, tmp, flush_list, flush_node) {
local_lock_nested_bh(&bq->dev->xdp_bulkq->bq_lock);
bq_xmit_all(bq, XDP_XMIT_FLUSH);
bq->dev_rx = NULL;
bq->xdp_prog = NULL;
__list_del_clearprev(&bq->flush_node);
local_unlock_nested_bh(&bq->dev->xdp_bulkq->bq_lock);
}
}

Expand All @@ -451,12 +457,16 @@ static void *__dev_map_lookup_elem(struct bpf_map *map, u32 key)

/* Runs in NAPI, i.e., softirq under local_bh_disable(). Thus, safe percpu
* variable access, and map elements stick around. See comment above
* xdp_do_flush() in filter.c.
* xdp_do_flush() in filter.c. PREEMPT_RT relies on local_lock_nested_bh()
* to serialise access to the per-CPU bq.
*/
static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
struct net_device *dev_rx, struct bpf_prog *xdp_prog)
{
struct xdp_dev_bulk_queue *bq = this_cpu_ptr(dev->xdp_bulkq);
struct xdp_dev_bulk_queue *bq;

local_lock_nested_bh(&dev->xdp_bulkq->bq_lock);
bq = this_cpu_ptr(dev->xdp_bulkq);

if (unlikely(bq->count == DEV_MAP_BULK_SIZE))
bq_xmit_all(bq, 0);
Expand All @@ -477,6 +487,8 @@ static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
}

bq->q[bq->count++] = xdpf;

local_unlock_nested_bh(&dev->xdp_bulkq->bq_lock);
}

static inline int __xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
Expand Down Expand Up @@ -1115,8 +1127,13 @@ static int dev_map_notification(struct notifier_block *notifier,
if (!netdev->xdp_bulkq)
return NOTIFY_BAD;

for_each_possible_cpu(cpu)
per_cpu_ptr(netdev->xdp_bulkq, cpu)->dev = netdev;
for_each_possible_cpu(cpu) {
struct xdp_dev_bulk_queue *bq;

bq = per_cpu_ptr(netdev->xdp_bulkq, cpu);
bq->dev = netdev;
local_lock_init(&bq->bq_lock);
}
break;
case NETDEV_UNREGISTER:
/* This rcu_read_lock/unlock pair is needed because
Expand Down
Loading