Skip to content

bpf: Introduce BPF_F_CPU flag for percpu_array maps #5688

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kernel-patches-daemon-bpf-rc wants to merge 3 commits into from
Closed

bpf: Introduce BPF_F_CPU flag for percpu_array maps #5688

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
def4741
bpf: Introduce BPF_F_CPU flag for percpu_array maps
Asphaltt Aug 5, 2025
375c8b4
libbpf: Support BPF_F_CPU for percpu_array maps
Asphaltt Aug 5, 2025
026b675
selftests/bpf: Add case to test BPF_F_CPU
Asphaltt Aug 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion include/linux/bpf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2692,7 +2692,8 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env,
struct bpf_func_state *callee);

int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value);
int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value);
int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value,
u64 flags);
int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
u64 flags);
int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
Expand Down
6 changes: 6 additions & 0 deletions include/uapi/linux/bpf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1372,6 +1372,12 @@ enum {
BPF_NOEXIST = 1, /* create new element if it didn't exist */
BPF_EXIST = 2, /* update existing element */
BPF_F_LOCK = 4, /* spin_lock-ed map_lookup/map_update */
BPF_F_CPU = 8, /* map_update for percpu_array */
};

enum {
/* indicate updating value across all CPUs for percpu maps. */
BPF_ALL_CPUS = (__u32)~0,
};

/* flags for BPF_MAP_CREATE command */
Expand Down
54 changes: 41 additions & 13 deletions kernel/bpf/arraymap.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,28 +295,40 @@ static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key,
return per_cpu_ptr(array->pptrs[index & array->index_mask], cpu);
}

int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)
int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value, u64 flags)
{
struct bpf_array *array = container_of(map, struct bpf_array, map);
u32 index = *(u32 *)key;
void __percpu *pptr;
int cpu, off = 0;
u32 size;
u32 size, cpu;
int off = 0;

if (unlikely(index >= array->map.max_entries))
return -ENOENT;

cpu = flags >> 32;
flags &= (u32)~0;
if (unlikely(flags > BPF_F_CPU))
return -EINVAL;
if (unlikely((flags & BPF_F_CPU) && cpu >= num_possible_cpus()))
return -ERANGE;

/* per_cpu areas are zero-filled and bpf programs can only
* access 'value_size' of them, so copying rounded areas
* will not leak any kernel data
*/
size = array->elem_size;
rcu_read_lock();
pptr = array->pptrs[index & array->index_mask];
for_each_possible_cpu(cpu) {
copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu));
check_and_init_map_value(map, value + off);
off += size;
if (flags & BPF_F_CPU) {
copy_map_value_long(map, value, per_cpu_ptr(pptr, cpu));
check_and_init_map_value(map, value);
} else {
for_each_possible_cpu(cpu) {
copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu));
check_and_init_map_value(map, value + off);
off += size;
}
}
rcu_read_unlock();
return 0;
Expand Down Expand Up @@ -387,13 +399,20 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
struct bpf_array *array = container_of(map, struct bpf_array, map);
u32 index = *(u32 *)key;
void __percpu *pptr;
int cpu, off = 0;
u32 size;
u32 size, cpu;
int off = 0;

if (unlikely(map_flags > BPF_EXIST))
cpu = map_flags >> 32;
map_flags &= (u32)~0;
if (unlikely(map_flags > BPF_F_CPU))
/* unknown flags */
return -EINVAL;

if (unlikely((map_flags & BPF_F_CPU) && cpu != BPF_ALL_CPUS &&
cpu >= num_possible_cpus()))
/* invalid cpu */
return -ERANGE;

if (unlikely(index >= array->map.max_entries))
/* all elements were pre-allocated, cannot insert a new one */
return -E2BIG;
Expand All @@ -411,10 +430,19 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
size = array->elem_size;
rcu_read_lock();
pptr = array->pptrs[index & array->index_mask];
for_each_possible_cpu(cpu) {
copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value + off);
if ((map_flags & BPF_F_CPU) && cpu != BPF_ALL_CPUS) {
copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value);
bpf_obj_free_fields(array->map.record, per_cpu_ptr(pptr, cpu));
off += size;
} else {
for_each_possible_cpu(cpu) {
copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value + off);
/* same user-provided value is used if BPF_F_CPU is specified,
* otherwise value is an array of per-cpu values.
*/
if (!(map_flags & BPF_F_CPU))
off += size;
bpf_obj_free_fields(array->map.record, per_cpu_ptr(pptr, cpu));
}
}
rcu_read_unlock();
return 0;
Expand Down
77 changes: 48 additions & 29 deletions kernel/bpf/syscall.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,11 @@ bool bpf_map_write_active(const struct bpf_map *map)
return atomic64_read(&map->writecnt) != 0;
}

static u32 bpf_map_value_size(const struct bpf_map *map)
static u32 bpf_map_value_size(const struct bpf_map *map, u64 flags)
{
if ((flags & BPF_F_CPU) && map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
return round_up(map->value_size, 8);

if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
Expand Down Expand Up @@ -314,7 +317,7 @@ static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
err = bpf_percpu_hash_copy(map, key, value);
} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
err = bpf_percpu_array_copy(map, key, value);
err = bpf_percpu_array_copy(map, key, value, flags);
} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
err = bpf_percpu_cgroup_storage_copy(map, key, value);
} else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
Expand Down Expand Up @@ -1669,7 +1672,10 @@ static int map_lookup_elem(union bpf_attr *attr)
if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
return -EINVAL;

if (attr->flags & ~BPF_F_LOCK)
if ((u32)attr->flags & ~(BPF_F_LOCK | BPF_F_CPU))
return -EINVAL;

if (!((u32)attr->flags & BPF_F_CPU) && attr->flags >> 32)
return -EINVAL;

CLASS(fd, f)(attr->map_fd);
Expand All @@ -1679,15 +1685,15 @@ static int map_lookup_elem(union bpf_attr *attr)
if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ))
return -EPERM;

if ((attr->flags & BPF_F_LOCK) &&
if (((u32)attr->flags & BPF_F_LOCK) &&
!btf_record_has_field(map->record, BPF_SPIN_LOCK))
return -EINVAL;

key = __bpf_copy_key(ukey, map->key_size);
if (IS_ERR(key))
return PTR_ERR(key);

value_size = bpf_map_value_size(map);
value_size = bpf_map_value_size(map, attr->flags);

err = -ENOMEM;
value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
Expand Down Expand Up @@ -1744,19 +1750,24 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr)
goto err_put;
}

if ((attr->flags & BPF_F_LOCK) &&
if (((u32)attr->flags & BPF_F_LOCK) &&
!btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
err = -EINVAL;
goto err_put;
}

if (!((u32)attr->flags & BPF_F_CPU) && attr->flags >> 32) {
err = -EINVAL;
goto err_put;
}

key = ___bpf_copy_key(ukey, map->key_size);
if (IS_ERR(key)) {
err = PTR_ERR(key);
goto err_put;
}

value_size = bpf_map_value_size(map);
value_size = bpf_map_value_size(map, attr->flags);
value = kvmemdup_bpfptr(uvalue, value_size);
if (IS_ERR(value)) {
err = PTR_ERR(value);
Expand Down Expand Up @@ -1942,6 +1953,25 @@ int generic_map_delete_batch(struct bpf_map *map,
return err;
}

static int check_map_batch_elem_flags(struct bpf_map *map, u64 elem_flags)
{
u32 flags = elem_flags;

if (flags & ~(BPF_F_LOCK | BPF_F_CPU))
return -EINVAL;

if ((flags & BPF_F_LOCK) && !btf_record_has_field(map->record, BPF_SPIN_LOCK))
return -EINVAL;

if (!(flags & BPF_F_CPU) && elem_flags >> 32)
return -EINVAL;

if ((flags & BPF_F_CPU) && map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY)
return -EINVAL;

return 0;
}

int generic_map_update_batch(struct bpf_map *map, struct file *map_file,
const union bpf_attr *attr,
union bpf_attr __user *uattr)
Expand All @@ -1952,15 +1982,11 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file,
void *key, *value;
int err = 0;

if (attr->batch.elem_flags & ~BPF_F_LOCK)
return -EINVAL;

if ((attr->batch.elem_flags & BPF_F_LOCK) &&
!btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
return -EINVAL;
}
err = check_map_batch_elem_flags(map, attr->batch.elem_flags);
if (err)
return err;

value_size = bpf_map_value_size(map);
value_size = bpf_map_value_size(map, attr->batch.elem_flags);

max_count = attr->batch.count;
if (!max_count)
Expand All @@ -1986,9 +2012,7 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file,
copy_from_user(value, values + cp * value_size, value_size))
break;

err = bpf_map_update_value(map, map_file, key, value,
attr->batch.elem_flags);

err = bpf_map_update_value(map, map_file, key, value, attr->batch.elem_flags);
if (err)
break;
cond_resched();
Expand All @@ -2015,14 +2039,11 @@ int generic_map_lookup_batch(struct bpf_map *map,
u32 value_size, cp, max_count;
int err;

if (attr->batch.elem_flags & ~BPF_F_LOCK)
return -EINVAL;

if ((attr->batch.elem_flags & BPF_F_LOCK) &&
!btf_record_has_field(map->record, BPF_SPIN_LOCK))
return -EINVAL;
err = check_map_batch_elem_flags(map, attr->batch.elem_flags);
if (err)
return err;

value_size = bpf_map_value_size(map);
value_size = bpf_map_value_size(map, attr->batch.elem_flags);

max_count = attr->batch.count;
if (!max_count)
Expand Down Expand Up @@ -2056,9 +2077,7 @@ int generic_map_lookup_batch(struct bpf_map *map,
rcu_read_unlock();
if (err)
break;
err = bpf_map_copy_value(map, key, value,
attr->batch.elem_flags);

err = bpf_map_copy_value(map, key, value, attr->batch.elem_flags);
if (err == -ENOENT)
goto next_key;

Expand Down Expand Up @@ -2144,7 +2163,7 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr)
goto err_put;
}

value_size = bpf_map_value_size(map);
value_size = bpf_map_value_size(map, 0);

err = -ENOMEM;
value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
Expand Down
6 changes: 6 additions & 0 deletions tools/include/uapi/linux/bpf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1372,6 +1372,12 @@ enum {
BPF_NOEXIST = 1, /* create new element if it didn't exist */
BPF_EXIST = 2, /* update existing element */
BPF_F_LOCK = 4, /* spin_lock-ed map_lookup/map_update */
BPF_F_CPU = 8, /* map_update for percpu_array */
};

enum {
/* indicate updating value across all CPUs for percpu maps. */
BPF_ALL_CPUS = (__u32)~0,
};

/* flags for BPF_MAP_CREATE command */
Expand Down
5 changes: 5 additions & 0 deletions tools/lib/bpf/bpf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,11 @@ LIBBPF_API int bpf_map_lookup_and_delete_batch(int fd, void *in_batch,
* Update spin_lock-ed map elements. This must be
* specified if the map value contains a spinlock.
*
* **BPF_F_CPU**
* As for percpu map, the cpu info is embedded into the high 32 bits of
* **opts->elem_flags**. Update value across all CPUs if cpu is (__u32)~0,
* or on specified CPU otherwise.
*
* @param fd BPF map file descriptor
* @param keys pointer to an array of *count* keys
* @param values pointer to an array of *count* values
Expand Down
28 changes: 22 additions & 6 deletions tools/lib/bpf/libbpf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10593,8 +10593,10 @@ bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
}

static int validate_map_op(const struct bpf_map *map, size_t key_sz,
size_t value_sz, bool check_value_sz)
size_t value_sz, bool check_value_sz, __u64 flags)
{
__u32 cpu;

if (!map_is_created(map)) /* map is not yet created */
return -ENOENT;

Expand All @@ -10612,6 +10614,20 @@ static int validate_map_op(const struct bpf_map *map, size_t key_sz,
if (!check_value_sz)
return 0;

if (flags & BPF_F_CPU) {
if (map->def.type != BPF_MAP_TYPE_PERCPU_ARRAY)
return -EINVAL;
cpu = flags >> 32;
if (cpu != BPF_ALL_CPUS && cpu >= libbpf_num_possible_cpus())
return -ERANGE;
if (map->def.value_size != value_sz) {
pr_warn("map '%s': unexpected value size %zu provided, expected %u\n",
map->name, value_sz, map->def.value_size);
return -EINVAL;
}
return 0;
}

switch (map->def.type) {
case BPF_MAP_TYPE_PERCPU_ARRAY:
case BPF_MAP_TYPE_PERCPU_HASH:
Expand Down Expand Up @@ -10644,7 +10660,7 @@ int bpf_map__lookup_elem(const struct bpf_map *map,
{
int err;

err = validate_map_op(map, key_sz, value_sz, true);
err = validate_map_op(map, key_sz, value_sz, true, flags);
if (err)
return libbpf_err(err);

Expand All @@ -10657,7 +10673,7 @@ int bpf_map__update_elem(const struct bpf_map *map,
{
int err;

err = validate_map_op(map, key_sz, value_sz, true);
err = validate_map_op(map, key_sz, value_sz, true, flags);
if (err)
return libbpf_err(err);

Expand All @@ -10669,7 +10685,7 @@ int bpf_map__delete_elem(const struct bpf_map *map,
{
int err;

err = validate_map_op(map, key_sz, 0, false /* check_value_sz */);
err = validate_map_op(map, key_sz, 0, false /* check_value_sz */, 0);
if (err)
return libbpf_err(err);

Expand All @@ -10682,7 +10698,7 @@ int bpf_map__lookup_and_delete_elem(const struct bpf_map *map,
{
int err;

err = validate_map_op(map, key_sz, value_sz, true);
err = validate_map_op(map, key_sz, value_sz, true, 0);
if (err)
return libbpf_err(err);

Expand All @@ -10694,7 +10710,7 @@ int bpf_map__get_next_key(const struct bpf_map *map,
{
int err;

err = validate_map_op(map, key_sz, 0, false /* check_value_sz */);
err = validate_map_op(map, key_sz, 0, false /* check_value_sz */, 0);
if (err)
return libbpf_err(err);

Expand Down
Loading
Loading