Skip to content

Commit da58ea3

Browse files
AsphalttKernel Patches Daemon
Asphaltt
authored and
Kernel Patches Daemon
committed
bpf: Introduce BPF_F_CPU flag for percpu_array maps
Introduce support for the BPF_F_CPU flag in percpu_array maps to allow updating values for specified CPU or for all CPUs with a single value. This enhancement enables: * Efficient update of all CPUs using a single value when cpu == (u32)~0. * Targeted update or lookup for a specified CPU otherwise. The flag is passed via: * map_flags in bpf_percpu_array_update() along with embedded cpu field. * elem_flags in generic_map_update_batch() along with embedded cpu field. Signed-off-by: Leon Hwang <[email protected]>
1 parent 1a33a8f commit da58ea3

File tree

5 files changed

+103
-43
lines changed

5 files changed

+103
-43
lines changed

‎include/linux/bpf.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2692,7 +2692,8 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env,
26922692
struct bpf_func_state *callee);
26932693

26942694
int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value);
2695-
int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value);
2695+
int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value,
2696+
u64 flags);
26962697
int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
26972698
u64 flags);
26982699
int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,

‎include/uapi/linux/bpf.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1372,6 +1372,12 @@ enum {
13721372
BPF_NOEXIST = 1, /* create new element if it didn't exist */
13731373
BPF_EXIST = 2, /* update existing element */
13741374
BPF_F_LOCK = 4, /* spin_lock-ed map_lookup/map_update */
1375+
BPF_F_CPU = 8, /* map_update for percpu_array */
1376+
};
1377+
1378+
enum {
1379+
/* indicate updating value across all CPUs for percpu maps. */
1380+
BPF_ALL_CPUS = (__u32)~0,
13751381
};
13761382

13771383
/* flags for BPF_MAP_CREATE command */

‎kernel/bpf/arraymap.c‎

Lines changed: 41 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -295,28 +295,40 @@ static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key,
295295
return per_cpu_ptr(array->pptrs[index & array->index_mask], cpu);
296296
}
297297

298-
int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)
298+
int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value, u64 flags)
299299
{
300300
struct bpf_array *array = container_of(map, struct bpf_array, map);
301301
u32 index = *(u32 *)key;
302302
void __percpu *pptr;
303-
int cpu, off = 0;
304-
u32 size;
303+
u32 size, cpu;
304+
int off = 0;
305305

306306
if (unlikely(index >= array->map.max_entries))
307307
return -ENOENT;
308308

309+
cpu = flags >> 32;
310+
flags &= (u32)~0;
311+
if (unlikely(flags > BPF_F_CPU))
312+
return -EINVAL;
313+
if (unlikely((flags & BPF_F_CPU) && cpu >= num_possible_cpus()))
314+
return -ERANGE;
315+
309316
/* per_cpu areas are zero-filled and bpf programs can only
310317
* access 'value_size' of them, so copying rounded areas
311318
* will not leak any kernel data
312319
*/
313320
size = array->elem_size;
314321
rcu_read_lock();
315322
pptr = array->pptrs[index & array->index_mask];
316-
for_each_possible_cpu(cpu) {
317-
copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu));
318-
check_and_init_map_value(map, value + off);
319-
off += size;
323+
if (flags & BPF_F_CPU) {
324+
copy_map_value_long(map, value, per_cpu_ptr(pptr, cpu));
325+
check_and_init_map_value(map, value);
326+
} else {
327+
for_each_possible_cpu(cpu) {
328+
copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu));
329+
check_and_init_map_value(map, value + off);
330+
off += size;
331+
}
320332
}
321333
rcu_read_unlock();
322334
return 0;
@@ -387,13 +399,20 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
387399
struct bpf_array *array = container_of(map, struct bpf_array, map);
388400
u32 index = *(u32 *)key;
389401
void __percpu *pptr;
390-
int cpu, off = 0;
391-
u32 size;
402+
u32 size, cpu;
403+
int off = 0;
392404

393-
if (unlikely(map_flags > BPF_EXIST))
405+
cpu = map_flags >> 32;
406+
map_flags &= (u32)~0;
407+
if (unlikely(map_flags > BPF_F_CPU))
394408
/* unknown flags */
395409
return -EINVAL;
396410

411+
if (unlikely((map_flags & BPF_F_CPU) && cpu != BPF_ALL_CPUS &&
412+
cpu >= num_possible_cpus()))
413+
/* invalid cpu */
414+
return -ERANGE;
415+
397416
if (unlikely(index >= array->map.max_entries))
398417
/* all elements were pre-allocated, cannot insert a new one */
399418
return -E2BIG;
@@ -411,10 +430,19 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
411430
size = array->elem_size;
412431
rcu_read_lock();
413432
pptr = array->pptrs[index & array->index_mask];
414-
for_each_possible_cpu(cpu) {
415-
copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value + off);
433+
if ((map_flags & BPF_F_CPU) && cpu != BPF_ALL_CPUS) {
434+
copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value);
416435
bpf_obj_free_fields(array->map.record, per_cpu_ptr(pptr, cpu));
417-
off += size;
436+
} else {
437+
for_each_possible_cpu(cpu) {
438+
copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value + off);
439+
/* same user-provided value is used if BPF_F_CPU is specified,
440+
* otherwise value is an array of per-cpu values.
441+
*/
442+
if (!(map_flags & BPF_F_CPU))
443+
off += size;
444+
bpf_obj_free_fields(array->map.record, per_cpu_ptr(pptr, cpu));
445+
}
418446
}
419447
rcu_read_unlock();
420448
return 0;

‎kernel/bpf/syscall.c‎

Lines changed: 48 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -131,8 +131,11 @@ bool bpf_map_write_active(const struct bpf_map *map)
131131
return atomic64_read(&map->writecnt) != 0;
132132
}
133133

134-
static u32 bpf_map_value_size(const struct bpf_map *map)
134+
static u32 bpf_map_value_size(const struct bpf_map *map, u64 flags)
135135
{
136+
if ((flags & BPF_F_CPU) && map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
137+
return round_up(map->value_size, 8);
138+
136139
if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
137140
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
138141
map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
@@ -314,7 +317,7 @@ static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,
314317
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
315318
err = bpf_percpu_hash_copy(map, key, value);
316319
} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
317-
err = bpf_percpu_array_copy(map, key, value);
320+
err = bpf_percpu_array_copy(map, key, value, flags);
318321
} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
319322
err = bpf_percpu_cgroup_storage_copy(map, key, value);
320323
} else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
@@ -1669,7 +1672,10 @@ static int map_lookup_elem(union bpf_attr *attr)
16691672
if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
16701673
return -EINVAL;
16711674

1672-
if (attr->flags & ~BPF_F_LOCK)
1675+
if ((u32)attr->flags & ~(BPF_F_LOCK | BPF_F_CPU))
1676+
return -EINVAL;
1677+
1678+
if (!((u32)attr->flags & BPF_F_CPU) && attr->flags >> 32)
16731679
return -EINVAL;
16741680

16751681
CLASS(fd, f)(attr->map_fd);
@@ -1679,15 +1685,15 @@ static int map_lookup_elem(union bpf_attr *attr)
16791685
if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ))
16801686
return -EPERM;
16811687

1682-
if ((attr->flags & BPF_F_LOCK) &&
1688+
if (((u32)attr->flags & BPF_F_LOCK) &&
16831689
!btf_record_has_field(map->record, BPF_SPIN_LOCK))
16841690
return -EINVAL;
16851691

16861692
key = __bpf_copy_key(ukey, map->key_size);
16871693
if (IS_ERR(key))
16881694
return PTR_ERR(key);
16891695

1690-
value_size = bpf_map_value_size(map);
1696+
value_size = bpf_map_value_size(map, attr->flags);
16911697

16921698
err = -ENOMEM;
16931699
value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
@@ -1744,19 +1750,24 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr)
17441750
goto err_put;
17451751
}
17461752

1747-
if ((attr->flags & BPF_F_LOCK) &&
1753+
if (((u32)attr->flags & BPF_F_LOCK) &&
17481754
!btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
17491755
err = -EINVAL;
17501756
goto err_put;
17511757
}
17521758

1759+
if (!((u32)attr->flags & BPF_F_CPU) && attr->flags >> 32) {
1760+
err = -EINVAL;
1761+
goto err_put;
1762+
}
1763+
17531764
key = ___bpf_copy_key(ukey, map->key_size);
17541765
if (IS_ERR(key)) {
17551766
err = PTR_ERR(key);
17561767
goto err_put;
17571768
}
17581769

1759-
value_size = bpf_map_value_size(map);
1770+
value_size = bpf_map_value_size(map, attr->flags);
17601771
value = kvmemdup_bpfptr(uvalue, value_size);
17611772
if (IS_ERR(value)) {
17621773
err = PTR_ERR(value);
@@ -1942,6 +1953,25 @@ int generic_map_delete_batch(struct bpf_map *map,
19421953
return err;
19431954
}
19441955

1956+
static int check_map_batch_elem_flags(struct bpf_map *map, u64 elem_flags)
1957+
{
1958+
u32 flags = elem_flags;
1959+
1960+
if (flags & ~(BPF_F_LOCK | BPF_F_CPU))
1961+
return -EINVAL;
1962+
1963+
if ((flags & BPF_F_LOCK) && !btf_record_has_field(map->record, BPF_SPIN_LOCK))
1964+
return -EINVAL;
1965+
1966+
if (!(flags & BPF_F_CPU) && elem_flags >> 32)
1967+
return -EINVAL;
1968+
1969+
if ((flags & BPF_F_CPU) && map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY)
1970+
return -EINVAL;
1971+
1972+
return 0;
1973+
}
1974+
19451975
int generic_map_update_batch(struct bpf_map *map, struct file *map_file,
19461976
const union bpf_attr *attr,
19471977
union bpf_attr __user *uattr)
@@ -1952,15 +1982,11 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file,
19521982
void *key, *value;
19531983
int err = 0;
19541984

1955-
if (attr->batch.elem_flags & ~BPF_F_LOCK)
1956-
return -EINVAL;
1957-
1958-
if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1959-
!btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
1960-
return -EINVAL;
1961-
}
1985+
err = check_map_batch_elem_flags(map, attr->batch.elem_flags);
1986+
if (err)
1987+
return err;
19621988

1963-
value_size = bpf_map_value_size(map);
1989+
value_size = bpf_map_value_size(map, attr->batch.elem_flags);
19641990

19651991
max_count = attr->batch.count;
19661992
if (!max_count)
@@ -1986,9 +2012,7 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file,
19862012
copy_from_user(value, values + cp * value_size, value_size))
19872013
break;
19882014

1989-
err = bpf_map_update_value(map, map_file, key, value,
1990-
attr->batch.elem_flags);
1991-
2015+
err = bpf_map_update_value(map, map_file, key, value, attr->batch.elem_flags);
19922016
if (err)
19932017
break;
19942018
cond_resched();
@@ -2015,14 +2039,11 @@ int generic_map_lookup_batch(struct bpf_map *map,
20152039
u32 value_size, cp, max_count;
20162040
int err;
20172041

2018-
if (attr->batch.elem_flags & ~BPF_F_LOCK)
2019-
return -EINVAL;
2020-
2021-
if ((attr->batch.elem_flags & BPF_F_LOCK) &&
2022-
!btf_record_has_field(map->record, BPF_SPIN_LOCK))
2023-
return -EINVAL;
2042+
err = check_map_batch_elem_flags(map, attr->batch.elem_flags);
2043+
if (err)
2044+
return err;
20242045

2025-
value_size = bpf_map_value_size(map);
2046+
value_size = bpf_map_value_size(map, attr->batch.elem_flags);
20262047

20272048
max_count = attr->batch.count;
20282049
if (!max_count)
@@ -2056,9 +2077,7 @@ int generic_map_lookup_batch(struct bpf_map *map,
20562077
rcu_read_unlock();
20572078
if (err)
20582079
break;
2059-
err = bpf_map_copy_value(map, key, value,
2060-
attr->batch.elem_flags);
2061-
2080+
err = bpf_map_copy_value(map, key, value, attr->batch.elem_flags);
20622081
if (err == -ENOENT)
20632082
goto next_key;
20642083

@@ -2144,7 +2163,7 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr)
21442163
goto err_put;
21452164
}
21462165

2147-
value_size = bpf_map_value_size(map);
2166+
value_size = bpf_map_value_size(map, 0);
21482167

21492168
err = -ENOMEM;
21502169
value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);

‎tools/include/uapi/linux/bpf.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1372,6 +1372,12 @@ enum {
13721372
BPF_NOEXIST = 1, /* create new element if it didn't exist */
13731373
BPF_EXIST = 2, /* update existing element */
13741374
BPF_F_LOCK = 4, /* spin_lock-ed map_lookup/map_update */
1375+
BPF_F_CPU = 8, /* map_update for percpu_array */
1376+
};
1377+
1378+
enum {
1379+
/* indicate updating value across all CPUs for percpu maps. */
1380+
BPF_ALL_CPUS = (__u32)~0,
13751381
};
13761382

13771383
/* flags for BPF_MAP_CREATE command */

0 commit comments

Comments
 (0)