Skip to content

Commit befd01d

Browse files
AsphalttKernel Patches Daemon
Asphaltt
authored and
Kernel Patches Daemon
committed
bpf: Introduce BPF_F_CPU and BPF_F_ALL_CPUS flags
Introduce BPF_F_CPU and BPF_F_ALL_CPUS flags and the following internal helper functions for percpu maps: * bpf_percpu_copy_to_user: For lookup_elem and lookup_batch user APIs, copy data to user-provided value pointer. * bpf_percpu_copy_from_user: For update_elem and update_batch user APIs, copy data from user-provided value pointer. * bpf_map_check_cpu_flags: Check BPF_F_CPU, BPF_F_ALL_CPUS and cpu info in flags. And, get the correct value size for these user APIs. Signed-off-by: Leon Hwang <[email protected]>
1 parent b38cb5e commit befd01d

File tree

4 files changed

+103
-14
lines changed

4 files changed

+103
-14
lines changed

‎include/linux/bpf.h‎

Lines changed: 86 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -547,6 +547,56 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src
547547
bpf_obj_memcpy(map->record, dst, src, map->value_size, true);
548548
}
549549

550+
#ifdef CONFIG_BPF_SYSCALL
551+
static inline void bpf_percpu_copy_to_user(struct bpf_map *map, void __percpu *pptr, void *value,
552+
u32 size, u64 flags)
553+
{
554+
int current_cpu = raw_smp_processor_id();
555+
int cpu, off = 0;
556+
557+
if (flags & BPF_F_CPU) {
558+
cpu = flags >> 32;
559+
copy_map_value_long(map, value, cpu != current_cpu ? per_cpu_ptr(pptr, cpu) :
560+
this_cpu_ptr(pptr));
561+
check_and_init_map_value(map, value);
562+
} else {
563+
for_each_possible_cpu(cpu) {
564+
copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu));
565+
check_and_init_map_value(map, value + off);
566+
off += size;
567+
}
568+
}
569+
}
570+
571+
void bpf_obj_free_fields(const struct btf_record *rec, void *obj);
572+
573+
static inline void bpf_percpu_copy_from_user(struct bpf_map *map, void __percpu *pptr, void *value,
574+
u32 size, u64 flags)
575+
{
576+
int current_cpu = raw_smp_processor_id();
577+
int cpu, off = 0;
578+
void *ptr;
579+
580+
if (flags & BPF_F_CPU) {
581+
cpu = flags >> 32;
582+
ptr = cpu == current_cpu ? this_cpu_ptr(pptr) : per_cpu_ptr(pptr, cpu);
583+
copy_map_value_long(map, ptr, value);
584+
bpf_obj_free_fields(map->record, ptr);
585+
} else {
586+
for_each_possible_cpu(cpu) {
587+
copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value + off);
588+
/* same user-provided value is used if
589+
* BPF_F_ALL_CPUS is specified, otherwise value is
590+
* an array of per-cpu values.
591+
*/
592+
if (!(flags & BPF_F_ALL_CPUS))
593+
off += size;
594+
bpf_obj_free_fields(map->record, per_cpu_ptr(pptr, cpu));
595+
}
596+
}
597+
}
598+
#endif
599+
550600
static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src)
551601
{
552602
unsigned long *src_uptr, *dst_uptr;
@@ -2417,7 +2467,6 @@ struct btf_record *btf_record_dup(const struct btf_record *rec);
24172467
bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b);
24182468
void bpf_obj_free_timer(const struct btf_record *rec, void *obj);
24192469
void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj);
2420-
void bpf_obj_free_fields(const struct btf_record *rec, void *obj);
24212470
void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu);
24222471

24232472
struct bpf_map *bpf_map_get(u32 ufd);
@@ -3709,14 +3758,25 @@ int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char *
37093758
const char **linep, int *nump);
37103759
struct bpf_prog *bpf_prog_find_from_stack(void);
37113760

3761+
static inline bool bpf_map_supports_cpu_flags(enum bpf_map_type map_type)
3762+
{
3763+
return false;
3764+
}
3765+
37123766
static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 extra_flags_mask)
37133767
{
3714-
if (extra_flags_mask && (flags & extra_flags_mask))
3768+
if (extra_flags_mask && ((u32)flags & extra_flags_mask))
37153769
return -EINVAL;
37163770

37173771
if ((flags & BPF_F_LOCK) && !btf_record_has_field(map->record, BPF_SPIN_LOCK))
37183772
return -EINVAL;
37193773

3774+
if (!(flags & BPF_F_CPU) && flags >> 32)
3775+
return -EINVAL;
3776+
3777+
if ((flags & (BPF_F_CPU | BPF_F_ALL_CPUS)) && !bpf_map_supports_cpu_flags(map->map_type))
3778+
return -EINVAL;
3779+
37203780
return 0;
37213781
}
37223782

@@ -3725,7 +3785,7 @@ static inline int bpf_map_check_update_flags(struct bpf_map *map, u64 flags)
37253785
return bpf_map_check_op_flags(map, flags, 0);
37263786
}
37273787

3728-
#define BPF_MAP_LOOKUP_ELEM_EXTRA_FLAGS_MASK (~BPF_F_LOCK)
3788+
#define BPF_MAP_LOOKUP_ELEM_EXTRA_FLAGS_MASK (~(BPF_F_LOCK | BPF_F_CPU | BPF_F_ALL_CPUS))
37293789

37303790
static inline int bpf_map_check_lookup_flags(struct bpf_map *map, u64 flags)
37313791
{
@@ -3737,4 +3797,27 @@ static inline int bpf_map_check_batch_flags(struct bpf_map *map, u64 flags)
37373797
return bpf_map_check_op_flags(map, flags, BPF_MAP_LOOKUP_ELEM_EXTRA_FLAGS_MASK);
37383798
}
37393799

3800+
static inline int bpf_map_check_cpu_flags(u64 flags, bool check_all_cpus_flag)
3801+
{
3802+
const u64 cpu_flags = BPF_F_CPU | BPF_F_ALL_CPUS;
3803+
u32 cpu;
3804+
3805+
if (check_all_cpus_flag) {
3806+
if (unlikely((u32)flags > BPF_F_ALL_CPUS))
3807+
/* unknown flags */
3808+
return -EINVAL;
3809+
if (unlikely((flags & cpu_flags) == cpu_flags))
3810+
return -EINVAL;
3811+
} else {
3812+
if (unlikely((u32)flags & ~BPF_F_CPU))
3813+
return -EINVAL;
3814+
}
3815+
3816+
cpu = flags >> 32;
3817+
if (unlikely((flags & BPF_F_CPU) && cpu >= num_possible_cpus()))
3818+
return -ERANGE;
3819+
3820+
return 0;
3821+
}
3822+
37403823
#endif /* _LINUX_BPF_H */

‎include/uapi/linux/bpf.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1372,6 +1372,8 @@ enum {
13721372
BPF_NOEXIST = 1, /* create new element if it didn't exist */
13731373
BPF_EXIST = 2, /* update existing element */
13741374
BPF_F_LOCK = 4, /* spin_lock-ed map_lookup/map_update */
1375+
BPF_F_CPU = 8, /* cpu flag for percpu maps, upper 32-bit of flags is a cpu number */
1376+
BPF_F_ALL_CPUS = 16, /* update value across all CPUs for percpu maps */
13751377
};
13761378

13771379
/* flags for BPF_MAP_CREATE command */

‎kernel/bpf/syscall.c‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -131,12 +131,14 @@ bool bpf_map_write_active(const struct bpf_map *map)
131131
return atomic64_read(&map->writecnt) != 0;
132132
}
133133

134-
static u32 bpf_map_value_size(const struct bpf_map *map)
135-
{
136-
if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
137-
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
138-
map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
139-
map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
134+
static u32 bpf_map_value_size(const struct bpf_map *map, u64 flags)
135+
{
136+
if (flags & (BPF_F_CPU | BPF_F_ALL_CPUS))
137+
return round_up(map->value_size, 8);
138+
else if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
139+
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
140+
map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
141+
map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
140142
return round_up(map->value_size, 8) * num_possible_cpus();
141143
else if (IS_FD_MAP(map))
142144
return sizeof(u32);
@@ -1684,7 +1686,7 @@ static int map_lookup_elem(union bpf_attr *attr)
16841686
if (IS_ERR(key))
16851687
return PTR_ERR(key);
16861688

1687-
value_size = bpf_map_value_size(map);
1689+
value_size = bpf_map_value_size(map, attr->flags);
16881690

16891691
err = -ENOMEM;
16901692
value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
@@ -1751,7 +1753,7 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr)
17511753
goto err_put;
17521754
}
17531755

1754-
value_size = bpf_map_value_size(map);
1756+
value_size = bpf_map_value_size(map, attr->flags);
17551757
value = kvmemdup_bpfptr(uvalue, value_size);
17561758
if (IS_ERR(value)) {
17571759
err = PTR_ERR(value);
@@ -1951,7 +1953,7 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file,
19511953
if (err)
19521954
return err;
19531955

1954-
value_size = bpf_map_value_size(map);
1956+
value_size = bpf_map_value_size(map, attr->batch.elem_flags);
19551957

19561958
max_count = attr->batch.count;
19571959
if (!max_count)
@@ -2010,7 +2012,7 @@ int generic_map_lookup_batch(struct bpf_map *map,
20102012
if (err)
20112013
return err;
20122014

2013-
value_size = bpf_map_value_size(map);
2015+
value_size = bpf_map_value_size(map, attr->batch.elem_flags);
20142016

20152017
max_count = attr->batch.count;
20162018
if (!max_count)
@@ -2132,7 +2134,7 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr)
21322134
goto err_put;
21332135
}
21342136

2135-
value_size = bpf_map_value_size(map);
2137+
value_size = bpf_map_value_size(map, 0);
21362138

21372139
err = -ENOMEM;
21382140
value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);

‎tools/include/uapi/linux/bpf.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1372,6 +1372,8 @@ enum {
13721372
BPF_NOEXIST = 1, /* create new element if it didn't exist */
13731373
BPF_EXIST = 2, /* update existing element */
13741374
BPF_F_LOCK = 4, /* spin_lock-ed map_lookup/map_update */
1375+
BPF_F_CPU = 8, /* cpu flag for percpu maps, upper 32-bit of flags is a cpu number */
1376+
BPF_F_ALL_CPUS = 16, /* update value across all CPUs for percpu maps */
13751377
};
13761378

13771379
/* flags for BPF_MAP_CREATE command */

0 commit comments

Comments
 (0)