Skip to content

Commit b725441

Browse files
Tao Chenanakryiko
Tao Chen
authored and
anakryiko
committed
bpf: Add attach_type field to bpf_link
Attach_type will be set when a link is created by user. It is better to record attach_type in bpf_link generically and have it available universally for all link types. So add the attach_type field in bpf_link and move the sleepable field to avoid unnecessary gap padding. Signed-off-by: Tao Chen <[email protected]> Signed-off-by: Andrii Nakryiko <[email protected]> Acked-by: Daniel Borkmann <[email protected]> Acked-by: Jiri Olsa <[email protected]> Link: https://lore.kernel.org/bpf/[email protected]
1 parent d81526a commit b725441

File tree

14 files changed

+66
-42
lines changed

14 files changed

+66
-42
lines changed

‎drivers/net/netkit.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -775,7 +775,7 @@ static int netkit_link_init(struct netkit_link *nkl,
775775
struct bpf_prog *prog)
776776
{
777777
bpf_link_init(&nkl->link, BPF_LINK_TYPE_NETKIT,
778-
&netkit_link_lops, prog);
778+
&netkit_link_lops, prog, attr->link_create.attach_type);
779779
nkl->location = attr->link_create.attach_type;
780780
nkl->dev = dev;
781781
return bpf_link_prime(&nkl->link, link_primer);

‎include/linux/bpf.h

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1729,19 +1729,22 @@ struct bpf_link {
17291729
enum bpf_link_type type;
17301730
const struct bpf_link_ops *ops;
17311731
struct bpf_prog *prog;
1732-
/* whether BPF link itself has "sleepable" semantics, which can differ
1733-
* from underlying BPF program having a "sleepable" semantics, as BPF
1734-
* link's semantics is determined by target attach hook
1735-
*/
1736-
bool sleepable;
1732+
17371733
u32 flags;
1734+
enum bpf_attach_type attach_type;
1735+
17381736
/* rcu is used before freeing, work can be used to schedule that
17391737
* RCU-based freeing before that, so they never overlap
17401738
*/
17411739
union {
17421740
struct rcu_head rcu;
17431741
struct work_struct work;
17441742
};
1743+
/* whether BPF link itself has "sleepable" semantics, which can differ
1744+
* from underlying BPF program having a "sleepable" semantics, as BPF
1745+
* link's semantics is determined by target attach hook
1746+
*/
1747+
bool sleepable;
17451748
};
17461749

17471750
struct bpf_link_ops {
@@ -2034,11 +2037,13 @@ int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog,
20342037

20352038
#if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
20362039
int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
2037-
int cgroup_atype);
2040+
int cgroup_atype,
2041+
enum bpf_attach_type attach_type);
20382042
void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog);
20392043
#else
20402044
static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
2041-
int cgroup_atype)
2045+
int cgroup_atype,
2046+
enum bpf_attach_type attach_type)
20422047
{
20432048
return -EOPNOTSUPP;
20442049
}
@@ -2528,10 +2533,11 @@ int bpf_map_new_fd(struct bpf_map *map, int flags);
25282533
int bpf_prog_new_fd(struct bpf_prog *prog);
25292534

25302535
void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2531-
const struct bpf_link_ops *ops, struct bpf_prog *prog);
2536+
const struct bpf_link_ops *ops, struct bpf_prog *prog,
2537+
enum bpf_attach_type attach_type);
25322538
void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
25332539
const struct bpf_link_ops *ops, struct bpf_prog *prog,
2534-
bool sleepable);
2540+
enum bpf_attach_type attach_type, bool sleepable);
25352541
int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer);
25362542
int bpf_link_settle(struct bpf_link_primer *primer);
25372543
void bpf_link_cleanup(struct bpf_link_primer *primer);
@@ -2883,13 +2889,13 @@ bpf_prog_inc_not_zero(struct bpf_prog *prog)
28832889

28842890
static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
28852891
const struct bpf_link_ops *ops,
2886-
struct bpf_prog *prog)
2892+
struct bpf_prog *prog, enum bpf_attach_type attach_type)
28872893
{
28882894
}
28892895

28902896
static inline void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
28912897
const struct bpf_link_ops *ops, struct bpf_prog *prog,
2892-
bool sleepable)
2898+
enum bpf_attach_type attach_type, bool sleepable)
28932899
{
28942900
}
28952901

‎kernel/bpf/bpf_iter.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -552,7 +552,8 @@ int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr,
552552
if (!link)
553553
return -ENOMEM;
554554

555-
bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog);
555+
bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog,
556+
attr->link_create.attach_type);
556557
link->tinfo = tinfo;
557558

558559
err = bpf_link_prime(&link->link, &link_primer);

‎kernel/bpf/bpf_struct_ops.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -808,7 +808,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
808808
goto reset_unlock;
809809
}
810810
bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS,
811-
&bpf_struct_ops_link_lops, prog);
811+
&bpf_struct_ops_link_lops, prog, prog->expected_attach_type);
812812
*plink++ = &link->link;
813813

814814
ksym = kzalloc(sizeof(*ksym), GFP_USER);
@@ -1351,7 +1351,8 @@ int bpf_struct_ops_link_create(union bpf_attr *attr)
13511351
err = -ENOMEM;
13521352
goto err_out;
13531353
}
1354-
bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, &bpf_struct_ops_map_lops, NULL);
1354+
bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, &bpf_struct_ops_map_lops, NULL,
1355+
attr->link_create.attach_type);
13551356

13561357
err = bpf_link_prime(&link->link, &link_primer);
13571358
if (err)

‎kernel/bpf/cgroup.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -867,7 +867,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
867867
cgrp->bpf.flags[atype] = saved_flags;
868868

869869
if (type == BPF_LSM_CGROUP) {
870-
err = bpf_trampoline_link_cgroup_shim(new_prog, atype);
870+
err = bpf_trampoline_link_cgroup_shim(new_prog, atype, type);
871871
if (err)
872872
goto cleanup;
873873
}
@@ -1495,7 +1495,7 @@ int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
14951495
goto out_put_cgroup;
14961496
}
14971497
bpf_link_init(&link->link, BPF_LINK_TYPE_CGROUP, &bpf_cgroup_link_lops,
1498-
prog);
1498+
prog, attr->link_create.attach_type);
14991499
link->cgroup = cgrp;
15001500
link->type = attr->link_create.attach_type;
15011501

‎kernel/bpf/net_namespace.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -501,7 +501,7 @@ int netns_bpf_link_create(const union bpf_attr *attr, struct bpf_prog *prog)
501501
goto out_put_net;
502502
}
503503
bpf_link_init(&net_link->link, BPF_LINK_TYPE_NETNS,
504-
&bpf_netns_link_ops, prog);
504+
&bpf_netns_link_ops, prog, type);
505505
net_link->net = net;
506506
net_link->type = type;
507507
net_link->netns_type = netns_type;

‎kernel/bpf/syscall.c

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3069,7 +3069,7 @@ static int bpf_obj_get(const union bpf_attr *attr)
30693069
*/
30703070
void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
30713071
const struct bpf_link_ops *ops, struct bpf_prog *prog,
3072-
bool sleepable)
3072+
enum bpf_attach_type attach_type, bool sleepable)
30733073
{
30743074
WARN_ON(ops->dealloc && ops->dealloc_deferred);
30753075
atomic64_set(&link->refcnt, 1);
@@ -3078,12 +3078,14 @@ void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
30783078
link->id = 0;
30793079
link->ops = ops;
30803080
link->prog = prog;
3081+
link->attach_type = attach_type;
30813082
}
30823083

30833084
void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
3084-
const struct bpf_link_ops *ops, struct bpf_prog *prog)
3085+
const struct bpf_link_ops *ops, struct bpf_prog *prog,
3086+
enum bpf_attach_type attach_type)
30853087
{
3086-
bpf_link_init_sleepable(link, type, ops, prog, false);
3088+
bpf_link_init_sleepable(link, type, ops, prog, attach_type, false);
30873089
}
30883090

30893091
static void bpf_link_free_id(int id)
@@ -3443,7 +3445,8 @@ static const struct bpf_link_ops bpf_tracing_link_lops = {
34433445
static int bpf_tracing_prog_attach(struct bpf_prog *prog,
34443446
int tgt_prog_fd,
34453447
u32 btf_id,
3446-
u64 bpf_cookie)
3448+
u64 bpf_cookie,
3449+
enum bpf_attach_type attach_type)
34473450
{
34483451
struct bpf_link_primer link_primer;
34493452
struct bpf_prog *tgt_prog = NULL;
@@ -3511,7 +3514,8 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
35113514
goto out_put_prog;
35123515
}
35133516
bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING,
3514-
&bpf_tracing_link_lops, prog);
3517+
&bpf_tracing_link_lops, prog, attach_type);
3518+
35153519
link->attach_type = prog->expected_attach_type;
35163520
link->link.cookie = bpf_cookie;
35173521

@@ -4049,7 +4053,8 @@ static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *pro
40494053
err = -ENOMEM;
40504054
goto out_put_file;
40514055
}
4052-
bpf_link_init(&link->link, BPF_LINK_TYPE_PERF_EVENT, &bpf_perf_link_lops, prog);
4056+
bpf_link_init(&link->link, BPF_LINK_TYPE_PERF_EVENT, &bpf_perf_link_lops, prog,
4057+
attr->link_create.attach_type);
40534058
link->perf_file = perf_file;
40544059

40554060
err = bpf_link_prime(&link->link, &link_primer);
@@ -4081,7 +4086,8 @@ static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *pro
40814086
#endif /* CONFIG_PERF_EVENTS */
40824087

40834088
static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
4084-
const char __user *user_tp_name, u64 cookie)
4089+
const char __user *user_tp_name, u64 cookie,
4090+
enum bpf_attach_type attach_type)
40854091
{
40864092
struct bpf_link_primer link_primer;
40874093
struct bpf_raw_tp_link *link;
@@ -4104,7 +4110,7 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
41044110
tp_name = prog->aux->attach_func_name;
41054111
break;
41064112
}
4107-
return bpf_tracing_prog_attach(prog, 0, 0, 0);
4113+
return bpf_tracing_prog_attach(prog, 0, 0, 0, attach_type);
41084114
case BPF_PROG_TYPE_RAW_TRACEPOINT:
41094115
case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
41104116
if (strncpy_from_user(buf, user_tp_name, sizeof(buf) - 1) < 0)
@@ -4126,7 +4132,7 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
41264132
goto out_put_btp;
41274133
}
41284134
bpf_link_init_sleepable(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT,
4129-
&bpf_raw_tp_link_lops, prog,
4135+
&bpf_raw_tp_link_lops, prog, attach_type,
41304136
tracepoint_is_faultable(btp->tp));
41314137
link->btp = btp;
41324138
link->cookie = cookie;
@@ -4168,7 +4174,7 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
41684174

41694175
tp_name = u64_to_user_ptr(attr->raw_tracepoint.name);
41704176
cookie = attr->raw_tracepoint.cookie;
4171-
fd = bpf_raw_tp_link_attach(prog, tp_name, cookie);
4177+
fd = bpf_raw_tp_link_attach(prog, tp_name, cookie, prog->expected_attach_type);
41724178
if (fd < 0)
41734179
bpf_prog_put(prog);
41744180
return fd;
@@ -5525,7 +5531,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)
55255531
ret = bpf_tracing_prog_attach(prog,
55265532
attr->link_create.target_fd,
55275533
attr->link_create.target_btf_id,
5528-
attr->link_create.tracing.cookie);
5534+
attr->link_create.tracing.cookie,
5535+
attr->link_create.attach_type);
55295536
break;
55305537
case BPF_PROG_TYPE_LSM:
55315538
case BPF_PROG_TYPE_TRACING:
@@ -5534,7 +5541,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)
55345541
goto out;
55355542
}
55365543
if (prog->expected_attach_type == BPF_TRACE_RAW_TP)
5537-
ret = bpf_raw_tp_link_attach(prog, NULL, attr->link_create.tracing.cookie);
5544+
ret = bpf_raw_tp_link_attach(prog, NULL, attr->link_create.tracing.cookie,
5545+
attr->link_create.attach_type);
55385546
else if (prog->expected_attach_type == BPF_TRACE_ITER)
55395547
ret = bpf_iter_link_attach(attr, uattr, prog);
55405548
else if (prog->expected_attach_type == BPF_LSM_CGROUP)
@@ -5543,7 +5551,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)
55435551
ret = bpf_tracing_prog_attach(prog,
55445552
attr->link_create.target_fd,
55455553
attr->link_create.target_btf_id,
5546-
attr->link_create.tracing.cookie);
5554+
attr->link_create.tracing.cookie,
5555+
attr->link_create.attach_type);
55475556
break;
55485557
case BPF_PROG_TYPE_FLOW_DISSECTOR:
55495558
case BPF_PROG_TYPE_SK_LOOKUP:

‎kernel/bpf/tcx.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,8 @@ static int tcx_link_init(struct tcx_link *tcx,
301301
struct net_device *dev,
302302
struct bpf_prog *prog)
303303
{
304-
bpf_link_init(&tcx->link, BPF_LINK_TYPE_TCX, &tcx_link_lops, prog);
304+
bpf_link_init(&tcx->link, BPF_LINK_TYPE_TCX, &tcx_link_lops, prog,
305+
attr->link_create.attach_type);
305306
tcx->location = attr->link_create.attach_type;
306307
tcx->dev = dev;
307308
return bpf_link_prime(&tcx->link, link_primer);

‎kernel/bpf/trampoline.c

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -674,7 +674,8 @@ static const struct bpf_link_ops bpf_shim_tramp_link_lops = {
674674

675675
static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog,
676676
bpf_func_t bpf_func,
677-
int cgroup_atype)
677+
int cgroup_atype,
678+
enum bpf_attach_type attach_type)
678679
{
679680
struct bpf_shim_tramp_link *shim_link = NULL;
680681
struct bpf_prog *p;
@@ -701,7 +702,7 @@ static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog
701702
p->expected_attach_type = BPF_LSM_MAC;
702703
bpf_prog_inc(p);
703704
bpf_link_init(&shim_link->link.link, BPF_LINK_TYPE_UNSPEC,
704-
&bpf_shim_tramp_link_lops, p);
705+
&bpf_shim_tramp_link_lops, p, attach_type);
705706
bpf_cgroup_atype_get(p->aux->attach_btf_id, cgroup_atype);
706707

707708
return shim_link;
@@ -726,7 +727,8 @@ static struct bpf_shim_tramp_link *cgroup_shim_find(struct bpf_trampoline *tr,
726727
}
727728

728729
int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
729-
int cgroup_atype)
730+
int cgroup_atype,
731+
enum bpf_attach_type attach_type)
730732
{
731733
struct bpf_shim_tramp_link *shim_link = NULL;
732734
struct bpf_attach_target_info tgt_info = {};
@@ -763,7 +765,7 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
763765

764766
/* Allocate and install new shim. */
765767

766-
shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype);
768+
shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype, attach_type);
767769
if (!shim_link) {
768770
err = -ENOMEM;
769771
goto err;

‎kernel/trace/bpf_trace.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2986,7 +2986,7 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr
29862986
}
29872987

29882988
bpf_link_init(&link->link, BPF_LINK_TYPE_KPROBE_MULTI,
2989-
&bpf_kprobe_multi_link_lops, prog);
2989+
&bpf_kprobe_multi_link_lops, prog, attr->link_create.attach_type);
29902990

29912991
err = bpf_link_prime(&link->link, &link_primer);
29922992
if (err)
@@ -3441,7 +3441,7 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr
34413441
link->link.flags = flags;
34423442

34433443
bpf_link_init(&link->link, BPF_LINK_TYPE_UPROBE_MULTI,
3444-
&bpf_uprobe_multi_link_lops, prog);
3444+
&bpf_uprobe_multi_link_lops, prog, attr->link_create.attach_type);
34453445

34463446
for (i = 0; i < cnt; i++) {
34473447
uprobes[i].uprobe = uprobe_register(d_real_inode(link->path.dentry),

0 commit comments

Comments
 (0)