diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index 513c8e9704f65..4f2d3268a6769 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -160,13 +160,23 @@ static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first) return ret; } -static inline void xsk_buff_del_tail(struct xdp_buff *tail) +static inline void xsk_buff_del_frag(struct xdp_buff *xdp) { - struct xdp_buff_xsk *xskb = container_of(tail, struct xdp_buff_xsk, xdp); + struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); list_del(&xskb->list_node); } +static inline struct xdp_buff *xsk_buff_get_head(struct xdp_buff *first) +{ + struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp); + struct xdp_buff_xsk *frag; + + frag = list_first_entry(&xskb->pool->xskb_list, struct xdp_buff_xsk, + list_node); + return &frag->xdp; +} + static inline struct xdp_buff *xsk_buff_get_tail(struct xdp_buff *first) { struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp); @@ -389,8 +399,13 @@ static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first) return NULL; } -static inline void xsk_buff_del_tail(struct xdp_buff *tail) +static inline void xsk_buff_del_frag(struct xdp_buff *xdp) +{ +} + +static inline struct xdp_buff *xsk_buff_get_head(struct xdp_buff *first) { + return NULL; } static inline struct xdp_buff *xsk_buff_get_tail(struct xdp_buff *first) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 4e47992361ea1..80c64b42596c9 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -12235,6 +12235,7 @@ enum special_kfunc_type { KF_bpf_dynptr_from_skb, KF_bpf_dynptr_from_xdp, KF_bpf_dynptr_from_skb_meta, + KF_bpf_xdp_pull_data, KF_bpf_dynptr_slice, KF_bpf_dynptr_slice_rdwr, KF_bpf_dynptr_clone, @@ -12285,10 +12286,12 @@ BTF_ID(func, bpf_rbtree_right) BTF_ID(func, bpf_dynptr_from_skb) BTF_ID(func, bpf_dynptr_from_xdp) BTF_ID(func, bpf_dynptr_from_skb_meta) +BTF_ID(func, bpf_xdp_pull_data) #else BTF_ID_UNUSED BTF_ID_UNUSED BTF_ID_UNUSED +BTF_ID_UNUSED #endif BTF_ID(func, bpf_dynptr_slice) BTF_ID(func, bpf_dynptr_slice_rdwr) @@ -12358,6 +12361,11 @@ static bool is_kfunc_bpf_preempt_enable(struct bpf_kfunc_call_arg_meta *meta) return meta->func_id == special_kfunc_list[KF_bpf_preempt_enable]; } +static bool is_kfunc_pkt_changing(struct bpf_kfunc_call_arg_meta *meta) +{ + return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data]; +} + static enum kfunc_ptr_arg_type get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, @@ -14077,6 +14085,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } + if (is_kfunc_pkt_changing(&meta)) + clear_all_pkt_pointers(env); + nargs = btf_type_vlen(meta.func_proto); args = (const struct btf_param *)(meta.func_proto + 1); for (i = 0; i < nargs; i++) { @@ -17794,6 +17805,8 @@ static int visit_insn(int t, struct bpf_verifier_env *env) */ if (ret == 0 && is_kfunc_sleepable(&meta)) mark_subprog_might_sleep(env, t); + if (ret == 0 && is_kfunc_pkt_changing(&meta)) + mark_subprog_changes_pkt_data(env, t); } return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL); diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 4a862d6053861..6fcfa31ebf1d4 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -1207,8 +1207,8 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, { bool do_live = (kattr->test.flags & BPF_F_TEST_XDP_LIVE_FRAMES); u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); + u32 retval = 0, duration, max_data_sz, linear_data_sz; u32 batch_size = kattr->test.batch_size; - u32 retval = 0, duration, max_data_sz; u32 size = kattr->test.data_size_in; u32 headroom = XDP_PACKET_HEADROOM; u32 repeat = kattr->test.repeat; @@ -1246,7 +1246,7 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, if (ctx) { /* There can't be user provided data before the meta data */ - if (ctx->data_meta || ctx->data_end != size || + if (ctx->data_meta || ctx->data_end > size || ctx->data > ctx->data_end || unlikely(xdp_metalen_invalid(ctx->data)) || (do_live && (kattr->test.data_out || kattr->test.ctx_out))) @@ -1256,11 +1256,12 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, } max_data_sz = PAGE_SIZE - headroom - tailroom; - if (size > max_data_sz) { + linear_data_sz = (ctx && ctx->data_end != size) ? ctx->data_end : max_data_sz; + if (size > linear_data_sz) { /* disallow live data mode for jumbo frames */ if (do_live) goto free_ctx; - size = max_data_sz; + size = linear_data_sz; } data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom); diff --git a/net/core/filter.c b/net/core/filter.c index 63f3baee2dafa..ad1a66fd1cfa5 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -4153,34 +4153,43 @@ static int bpf_xdp_frags_increase_tail(struct xdp_buff *xdp, int offset) return 0; } -static void bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink, +static void bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink, bool tail, enum xdp_mem_type mem_type, bool release) { - struct xdp_buff *zc_frag = xsk_buff_get_tail(xdp); + struct xdp_buff *zc_frag = tail? xsk_buff_get_tail(xdp) : + xsk_buff_get_head(xdp); if (release) { - xsk_buff_del_tail(zc_frag); + xsk_buff_del_frag(zc_frag); __xdp_return(0, mem_type, false, zc_frag); } else { - zc_frag->data_end -= shrink; + if (tail) + zc_frag->data_end -= shrink; + else + zc_frag->data += shrink; } } static bool bpf_xdp_shrink_data(struct xdp_buff *xdp, skb_frag_t *frag, - int shrink) + int shrink, bool tail) { enum xdp_mem_type mem_type = xdp->rxq->mem.type; bool release = skb_frag_size(frag) == shrink; if (mem_type == MEM_TYPE_XSK_BUFF_POOL) { - bpf_xdp_shrink_data_zc(xdp, shrink, mem_type, release); + bpf_xdp_shrink_data_zc(xdp, shrink, tail, mem_type, release); goto out; } if (release) __xdp_return(skb_frag_netmem(frag), mem_type, false, NULL); - out: + if (!release) { + if (!tail) + skb_frag_off_add(frag, shrink); + skb_frag_size_sub(frag, shrink); + } + return release; } @@ -4198,12 +4207,8 @@ static int bpf_xdp_frags_shrink_tail(struct xdp_buff *xdp, int offset) len_free += shrink; offset -= shrink; - if (bpf_xdp_shrink_data(xdp, frag, shrink)) { + if (bpf_xdp_shrink_data(xdp, frag, shrink, true)) n_frags_free++; - } else { - skb_frag_size_sub(frag, shrink); - break; - } } sinfo->nr_frags -= n_frags_free; sinfo->xdp_frags_size -= len_free; @@ -12206,6 +12211,57 @@ __bpf_kfunc int bpf_sock_ops_enable_tx_tstamp(struct bpf_sock_ops_kern *skops, return 0; } +__bpf_kfunc int bpf_xdp_pull_data(struct xdp_md *x, u32 len, u64 flags) +{ + struct xdp_buff *xdp = (struct xdp_buff *)x; + struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); + void *data_end, *data_hard_end = xdp_data_hard_end(xdp); + int i, delta, buff_len, n_frags_free = 0, len_free = 0; + + buff_len = xdp_get_buff_len(xdp); + + if (unlikely(len > buff_len)) + return -EINVAL; + + if (!len) + len = xdp_get_buff_len(xdp); + + data_end = xdp->data + len; + delta = data_end - xdp->data_end; + + if (delta <= 0) + return 0; + + if (unlikely(data_end > data_hard_end)) + return -EINVAL; + + for (i = 0; i < sinfo->nr_frags && delta; i++) { + skb_frag_t *frag = &sinfo->frags[i]; + u32 shrink = min_t(u32, delta, skb_frag_size(frag)); + + memcpy(xdp->data_end + len_free, skb_frag_address(frag), shrink); + + len_free += shrink; + delta -= shrink; + if (bpf_xdp_shrink_data(xdp, frag, shrink, false)) + n_frags_free++; + } + + for (i = 0; i < sinfo->nr_frags - n_frags_free; i++) { + memcpy(&sinfo->frags[i], &sinfo->frags[i + n_frags_free], + sizeof(skb_frag_t)); + } + + sinfo->nr_frags -= n_frags_free; + sinfo->xdp_frags_size -= len_free; + xdp->data_end = data_end; + + if (unlikely(!sinfo->nr_frags)) + xdp_buff_clear_frags_flag(xdp); + + return 0; +} + __bpf_kfunc_end_defs(); int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags, @@ -12233,6 +12289,7 @@ BTF_KFUNCS_END(bpf_kfunc_check_set_skb_meta) BTF_KFUNCS_START(bpf_kfunc_check_set_xdp) BTF_ID_FLAGS(func, bpf_dynptr_from_xdp) +BTF_ID_FLAGS(func, bpf_xdp_pull_data) BTF_KFUNCS_END(bpf_kfunc_check_set_xdp) BTF_KFUNCS_START(bpf_kfunc_check_set_sock_addr) diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c b/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c index 46e0730174ed6..178292d1251a1 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c @@ -97,9 +97,7 @@ void test_xdp_context_test_run(void) /* Meta data must be 255 bytes or smaller */ test_xdp_context_error(prog_fd, opts, 0, 256, sizeof(data), 0, 0, 0); - /* Total size of data must match data_end - data_meta */ - test_xdp_context_error(prog_fd, opts, 0, sizeof(__u32), - sizeof(data) - 1, 0, 0, 0); + /* Total size of data must be data_end - data_meta or larger */ test_xdp_context_error(prog_fd, opts, 0, sizeof(__u32), sizeof(data) + 1, 0, 0, 0); diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_pull_data.c b/tools/testing/selftests/bpf/prog_tests/xdp_pull_data.c new file mode 100644 index 0000000000000..1e1cd48f8e4f1 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/xdp_pull_data.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "test_xdp_pull_data.skel.h" + +/* xdp_pull_data_prog will directly read a marker 0xbb stored at buf[1024] + * so caller expecting XDP_PASS should always pass pull_len no less than 1024 + */ +void test_xdp_pull_data_common(struct test_xdp_pull_data *skel, + int buf_len, int linear_len, + int pull_len, int retval) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct xdp_md ctx = {}; + int prog_fd, err; + __u8 *buf; + + buf = calloc(buf_len, sizeof(__u8)); + if (!ASSERT_OK_PTR(buf, "calloc buf")) + return; + + buf[1023] = 0xaa; + buf[1024] = 0xbb; + buf[1025] = 0xcc; + + topts.data_in = buf; + topts.data_out = buf; + topts.data_size_in = buf_len; + topts.data_size_out = buf_len; + ctx.data_end = linear_len; + topts.ctx_in = &ctx; + topts.ctx_out = &ctx; + topts.ctx_size_in = sizeof(ctx); + topts.ctx_size_out = sizeof(ctx); + + skel->bss->linear_len = linear_len; + skel->bss->pull_len = pull_len; + + prog_fd = bpf_program__fd(skel->progs.xdp_pull_data_prog); + err = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(err, "bpf_prog_test_run_opts"); + ASSERT_EQ(topts.retval, retval, "xdp_pull_data_prog retval"); + + if (retval == XDP_DROP) + goto out; + + ASSERT_EQ(ctx.data_end, pull_len, "linear data size"); + ASSERT_EQ(topts.data_size_out, buf_len, "linear + non-linear data size"); + /* Make sure data around xdp->data_end was not messed up + * by bpf_xdp_pull_data() */ + ASSERT_EQ(buf[1023], 0xaa, "buf[1023]"); + ASSERT_EQ(buf[1024], 0xbb, "buf[1024]"); + ASSERT_EQ(buf[1025], 0xcc, "buf[1025]"); +out: + free(buf); +} + +static void test_xdp_pull_data_basic(void) +{ + struct test_xdp_pull_data *skel; + u32 page_size; + + skel = test_xdp_pull_data__open_and_load(); + if (!ASSERT_OK_PTR(skel, "test_xdp_pull_data__open_and_load")) + return; + + page_size = sysconf(_SC_PAGE_SIZE); + + /* linear xdp pkt, pull 0 byte */ + test_xdp_pull_data_common(skel, 2048, 2048, 2048, XDP_PASS); + /* multi-buf pkt, pull results in linear xdp pkt */ + test_xdp_pull_data_common(skel, 2048, 1024, 2048, XDP_PASS); + /* multi-buf pkt, pull 1 byte to linear data area */ + test_xdp_pull_data_common(skel, 9000, 1024, 1025, XDP_PASS); + /* multi-buf pkt, pull 0 byte to linear data area */ + test_xdp_pull_data_common(skel, 9000, 1025, 1025, XDP_PASS); + + /* linear xdp pkt, pull more than total data len */ + test_xdp_pull_data_common(skel, 2048, 2048, 2049, XDP_DROP); + /* multi-buf pkt with no space left in linear data area. + * Since ctx.data_end (4096) > max_data_sz, bpf_prog_test_run_xdp() + * will fill the whole linear data area and put the reset into a + * fragment. + */ + test_xdp_pull_data_common(skel, page_size, page_size, page_size, XDP_DROP); + + test_xdp_pull_data__destroy(skel); +} + +void test_xdp_pull_data(void) +{ + if (test__start_subtest("xdp_pull_data")) + test_xdp_pull_data_basic(); +} diff --git a/tools/testing/selftests/bpf/progs/test_xdp_pull_data.c b/tools/testing/selftests/bpf/progs/test_xdp_pull_data.c new file mode 100644 index 0000000000000..f32e6b4a79f5f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_xdp_pull_data.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include + +int _version SEC("version") = 1; + +int linear_len; +int pull_len; + +SEC("xdp.frags") +int xdp_pull_data_prog(struct xdp_md *xdp) +{ + __u8 *data_end = (void *)(long)xdp->data_end; + __u8 *data = (void *)(long)xdp->data; + __u8 *val_p; + int err; + + if (linear_len != data_end - data) + return XDP_DROP; + + err = bpf_xdp_pull_data(xdp, pull_len, 0); + if (err) + return XDP_DROP; + + val_p = (void *)(long)xdp->data + 1024; + if (val_p + 1 > (void *)(long)xdp->data_end) + return XDP_DROP; + + if (*val_p != 0xbb) + return XDP_DROP; + + return XDP_PASS; +} + +char _license[] SEC("license") = "GPL";