bpf: Craft non-linear skbs in BPF_PROG_TEST_RUN

This patch adds support for crafting non-linear skbs in BPF test runs
for tc programs, via a new flag BPF_F_TEST_SKB_NON_LINEAR. When this
flag is set, the size of the linear area is given by ctx->data_end, with
a minimum of ETH_HLEN always pulled in the linear area.

This is particularly useful to test support for non-linear skbs in large
codebases such as Cilium. We've had multiple bugs in the past few years
where we were missing calls to bpf_skb_pull_data(). This support in
BPF_PROG_TEST_RUN would allow us to automatically cover this case in our
BPF tests.

In addition to the selftests introduced later in the series, this patch
was tested by setting BPF_F_TEST_SKB_NON_LINEAR for all tc selftests
programs and checking test failures were expected.

Suggested-by: Daniel Borkmann <[email protected]>
Signed-off-by: Paul Chaignon <[email protected]>

Author: pchaigno

include/uapi/linux/bpf.h

@@ -1448,6 +1448,10 @@ enum {
 #define BPF_F_TEST_XDP_LIVE_FRAMES	(1U << 1)
 /* If set, apply CHECKSUM_COMPLETE to skb and validate the checksum */
 #define BPF_F_TEST_SKB_CHECKSUM_COMPLETE	(1U << 2)
+/* If set, skb will be non-linear with the size of the linear area given
+ * by ctx.data_end or at least ETH_HLEN.
+ */
+#define BPF_F_TEST_SKB_NON_LINEAR	(1U << 3)
 
 /* type for BPF_ENABLE_STATS */
 enum bpf_stats_type {

net/bpf/test_run.c

@@ -660,20 +660,29 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_memb_release, KF_RELEASE)
 BTF_KFUNCS_END(test_sk_check_kfunc_ids)
 
 static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size,
-			   u32 size, u32 headroom, u32 tailroom)
+			   u32 size, u32 headroom, u32 tailroom, bool nonlinear)
 {
 	void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
-	void *data;
+	void *data, *dst;
 
 	if (user_size < ETH_HLEN || user_size > PAGE_SIZE - headroom - tailroom)
 		return ERR_PTR(-EINVAL);
 
-	size = SKB_DATA_ALIGN(size);
-	data = kzalloc(size + headroom + tailroom, GFP_USER);
+	/* In non-linear case, data_in is copied to the paged data */
+	if (nonlinear) {
+		data = alloc_page(GFP_USER);
+	} else {
+		size = SKB_DATA_ALIGN(size);
+		data = kzalloc(size + headroom + tailroom, GFP_USER);
+	}
 	if (!data)
 		return ERR_PTR(-ENOMEM);
 
-	if (copy_from_user(data + headroom, data_in, user_size)) {
+	if (nonlinear)
+		dst = page_address(data);
+	else
+		dst = data + headroom;
+	if (copy_from_user(dst, data_in, user_size)) {
 		kfree(data);
 		return ERR_PTR(-EFAULT);
 	}
@@ -910,6 +919,12 @@ static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
 	/* cb is allowed */
 
 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb),
+			   offsetof(struct __sk_buff, data_end)))
+		return -EINVAL;
+
+	/* data_end is allowed, but not copied to skb */
+
+	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, data_end),
 			   offsetof(struct __sk_buff, tstamp)))
 		return -EINVAL;
 
@@ -984,7 +999,7 @@ static struct proto bpf_dummy_proto = {
 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
 			  union bpf_attr __user *uattr)
 {
-	bool is_l2 = false, is_direct_pkt_access = false;
+	bool is_l2 = false, is_direct_pkt_access = false, is_nonlinear = false;
 	struct net *net = current->nsproxy->net_ns;
 	struct net_device *dev = net->loopback_dev;
 	u32 size = kattr->test.data_size_in;
@@ -994,25 +1009,14 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
 	struct sock *sk = NULL;
 	u32 retval, duration;
 	int hh_len = ETH_HLEN;
+	int linear_size, ret;
 	void *data;
-	int ret;
 
-	if ((kattr->test.flags & ~BPF_F_TEST_SKB_CHECKSUM_COMPLETE) ||
+	if ((kattr->test.flags & ~(BPF_F_TEST_SKB_CHECKSUM_COMPLETE | BPF_F_TEST_SKB_NON_LINEAR)) ||
 	    kattr->test.cpu || kattr->test.batch_size)
 		return -EINVAL;
 
-	data = bpf_test_init(kattr, kattr->test.data_size_in,
-			     size, NET_SKB_PAD + NET_IP_ALIGN,
-			     SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
-	if (IS_ERR(data))
-		return PTR_ERR(data);
-
-	ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
-	if (IS_ERR(ctx)) {
-		ret = PTR_ERR(ctx);
-		ctx = NULL;
-		goto out;
-	}
+	is_nonlinear = kattr->test.flags & BPF_F_TEST_SKB_NON_LINEAR;
 
 	switch (prog->type) {
 	case BPF_PROG_TYPE_SCHED_CLS:
@@ -1029,22 +1033,60 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
 		break;
 	}
 
+	if (is_nonlinear && !is_l2)
+		return -EINVAL;
+
+	data = bpf_test_init(kattr, kattr->test.data_size_in,
+			     size, NET_SKB_PAD + NET_IP_ALIGN,
+			     SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
+			     is_nonlinear);
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
+	ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
+	if (IS_ERR(ctx)) {
+		ret = PTR_ERR(ctx);
+		ctx = NULL;
+		goto out;
+	}
+
+	linear_size = hh_len;
+	if (is_nonlinear && ctx && ctx->data_end > linear_size)
+		linear_size = ctx->data_end;
+
 	sk = sk_alloc(net, AF_UNSPEC, GFP_USER, &bpf_dummy_proto, 1);
 	if (!sk) {
 		ret = -ENOMEM;
 		goto out;
 	}
 	sock_init_data(NULL, sk);
 
-	skb = slab_build_skb(data);
+	if (is_nonlinear)
+		skb = alloc_skb(NET_SKB_PAD + NET_IP_ALIGN + size +
+				SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
+				GFP_USER);
+	else
+		skb = slab_build_skb(data);
 	if (!skb) {
 		ret = -ENOMEM;
 		goto out;
 	}
+
 	skb->sk = sk;
 
 	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
-	__skb_put(skb, size);
+
+	if (is_nonlinear) {
+		skb_fill_page_desc(skb, 0, data, 0, size);
+		skb->truesize += PAGE_SIZE;
+		skb->data_len = size;
+		skb->len = size;
+
+		/* eth_type_trans expects the Ethernet header in the linear area. */
+		__pskb_pull_tail(skb, linear_size);
+	} else {
+		__skb_put(skb, size);
+	}
 
 	data = NULL; /* data released via kfree_skb */
 
@@ -1127,9 +1169,11 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
 	convert_skb_to___skb(skb, ctx);
 
 	size = skb->len;
-	/* bpf program can never convert linear skb to non-linear */
-	if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
+	if (skb_is_nonlinear(skb)) {
+		/* bpf program can never convert linear skb to non-linear */
+		WARN_ON_ONCE(!is_nonlinear);
 		size = skb_headlen(skb);
+	}
 	ret = bpf_test_finish(kattr, uattr, skb->data, NULL, size, retval,
 			      duration);
 	if (!ret)
@@ -1139,7 +1183,8 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
 	if (dev && dev != net->loopback_dev)
 		dev_put(dev);
 	kfree_skb(skb);
-	kfree(data);
+	if (data)
+		is_nonlinear ? __free_page(data) : kfree(data);
 	if (sk)
 		sk_free(sk);
 	kfree(ctx);
@@ -1265,7 +1310,7 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
 		size = max_data_sz;
 	}
 
-	data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom);
+	data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom, false);
 	if (IS_ERR(data)) {
 		ret = PTR_ERR(data);
 		goto free_ctx;
@@ -1388,7 +1433,7 @@ int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
 	if (size < ETH_HLEN)
 		return -EINVAL;
 
-	data = bpf_test_init(kattr, kattr->test.data_size_in, size, 0, 0);
+	data = bpf_test_init(kattr, kattr->test.data_size_in, size, 0, 0, false);
 	if (IS_ERR(data))
 		return PTR_ERR(data);
 
@@ -1661,7 +1706,8 @@ int bpf_prog_test_run_nf(struct bpf_prog *prog,
 
 	data = bpf_test_init(kattr, kattr->test.data_size_in, size,
 			     NET_SKB_PAD + NET_IP_ALIGN,
-			     SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
+			     SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
+			     false);
 	if (IS_ERR(data))
 		return PTR_ERR(data);
 

tools/include/uapi/linux/bpf.h

@@ -1448,6 +1448,10 @@ enum {
 #define BPF_F_TEST_XDP_LIVE_FRAMES	(1U << 1)
 /* If set, apply CHECKSUM_COMPLETE to skb and validate the checksum */
 #define BPF_F_TEST_SKB_CHECKSUM_COMPLETE	(1U << 2)
+/* If set, skb will be non-linear with the size of the linear area given
+ * by ctx.data_end or at least ETH_HLEN.
+ */
+#define BPF_F_TEST_SKB_NON_LINEAR	(1U << 3)
 
 /* type for BPF_ENABLE_STATS */
 enum bpf_stats_type {