Fix script loading synchronization issue

Author: VictorNogueiraRio

include/linux/filter.h

@@ -1099,13 +1099,4 @@ struct bpf_sock_ops_kern {
 					 */
 };
 
-#ifdef CONFIG_XDP_LUA
-extern struct list_head lua_state_cpu_list;
-
-struct lua_state_cpu {
-	struct lua_State 	*L;
-	int			cpu;
-	struct list_head	list;
-};
-#endif /* CONFIG_XDP_LUA */
 #endif /* __LINUX_FILTER_H__ */

include/net/xdp.h

@@ -63,6 +63,19 @@ struct xdp_rxq_info {
 	struct xdp_mem_info mem;
 } ____cacheline_aligned; /* perf critical, avoid false-sharing */
 
+#ifdef CONFIG_XDP_LUA
+struct xdplua_create_work {
+	char				lua_script[8192];
+	struct lua_State	*L;
+	struct work_struct	work;
+	spinlock_t		lock;
+	bool			init;
+};
+
+DECLARE_PER_CPU(struct xdplua_create_work, luaworks);
+#endif /* CONFIG_XDP_LUA */
+
+
 struct xdp_buff {
 	void *data;
 	void *data_end;

include/uapi/linux/bpf.h

@@ -2240,9 +2240,10 @@ union bpf_attr {
 	FN(lua_pushmap),		\
 	FN(lua_pushskb),		\
 	FN(lua_pushstring),		\
-	FN(lua_setstate),		\
 	FN(lua_toboolean),		\
-	FN(lua_tointeger),
+	FN(lua_tointeger),		\
+	FN(lua_newpacket),		\
+	FN(lua_type),
 	/* #endif CONFIG_XDP_LUA */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper

net/core/dev.c

@@ -75,6 +75,7 @@
 #ifdef CONFIG_XDP_LUA
 #include <lua.h>
 #include <lauxlib.h>
+#include <lstate.h>
 #include <lualib.h>
 #include <luadata.h>
 #endif /* CONFIG_XDP_LUA */
@@ -164,6 +165,9 @@
 
 static DEFINE_SPINLOCK(ptype_lock);
 static DEFINE_SPINLOCK(offload_lock);
+#ifdef CONFIG_XDP_LUA
+DEFINE_PER_CPU(struct xdplua_create_work, luaworks);
+#endif
 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
 struct list_head ptype_all __read_mostly;	/* Taps */
 static struct list_head offload_base __read_mostly;
@@ -173,10 +177,6 @@ static int call_netdevice_notifiers_info(unsigned long val,
 					 struct netdev_notifier_info *info);
 static struct napi_struct *napi_by_id(unsigned int napi_id);
 
-#ifdef CONFIG_XDP_LUA
-struct list_head lua_state_cpu_list;
-#endif /* CONFIG_XDP_LUA */
-
 /*
  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
  * semaphore.
@@ -4310,6 +4310,9 @@ static u32 netif_receive_generic_xdp(struct sk_buff *skb,
 	bool orig_bcast;
 	int hlen, off;
 	u32 mac_len;
+#ifdef CONFIG_XDP_LUA
+	struct xdplua_create_work *lw;
+#endif /* CONFIG_XDP_LUA */
 
 	/* Reinjected packets coming from act_mirred or similar should
 	 * not get XDP generic processing.
@@ -4355,11 +4358,21 @@ static u32 netif_receive_generic_xdp(struct sk_buff *skb,
 	rxqueue = netif_get_rxqueue(skb);
 	xdp->rxq = &rxqueue->xdp_rxq;
 #ifdef CONFIG_XDP_LUA
+	lw = this_cpu_ptr(&luaworks);
+
 	xdp->skb = skb;
+	xdp->L = lw->L;
 #endif /* CONFIG_XDP_LUA */
 
 	act = bpf_prog_run_xdp(xdp_prog, xdp);
 
+#ifdef CONFIG_XDP_LUA
+	if (lw->init) {
+		lw->init = false;
+		spin_unlock(&lw->lock);
+	}
+#endif /* CONFIG_XDP_LUA */
+
 	/* check if bpf_xdp_adjust_head was used */
 	off = xdp->data - orig_data;
 	if (off) {
@@ -5146,16 +5159,29 @@ static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
 }
 
 #ifdef CONFIG_XDP_LUA
-int generic_xdp_lua_install_prog(char *lua_prog)
+
+static void per_cpu_xdp_lua_install(struct work_struct *w) {
+	int this_cpu = smp_processor_id();
+	struct xdplua_create_work *lw =
+		container_of(w, struct xdplua_create_work, work);
+
+	spin_lock_bh(&lw->lock);
+	if (luaL_dostring(lw->L, lw->lua_script)) {
+		pr_err(KERN_INFO "error: %s\nOn cpu: %d\n",
+			lua_tostring(lw->L, -1), this_cpu);
+	}
+	spin_unlock_bh(&lw->lock);
+}
+
+int generic_xdp_lua_install_prog(char *lua_script)
 {
-	struct lua_state_cpu *sc;
+	int cpu;
+	struct xdplua_create_work *lw;
 
-	list_for_each_entry(sc, &lua_state_cpu_list, list) {
-		if (luaL_dostring(sc->L, lua_prog)) {
-			pr_err(KERN_INFO "error: %s\nOn cpu: %d\n",
-				lua_tostring(sc->L, -1), sc->cpu);
-			return -EINVAL;
-		}
+	for_each_possible_cpu(cpu) {
+		lw = per_cpu_ptr(&luaworks, cpu);
+		strcpy(lw->lua_script, lua_script);
+		schedule_work_on(cpu, &lw->work);
 	}
 	return 0;
 }
@@ -9831,9 +9857,6 @@ static struct pernet_operations __net_initdata default_device_ops = {
 static int __init net_dev_init(void)
 {
 	int i, rc = -ENOMEM;
-#ifdef CONFIG_XDP_LUA
-	struct lua_state_cpu *new_state_cpu;
-#endif /* CONFIG_XDP_LUA */
 
 	BUG_ON(!dev_boot_phase);
 
@@ -9848,9 +9871,6 @@ static int __init net_dev_init(void)
 		INIT_LIST_HEAD(&ptype_base[i]);
 
 	INIT_LIST_HEAD(&offload_base);
-#ifdef CONFIG_XDP_LUA
-	INIT_LIST_HEAD(&lua_state_cpu_list);
-#endif /* CONFIG_XDP_LUA */
 
 	if (register_pernet_subsys(&netdev_net_ops))
 		goto out;
@@ -9862,6 +9882,9 @@ static int __init net_dev_init(void)
 	for_each_possible_cpu(i) {
 		struct work_struct *flush = per_cpu_ptr(&flush_works, i);
 		struct softnet_data *sd = &per_cpu(softnet_data, i);
+#ifdef CONFIG_XDP_LUA
+		struct xdplua_create_work *lw = per_cpu_ptr(&luaworks, i);
+#endif
 
 		INIT_WORK(flush, flush_backlog);
 
@@ -9881,24 +9904,18 @@ static int __init net_dev_init(void)
 		init_gro_hash(&sd->backlog);
 		sd->backlog.poll = process_backlog;
 		sd->backlog.weight = weight_p;
-
 #ifdef CONFIG_XDP_LUA
-		new_state_cpu = (struct lua_state_cpu *)
-			kmalloc(sizeof(struct lua_state_cpu), GFP_ATOMIC);
-		if (!new_state_cpu)
-			continue;
+		lw->L = luaL_newstate();
+		WARN_ON(!lw->L);
 
-		new_state_cpu->L = luaL_newstate();
-		if  (!new_state_cpu->L) {
-			kfree(new_state_cpu);
+		if (!lw->L)
 			continue;
-		}
 
-		luaL_openlibs(new_state_cpu->L);
-		lua_pop(new_state_cpu->L, 1);
-		new_state_cpu->cpu = i;
+		luaL_openlibs(lw->L);
+		luaL_requiref(lw->L, "data", luaopen_data, 1);
+		lua_pop(lw->L, 1);
 
-		list_add(&new_state_cpu->list, &lua_state_cpu_list);
+		INIT_WORK(&lw->work, per_cpu_xdp_lua_install);
 #endif /* CONFIG_XDP_LUA */
 	}
 

net/core/filter.c

@@ -4797,10 +4797,21 @@ static const struct bpf_func_proto bpf_lwt_seg6_adjust_srh_proto = {
 #endif /* CONFIG_IPV6_SEG6_BPF */
 
 #ifdef CONFIG_XDP_LUA
+static inline void verify_and_lock(void) {
+	struct xdplua_create_work *lw;
+
+	lw = this_cpu_ptr(&luaworks);
+	if (!lw->init) {
+		lw->init = true;
+		spin_lock(&lw->lock);
+	}
+}
+
 BPF_CALL_2(bpf_lua_dataref, struct xdp_buff *, ctx, int, offset) {
 	if (offset + ctx->data < ctx->data_end) {
 		int data_ref;
 
+		verify_and_lock();
 		data_ref = ldata_newref(ctx->L, ctx->data + offset,
 				ctx->data_end - ctx->data - offset);
 		return data_ref;
@@ -4819,6 +4830,7 @@ static const struct bpf_func_proto bpf_lua_dataref_proto = {
 };
 
 BPF_CALL_2(bpf_lua_dataunref, struct xdp_buff *, ctx, int, data_ref) {
+	verify_and_lock();
 	ldata_unref(ctx->L, data_ref);
 	return 0;
 }
@@ -4833,19 +4845,28 @@ static const struct bpf_func_proto bpf_lua_dataunref_proto = {
 };
 
 BPF_CALL_4(bpf_lua_pcall, struct xdp_buff *, ctx, char *, funcname,
-			int, num_args, int, num_rets) {
+		int, num_args, int, num_rets) {
+	int base;
+
+	verify_and_lock();
+
+	base = lua_gettop(ctx->L) - num_args;
 	if (lua_getglobal(ctx->L, funcname) != LUA_TFUNCTION) {
 		pr_err("function %s not found\n", funcname);
-		lua_pop(ctx->L, num_args);
-		return 0;
+		num_rets = 0;
+		goto clean_state;
 	}
 
 	lua_insert(ctx->L, 1);
 	if (lua_pcall(ctx->L, num_args, num_rets, 0)) {
 		pr_err("%s\n", lua_tostring(ctx->L, -1));
-		lua_pop(ctx->L, 1);
-		return 0;
+		num_rets = 0;
+		goto clean_state;
 	}
+
+clean_state:
+	base += num_rets;
+	lua_settop(ctx->L, base);
 	return num_rets;
 }
 
@@ -4861,6 +4882,7 @@ static const struct bpf_func_proto bpf_lua_pcall_proto = {
 };
 
 BPF_CALL_2(bpf_lua_pop, struct xdp_buff *, ctx, int, index) {
+	verify_and_lock();
 	lua_pop(ctx->L, index);
 	return 0;
 }
@@ -4875,6 +4897,7 @@ static const struct bpf_func_proto bpf_lua_pop_proto = {
 };
 
 BPF_CALL_2(bpf_lua_pushinteger, struct xdp_buff *, ctx, int, num) {
+	verify_and_lock();
 	lua_pushinteger(ctx->L, num);
 	return 0;
 }
@@ -4889,6 +4912,7 @@ static const struct bpf_func_proto bpf_lua_pushinteger_proto = {
 };
 
 BPF_CALL_2(bpf_lua_pushlightuserdata, struct xdp_buff *, ctx, void *, ptr) {
+	verify_and_lock();
 	lua_pushlightuserdata(ctx->L, ptr);
 	return 0;
 }
@@ -4903,6 +4927,7 @@ static const struct bpf_func_proto bpf_lua_pushlightuserdata_proto = {
 };
 
 BPF_CALL_3(bpf_lua_pushlstring, struct xdp_buff *, ctx, const char *, str, size_t, len) {
+	verify_and_lock();
 	lua_pushlstring(ctx->L, str, len);
 	return 0;
 }
@@ -4918,6 +4943,7 @@ static const struct bpf_func_proto bpf_lua_pushlstring_proto = {
 };
 
 BPF_CALL_2(bpf_lua_pushmap, struct xdp_buff *, ctx, struct bpf_map *, map) {
+	verify_and_lock();
 	lua_pushlightuserdata(ctx->L, map);
 	return 0;
 }
@@ -4932,6 +4958,7 @@ static const struct bpf_func_proto bpf_lua_pushmap_proto = {
 };
 
 BPF_CALL_1(bpf_lua_pushskb, struct xdp_buff *, ctx) {
+	verify_and_lock();
 	lua_pushlightuserdata(ctx->L, ctx->skb);
 	return 0;
 }
@@ -4945,6 +4972,7 @@ static const struct bpf_func_proto bpf_lua_pushskb_proto = {
 };
 
 BPF_CALL_2(bpf_lua_pushstring, struct xdp_buff *, ctx, const char *, str) {
+	verify_and_lock();
 	lua_pushstring(ctx->L, str);
 	return 0;
 }
@@ -4958,28 +4986,8 @@ static const struct bpf_func_proto bpf_lua_pushstring_proto = {
 	.arg2_type	= ARG_ANYTHING,
 };
 
-BPF_CALL_1(bpf_lua_setstate, struct xdp_buff *, ctx){
-	struct lua_state_cpu *sc;
-	int cpu = smp_processor_id();
-
-	list_for_each_entry(sc, &lua_state_cpu_list, list) {
-		if (sc->cpu == cpu) {
-			ctx->L = sc->L;
-			break;
-		}
-	}
-	return 0;
-}
-
-static const struct bpf_func_proto bpf_lua_setstate_proto = {
-	.func		= bpf_lua_setstate,
-	.gpl_only	= false,
-	.pkt_access	= false,
-	.ret_type	= RET_VOID,
-	.arg1_type	= ARG_PTR_TO_CTX,
-};
-
 BPF_CALL_2(bpf_lua_toboolean, struct xdp_buff *, ctx, int, index) {
+	verify_and_lock();
 	return lua_toboolean(ctx->L, index);
 }
 
@@ -4993,6 +5001,7 @@ static const struct bpf_func_proto bpf_lua_toboolean_proto = {
 };
 
 BPF_CALL_2(bpf_lua_tointeger, struct xdp_buff *, ctx, int, index) {
+	verify_and_lock();
 	return lua_tointeger(ctx->L, index);
 }
 
@@ -5082,8 +5091,6 @@ bpf_base_func_proto(enum bpf_func_id func_id)
 		return &bpf_lua_pushskb_proto;
 	case BPF_FUNC_lua_pushstring:
 		return &bpf_lua_pushstring_proto;
-	case BPF_FUNC_lua_setstate:
-		return &bpf_lua_setstate_proto;
 	case BPF_FUNC_lua_toboolean:
 		return &bpf_lua_toboolean_proto;
 	case BPF_FUNC_lua_tointeger:

samples/bpf/xdplua_map_kern.c

@@ -27,8 +27,6 @@ int xdp_lua_test_map_prog(struct xdp_md *ctx)
 	char lookupname[] = "lookup";
 	char updatename[] = "update";
 
-	bpf_lua_setstate(ctx);
-
 	bpf_lua_pushmap(ctx, &test_map);
 	bpf_lua_pcall(ctx, updatename, 1, 0);
 

tools/include/uapi/linux/bpf.h

@@ -2238,7 +2238,6 @@ union bpf_attr {
 	FN(lua_pushmap),		\
 	FN(lua_pushskb),		\
 	FN(lua_pushstring),		\
-	FN(lua_setstate),		\
 	FN(lua_toboolean),		\
 	FN(lua_tointeger),
 	/* #endif CONFIG_XDP_LUA */