Bluetooth: Mesh: Introduce separate workq for ADV EXT

this PR is to make the host always send packets.

Signed-off-by: Lingao Meng <[email protected]>

Author: LingaoM

doc/releases/release-notes-4.0.rst

@@ -132,6 +132,11 @@ Bluetooth
 
 * HCI Drivers
 
+* Mesh
+
+  * Introduced a mesh-specific workqueue to increase reliability of the mesh messages
+    transmission. To get the old behavior enable :kconfig:option:`CONFIG_BT_MESH_WORKQ_SYS`.
+
 Boards & SoC Support
 ********************
 

subsys/bluetooth/mesh/Kconfig

@@ -103,6 +103,64 @@ menuconfig BT_MESH_ADV_EXT
 
 if BT_MESH_ADV_EXT
 
+choice BT_MESH_WORKQ_CONTEXT
+	prompt "Advertising thread selection"
+	default BT_MESH_WORKQ_MESH
+	help
+	  Defines a context for mesh messages transmission.
+
+config BT_MESH_WORKQ_MESH
+	bool "Mesh-specific workqueue"
+	help
+	  When this option is selected, the mesh sends messages from the
+	  mesh-specific workqueue. This will ensure that messages are always sent.
+	  The application needs to ensure the mesh-specific workqueue size is large
+	  enough. Refer to BT_MESH_ADV_STACK_SIZE for the recommended minimum.
+
+config BT_MESH_WORKQ_SYS
+	bool "System workqueue"
+	help
+	  When this option is selected, the mesh sends messages from
+	  the system work queue. The application needs to ensure the system
+	  workqueue stack size (SYSTEM_WORKQUEUE_STACK_SIZE) is large enough,
+	  refer to BT_MESH_ADV_STACK_SIZE for the recommended minimum.
+
+	  When this option is enabled and the mesh tries to send a message,
+	  and the host ran out the HCI command buffers controlled by
+	  CONFIG_BT_BUF_CMD_TX_COUNT, the host returns -ENOBUFS immediately
+	  and the mesh drops the message transmission. To mitigate this
+	  issue, make sure to have sufficient number of HCI command buffers.
+	  When this option is enabled, the latency of sending mesh messages
+	  will be affected by other users on the system work queue, resulting in
+	  reduced reliability for sending mesh messages.
+
+endchoice
+
+if BT_MESH_WORKQ_MESH
+
+config BT_MESH_ADV_STACK_SIZE
+	int "Mesh extended advertiser thread stack size"
+	default 1536 if BT_MESH_PROXY
+	default 1024 if BT_HOST_CRYPTO
+	default 776 if BT_MESH_PRIV_BEACONS
+	default 768
+	help
+	  Size of bt mesh adv thread stack.
+
+	  NOTE: This is an advanced setting and should not be changed unless
+	  absolutely necessary
+
+config BT_MESH_ADV_PRIO
+	int "Mesh advertiser thread priority"
+	default 7
+	help
+	  Priority of bt mesh adv thread.
+
+	  NOTE: This is an advanced setting and should not be changed unless
+	  absolutely necessary
+
+endif # BT_MESH_WORKQ_MESH
+
 config BT_MESH_RELAY_ADV_SETS
 	int "Maximum of simultaneous relay message support"
 	default 0

subsys/bluetooth/mesh/adv.h

@@ -123,4 +123,6 @@ int bt_mesh_scan_active_set(bool active);
 int bt_mesh_adv_bt_data_send(uint8_t num_events, uint16_t adv_interval,
 			     const struct bt_data *ad, size_t ad_len);
 
+int bt_mesh_wq_submit(struct k_work *work);
+
 #endif /* ZEPHYR_SUBSYS_BLUETOOTH_MESH_ADV_H_ */

subsys/bluetooth/mesh/adv_ext.c

@@ -33,6 +33,14 @@ LOG_MODULE_REGISTER(bt_mesh_adv_ext);
 #define CONFIG_BT_MESH_RELAY_ADV_SETS 0
 #endif
 
+#ifdef CONFIG_BT_MESH_ADV_STACK_SIZE
+#define MESH_WORKQ_PRIORITY   CONFIG_BT_MESH_ADV_PRIO
+#define MESH_WORKQ_STACK_SIZE CONFIG_BT_MESH_ADV_STACK_SIZE
+#else
+#define MESH_WORKQ_PRIORITY   0
+#define MESH_WORKQ_STACK_SIZE 0
+#endif
+
 enum {
 	/** Controller is currently advertising */
 	ADV_FLAG_ACTIVE,
@@ -69,6 +77,15 @@ struct bt_mesh_ext_adv {
 static void send_pending_adv(struct k_work *work);
 static bool schedule_send(struct bt_mesh_ext_adv *ext_adv);
 
+static struct k_work_q bt_mesh_workq;
+static K_KERNEL_STACK_DEFINE(thread_stack, MESH_WORKQ_STACK_SIZE);
+
+#if defined(CONFIG_BT_MESH_WORKQ_MESH)
+#define MESH_WORKQ &bt_mesh_workq
+#else /* CONFIG_BT_MESH_WORKQ_SYS */
+#define MESH_WORKQ &k_sys_work_q
+#endif /* CONFIG_BT_MESH_WORKQ_MESH */
+
 static struct bt_mesh_ext_adv advs[] = {
 	[0] = {
 		.tags = (
@@ -258,7 +275,7 @@ static bool schedule_send_with_mask(struct bt_mesh_ext_adv *ext_adv, int ignore_
 	}
 
 	atomic_clear_bit(ext_adv->flags, ADV_FLAG_SCHEDULE_PENDING);
-	k_work_submit(&ext_adv->work);
+	bt_mesh_wq_submit(&ext_adv->work);
 
 	return true;
 }
@@ -407,7 +424,7 @@ int bt_mesh_adv_terminate(struct bt_mesh_adv *adv)
 
 		atomic_set_bit(ext_adv->flags, ADV_FLAG_SENT);
 
-		k_work_submit(&ext_adv->work);
+		bt_mesh_wq_submit(&ext_adv->work);
 
 		return 0;
 	}
@@ -429,6 +446,13 @@ void bt_mesh_adv_init(void)
 	for (int i = 0; i < ARRAY_SIZE(advs); i++) {
 		(void)memcpy(&advs[i].adv_param, &adv_param, sizeof(adv_param));
 	}
+
+	if (IS_ENABLED(CONFIG_BT_MESH_WORKQ_MESH)) {
+		k_work_queue_init(&bt_mesh_workq);
+		k_work_queue_start(&bt_mesh_workq, thread_stack, MESH_WORKQ_STACK_SIZE,
+				   K_PRIO_COOP(MESH_WORKQ_PRIORITY), NULL);
+		k_thread_name_set(&bt_mesh_workq.thread, "BT MESH WQ");
+	}
 }
 
 static struct bt_mesh_ext_adv *adv_instance_find(struct bt_le_ext_adv *instance)
@@ -458,7 +482,7 @@ static void adv_sent(struct bt_le_ext_adv *instance,
 
 	atomic_set_bit(ext_adv->flags, ADV_FLAG_SENT);
 
-	k_work_submit(&ext_adv->work);
+	bt_mesh_wq_submit(&ext_adv->work);
 }
 
 #if defined(CONFIG_BT_MESH_GATT_SERVER)
@@ -503,13 +527,13 @@ int bt_mesh_adv_enable(void)
 
 int bt_mesh_adv_disable(void)
 {
-	int err;
 	struct k_work_sync sync;
+	int err;
 
 	for (int i = 0; i < ARRAY_SIZE(advs); i++) {
 		atomic_set_bit(advs[i].flags, ADV_FLAG_SUSPENDING);
 
-		if (k_current_get() != &k_sys_work_q.thread ||
+		if (k_current_get() != k_work_queue_thread_get(MESH_WORKQ) ||
 		    (k_work_busy_get(&advs[i].work) & K_WORK_RUNNING) == 0) {
 			k_work_flush(&advs[i].work, &sync);
 		}
@@ -562,3 +586,8 @@ int bt_mesh_adv_bt_data_send(uint8_t num_events, uint16_t adv_interval,
 {
 	return bt_data_send(advs, num_events, adv_interval, ad, ad_len);
 }
+
+int bt_mesh_wq_submit(struct k_work *work)
+{
+	return k_work_submit_to_queue(MESH_WORKQ, work);
+}

subsys/bluetooth/mesh/adv_legacy.c

@@ -258,3 +258,8 @@ int bt_mesh_adv_gatt_start(const struct bt_le_adv_param *param, int32_t duration
 	adv_timeout = duration;
 	return bt_le_adv_start(param, ad, ad_len, sd, sd_len);
 }
+
+int bt_mesh_wq_submit(struct k_work *work)
+{
+	return k_work_submit(work);
+}

subsys/bluetooth/mesh/proxy_msg.c

@@ -64,6 +64,15 @@ static void proxy_sar_timeout(struct k_work *work)
 	LOG_WRN("Proxy SAR timeout");
 
 	role = CONTAINER_OF(dwork, struct bt_mesh_proxy_role, sar_timer);
+
+	while (!k_fifo_is_empty(&role->pending)) {
+		struct bt_mesh_adv *adv = k_fifo_get(&role->pending, K_NO_WAIT);
+
+		__ASSERT_NO_MSG(adv);
+
+		bt_mesh_adv_unref(adv);
+	}
+
 	if (role->conn) {
 		bt_conn_disconnect(role->conn,
 				   BT_HCI_ERR_REMOTE_USER_TERM_CONN);
@@ -200,7 +209,7 @@ static void buf_send_end(struct bt_conn *conn, void *user_data)
 	bt_mesh_adv_unref(adv);
 }
 
-int bt_mesh_proxy_relay_send(struct bt_conn *conn, struct bt_mesh_adv *adv)
+static int proxy_relay_send(struct bt_conn *conn, struct bt_mesh_adv *adv)
 {
 	int err;
 
@@ -230,6 +239,41 @@ int bt_mesh_proxy_relay_send(struct bt_conn *conn, struct bt_mesh_adv *adv)
 	return err;
 }
 
+int bt_mesh_proxy_relay_send(struct bt_conn *conn, struct bt_mesh_adv *adv)
+{
+	struct bt_mesh_proxy_role *role = &roles[bt_conn_index(conn)];
+
+	k_fifo_put(&role->pending, bt_mesh_adv_ref(adv));
+
+	bt_mesh_wq_submit(&role->work);
+
+	return 0;
+}
+
+static void proxy_msg_send_pending(struct k_work *work)
+{
+	struct bt_mesh_proxy_role *role;
+	struct k_work_delayable *dwork = k_work_delayable_from_work(work);
+	struct bt_mesh_adv *adv;
+
+	role = CONTAINER_OF(dwork, struct bt_mesh_proxy_role, sar_timer);
+	if (!role->conn) {
+		return;
+	}
+
+	adv = k_fifo_get(&role->pending, K_NO_WAIT);
+	if (!adv) {
+		return;
+	}
+
+	(void)proxy_relay_send(role->conn, adv);
+	bt_mesh_adv_unref(adv);
+
+	if (!k_fifo_is_empty(&role->pending)) {
+		bt_mesh_wq_submit(&role->work);
+	}
+}
+
 static void proxy_msg_init(struct bt_mesh_proxy_role *role)
 {
 	/* Check if buf has been allocated, in this way, we no longer need
@@ -247,6 +291,9 @@ static void proxy_msg_init(struct bt_mesh_proxy_role *role)
 
 	net_buf_simple_reset(&role->buf);
 
+	k_fifo_init(&role->pending);
+	k_work_init(&role->work, proxy_msg_send_pending);
+
 	k_work_init_delayable(&role->sar_timer, proxy_sar_timeout);
 }
 

subsys/bluetooth/mesh/proxy_msg.h

@@ -37,6 +37,9 @@ struct bt_mesh_proxy_role {
 	struct bt_conn *conn;
 	uint8_t msg_type;
 
+	struct k_fifo pending;
+	struct k_work work;
+
 	struct {
 		proxy_send_cb_t send;
 		proxy_recv_cb_t recv;

subsys/bluetooth/mesh/proxy_srv.c

@@ -916,7 +916,7 @@ static ssize_t proxy_ccc_write(struct bt_conn *conn,
 	client = find_client(conn);
 	if (client->filter_type == NONE) {
 		client->filter_type = ACCEPT;
-		k_work_submit(&client->send_beacons);
+		bt_mesh_wq_submit(&client->send_beacons);
 	}
 
 	return sizeof(value);

tests/bsim/bluetooth/mesh/compile.sh

@@ -15,12 +15,15 @@ app=tests/bsim/bluetooth/mesh conf_overlay=overlay_pst.conf compile
 app=tests/bsim/bluetooth/mesh conf_overlay=overlay_gatt.conf compile
 app=tests/bsim/bluetooth/mesh conf_overlay=overlay_low_lat.conf compile
 app=tests/bsim/bluetooth/mesh conf_overlay=overlay_psa.conf compile
+app=tests/bsim/bluetooth/mesh conf_overlay=overlay_workq_sys.conf compile
 app=tests/bsim/bluetooth/mesh conf_overlay="overlay_pst.conf;overlay_psa.conf" compile
 app=tests/bsim/bluetooth/mesh conf_overlay="overlay_gatt.conf;overlay_psa.conf" compile
+app=tests/bsim/bluetooth/mesh conf_overlay="overlay_gatt.conf;overlay_workq_sys.conf" compile
 app=tests/bsim/bluetooth/mesh conf_overlay="overlay_low_lat.conf;overlay_psa.conf" compile
 app=tests/bsim/bluetooth/mesh conf_overlay="overlay_gatt.conf;overlay_low_lat.conf" compile
 app=tests/bsim/bluetooth/mesh conf_overlay="overlay_pst.conf;overlay_gatt.conf" compile
 app=tests/bsim/bluetooth/mesh \
   conf_overlay="overlay_pst.conf;overlay_gatt.conf;overlay_psa.conf" compile
-
+app=tests/bsim/bluetooth/mesh \
+  conf_overlay="overlay_pst.conf;overlay_gatt.conf;overlay_workq_sys.conf" compile
 wait_for_background_jobs

tests/bsim/bluetooth/mesh/overlay_workq_sys.conf

@@ -0,0 +1 @@
+CONFIG_BT_MESH_WORKQ_SYS=y