nrf_rpc: rpc_utils: system health command

Add RPC command for getting system health information from
a remote device. Currently, the system health information
only returns information about selected hung threads.

Signed-off-by: Damian Krolik <[email protected]>

Author: Damian-Nordic

include/nrf_rpc/rpc_utils/system_health.h

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2025 Nordic Semiconductor ASA
+ *
+ * SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
+ */
+
+#ifndef NRF_RPC_SYSTEM_HEALTH_H_
+#define NRF_RPC_SYSTEM_HEALTH_H_
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @addtogroup nrf_rpc_utils nRF RPC utility commands
+ * @{
+ * @defgroup nrf_rpc_system_health nRF RPC system health commands
+ * @{
+ */
+
+/** @brief System health information. */
+struct nrf_rpc_system_health {
+	/** @brief Bitmask of hung threads.
+	 *
+	 * Each bit represents a thread.
+	 * If a thread is hung, the corresponding bit is set.
+	 */
+	uint32_t hung_threads;
+};
+
+/** @brief Get system health information from the remote server.
+ *
+ * @param[out] out Pointer to the system health information.
+ */
+void nrf_rpc_system_health_get(struct nrf_rpc_system_health *out);
+
+/**
+ * @}
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* NRF_RPC_REMOTE_SHELL_H_ */

samples/nrf_rpc/protocols_serialization/client/prj.conf

@@ -28,3 +28,4 @@ CONFIG_NRF_RPC_UTILS=y
 CONFIG_NRF_RPC_UTILS_CLIENT=y
 CONFIG_NRF_RPC_UTILS_DEV_INFO=y
 CONFIG_NRF_RPC_UTILS_REMOTE_SHELL=y
+CONFIG_NRF_RPC_UTILS_SYSTEM_HEALTH=y

samples/nrf_rpc/protocols_serialization/client/src/rpc_utils_shell.c

@@ -8,6 +8,7 @@
 #include <nrf_rpc/rpc_utils/crash_gen.h>
 #include <nrf_rpc/rpc_utils/dev_info.h>
 #include <nrf_rpc/rpc_utils/remote_shell.h>
+#include <nrf_rpc/rpc_utils/system_health.h>
 
 #if defined(CONFIG_NRF_RPC_UTILS_DEV_INFO)
 static int remote_version_cmd(const struct shell *sh, size_t argc, char *argv[])
@@ -104,6 +105,19 @@ static int cmd_stack_overflow(const struct shell *sh, size_t argc, char *argv[])
 }
 #endif /* CONFIG_NRF_RPC_UTILS_CRASH_GEN */
 
+#if defined(CONFIG_NRF_RPC_UTILS_SYSTEM_HEALTH)
+static int cmd_system_health(const struct shell *sh, size_t argc, char *argv[])
+{
+	struct nrf_rpc_system_health health;
+
+	nrf_rpc_system_health_get(&health);
+
+	shell_print(sh, "Hung threads: %u", health.hung_threads);
+
+	return 0;
+}
+#endif /* CONFIG_NRF_RPC_UTILS_SYSTEM_HEALTH */
+
 SHELL_STATIC_SUBCMD_SET_CREATE(
 	util_cmds,
 #if defined(CONFIG_NRF_RPC_UTILS_DEV_INFO)
@@ -116,6 +130,9 @@ SHELL_STATIC_SUBCMD_SET_CREATE(
 	SHELL_CMD_ARG(assert, NULL, "Invoke assert", cmd_assert, 1, 1),
 	SHELL_CMD_ARG(hard_fault, NULL, "Invoke hard fault", cmd_hard_fault, 1, 1),
 	SHELL_CMD_ARG(stack_overflow, NULL, "Invoke stack overflow", cmd_stack_overflow, 1, 1),
+#endif
+#if defined(CONFIG_NRF_RPC_UTILS_SYSTEM_HEALTH)
+	SHELL_CMD_ARG(system_health, NULL, "Get system health", cmd_system_health, 0, 0),
 #endif
 	SHELL_SUBCMD_SET_END);
 

samples/nrf_rpc/protocols_serialization/server/prj.conf

@@ -20,6 +20,7 @@ CONFIG_NRF_RPC_UTILS=y
 CONFIG_NRF_RPC_UTILS_SERVER=y
 CONFIG_NRF_RPC_UTILS_DEV_INFO=y
 CONFIG_NRF_RPC_UTILS_REMOTE_SHELL=y
+CONFIG_NRF_RPC_UTILS_SYSTEM_HEALTH=y
 
 CONFIG_SETTINGS=y
 CONFIG_FLASH=y

subsys/nrf_rpc/rpc_utils/Kconfig

@@ -44,7 +44,24 @@ config NRF_RPC_UTILS_REMOTE_SHELL
 config NRF_RPC_UTILS_CRASH_GEN
 	bool "Crash generator"
 	help
-	  Enables commands for generating a crash on the remote device.
+	  Enables nRF RPC commands for generating a crash on the remote device.
+
+config NRF_RPC_UTILS_SYSTEM_HEALTH
+	bool "System health"
+	help
+	  Enables nRF RPC commands for getting the system health information.
+
+if NRF_RPC_UTILS_SYSTEM_HEALTH && NRF_RPC_UTILS_SERVER
+
+config NRF_RPC_UTILS_WATCHDOG_PERIOD
+	int "Watchdog period (s)"
+	default 10
+	depends on NRF_RPC_UTILS_SYSTEM_HEALTH && NRF_RPC_UTILS_SERVER
+	help
+	  Configures how frequently the nRF RPC utils server kicks watched
+	  threads, and how quickly it concludes a thread is hung.
+
+endif # NRF_RPC_UTILS_SYSTEM_HEALTH && NRF_RPC_UTILS_SERVER
 
 config SHELL_BACKEND_DUMMY_BUF_SIZE
 	default 1024 if NRF_RPC_UTILS_REMOTE_SHELL && NRF_RPC_UTILS_SERVER

subsys/nrf_rpc/rpc_utils/client/CMakeLists.txt

@@ -9,6 +9,7 @@ zephyr_library()
 zephyr_library_sources_ifdef(CONFIG_NRF_RPC_UTILS_CRASH_GEN crash_gen.c)
 zephyr_library_sources_ifdef(CONFIG_NRF_RPC_UTILS_DEV_INFO dev_info.c)
 zephyr_library_sources_ifdef(CONFIG_NRF_RPC_UTILS_REMOTE_SHELL remote_shell.c)
+zephyr_library_sources_ifdef(CONFIG_NRF_RPC_UTILS_SYSTEM_HEALTH system_health.c)
 
 zephyr_library_include_directories(
   ${CMAKE_CURRENT_SOURCE_DIR}/../common

subsys/nrf_rpc/rpc_utils/client/system_health.c

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2025 Nordic Semiconductor ASA
+ *
+ * SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
+ */
+
+#include <nrf_rpc/nrf_rpc_serialize.h>
+#include <nrf_rpc/rpc_utils/system_health.h>
+#include <rpc_utils_group.h>
+
+#include <nrf_rpc_cbor.h>
+
+void nrf_rpc_system_health_get(struct nrf_rpc_system_health *out)
+{
+	struct nrf_rpc_cbor_ctx ctx;
+
+	NRF_RPC_CBOR_ALLOC(&rpc_utils_group, ctx, 0);
+	nrf_rpc_cbor_cmd_rsp_no_err(&rpc_utils_group, RPC_UTIL_SYSTEM_HEALTH_GET, &ctx);
+
+	out->hung_threads = nrf_rpc_decode_uint(&ctx);
+
+	if (!nrf_rpc_decoding_done_and_check(&rpc_utils_group, &ctx)) {
+		nrf_rpc_err(-EBADMSG, NRF_RPC_ERR_SRC_RECV, &rpc_utils_group,
+			    RPC_UTIL_SYSTEM_HEALTH_GET, NRF_RPC_PACKET_TYPE_RSP);
+	}
+}

subsys/nrf_rpc/rpc_utils/common/rpc_utils_group.h

@@ -19,6 +19,7 @@ enum rpc_utils_cmd_server {
 	RPC_UTIL_CRASH_GEN_ASSERT = 2,
 	RPC_UTIL_CRASH_GEN_HARD_FAULT = 3,
 	RPC_UTIL_CRASH_GEN_STACK_OVERFLOW = 4,
+	RPC_UTIL_SYSTEM_HEALTH_GET = 5,
 };
 
 #endif /* RPC_UTILS_GROUP_H_ */

subsys/nrf_rpc/rpc_utils/server/CMakeLists.txt

@@ -9,6 +9,7 @@ zephyr_library()
 zephyr_library_sources_ifdef(CONFIG_NRF_RPC_UTILS_CRASH_GEN crash_gen.c)
 zephyr_library_sources_ifdef(CONFIG_NRF_RPC_UTILS_DEV_INFO dev_info.c)
 zephyr_library_sources_ifdef(CONFIG_NRF_RPC_UTILS_REMOTE_SHELL remote_shell.c)
+zephyr_library_sources_ifdef(CONFIG_NRF_RPC_UTILS_SYSTEM_HEALTH system_health.c)
 
 zephyr_library_include_directories(
   ${CMAKE_CURRENT_SOURCE_DIR}/../common

subsys/nrf_rpc/rpc_utils/server/system_health.c

@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2025 Nordic Semiconductor ASA
+ *
+ * SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
+ */
+
+#include <nrf_rpc/nrf_rpc_serialize.h>
+#include <rpc_utils_group.h>
+
+#include <nrf_rpc_cbor.h>
+
+#include <zephyr/kernel.h>
+#include <zephyr/sys/atomic.h>
+#include <zephyr/sys/util.h>
+
+#ifdef CONFIG_OPENTHREAD
+#include <zephyr/net/openthread.h>
+#endif
+
+/*
+ * Pointer to a function that "kicks" a watched thread.
+ *
+ * Currently, only work queue threads can be watched.
+ * "Kicking" means submitting a task to the thread, which marks the thread as alive (not hung).
+ */
+typedef void (*watched_thread_kick_fn)(struct k_work *work);
+
+static void kick_system_workq(struct k_work *work)
+{
+	(void)k_work_submit(work);
+}
+
+#ifdef CONFIG_OPENTHREAD
+static void kick_openthread(struct k_work *work)
+{
+	k_tid_t tid = openthread_thread_id_get();
+	struct k_work_q *queue = CONTAINER_OF(tid, struct k_work_q, thread);
+
+	(void)k_work_submit_to_queue(queue, work);
+}
+#endif
+
+#ifdef CONFIG_BT_LONG_WQ
+extern int bt_long_wq_submit(struct k_work *work);
+
+static void kick_bluetooth_long(struct k_work *work)
+{
+	(void)bt_long_wq_submit(work);
+}
+#endif
+
+const static watched_thread_kick_fn watched_thread_kick[] = {
+	kick_system_workq,
+#ifdef CONFIG_OPENTHREAD
+	kick_openthread,
+#endif
+#ifdef CONFIG_BT_LONG_WQ
+	kick_bluetooth_long,
+#endif
+};
+
+enum {
+	NUM_WATCHED_THREADS = ARRAY_SIZE(watched_thread_kick),
+};
+
+static atomic_t hung_threads;
+static atomic_t hung_thread_candidates;
+static void watchdog_timer_handler(struct k_timer *timer);
+static K_TIMER_DEFINE(watchdog_timer, watchdog_timer_handler, NULL);
+static struct k_work watched_thread_work[NUM_WATCHED_THREADS];
+
+/*
+ * Function called by a watched thread to prove it's not hung.
+ */
+static void watchdog_feed(struct k_work *work)
+{
+	const size_t index = ARRAY_INDEX(watched_thread_work, work);
+
+	atomic_clear_bit(&hung_thread_candidates, index);
+}
+
+static void watchdog_timer_handler(struct k_timer *timer)
+{
+	const atomic_val_t ALL_THREADS = GENMASK(NUM_WATCHED_THREADS - 1, 0);
+
+	atomic_set(&hung_threads, atomic_set(&hung_thread_candidates, ALL_THREADS));
+
+	for (size_t i = 0; i < NUM_WATCHED_THREADS; i++) {
+		watched_thread_kick[i](&watched_thread_work[i]);
+	}
+}
+
+static int watchdog_init(void)
+{
+	for (size_t i = 0; i < NUM_WATCHED_THREADS; i++) {
+		k_work_init(&watched_thread_work[i], watchdog_feed);
+	}
+
+	k_timer_start(&watchdog_timer, K_NO_WAIT, K_SECONDS(CONFIG_NRF_RPC_UTILS_WATCHDOG_PERIOD));
+
+	return 0;
+}
+
+SYS_INIT(watchdog_init, APPLICATION, CONFIG_APPLICATION_INIT_PRIORITY);
+
+static void system_health_get_handler(const struct nrf_rpc_group *group,
+				      struct nrf_rpc_cbor_ctx *ctx, void *handler_data)
+{
+	nrf_rpc_cbor_decoding_done(group, ctx);
+	nrf_rpc_rsp_send_uint(group, atomic_get(&hung_threads));
+}
+
+NRF_RPC_CBOR_CMD_DECODER(rpc_utils_group, system_health_get, RPC_UTIL_SYSTEM_HEALTH_GET,
+			 system_health_get_handler, NULL);