[nrf fromtree] shell: Make TX mutex timeout configurable

The 50 ms TX mutex timeouts introduced in commit b0a0feb cause
dropped messages when used with the shell log backend. This
commit makes time timeout configurable using the
CONFIG_SHELL_TX_TIMEOUT_MS Kconfig option and makes the
default timeout longer when shell log backend is enabled.

Link: #90215

Signed-off-by: Tommi Kangas <[email protected]>
(cherry picked from commit 941dcd3)

Author: tokangas

subsys/shell/Kconfig

@@ -326,6 +326,16 @@ config SHELL_CUSTOM_HEADER
 	  extension of the shell APIs at the macro level. Please use cautiously!
 	  The internal shell API may change in future releases.
 
+config SHELL_TX_TIMEOUT_MS
+	int "Shell TX timeout in milliseconds"
+	default 500 if SHELL_LOG_BACKEND
+	default 50
+	help
+	  The timeout in milliseconds for the shell TX mutex. This is used to
+	  prevent deadlocks when called from other than the shell's processing thread.
+	  When using the shell log backend, the timeout should be longer to avoid
+	  timeouts due to log processing. A value of 0 means no timeout.
+
 source "subsys/shell/modules/Kconfig"
 
 endif # SHELL

subsys/shell/shell.c

@@ -31,7 +31,11 @@
 	"WARNING: A print request was detected on not active shell backend.\n"
 #define SHELL_MSG_TOO_MANY_ARGS		"Too many arguments in the command.\n"
 #define SHELL_INIT_OPTION_PRINTER	(NULL)
-#define SHELL_TX_MTX_TIMEOUT_MS		50
+#if (CONFIG_SHELL_TX_TIMEOUT_MS == 0)
+#define SHELL_TX_MTX_TIMEOUT		K_FOREVER
+#else
+#define SHELL_TX_MTX_TIMEOUT		K_MSEC(CONFIG_SHELL_TX_TIMEOUT_MS)
+#endif
 
 #define SHELL_THREAD_PRIORITY \
 	COND_CODE_1(CONFIG_SHELL_THREAD_PRIORITY_OVERRIDE, \
@@ -1355,7 +1359,7 @@ void shell_thread(void *shell_handle, void *arg_log_backend,
 			     K_FOREVER);
 
 		if (err != 0) {
-			if (k_mutex_lock(&sh->ctx->wr_mtx, K_MSEC(SHELL_TX_MTX_TIMEOUT_MS)) != 0) {
+			if (k_mutex_lock(&sh->ctx->wr_mtx, SHELL_TX_MTX_TIMEOUT) != 0) {
 				return;
 			}
 			z_shell_fprintf(sh, SHELL_ERROR,
@@ -1448,7 +1452,7 @@ int shell_start(const struct shell *sh)
 		z_shell_log_backend_enable(sh->log_backend, (void *)sh, sh->ctx->log_level);
 	}
 
-	if (k_mutex_lock(&sh->ctx->wr_mtx, K_MSEC(SHELL_TX_MTX_TIMEOUT_MS)) != 0) {
+	if (k_mutex_lock(&sh->ctx->wr_mtx, SHELL_TX_MTX_TIMEOUT) != 0) {
 		return -EBUSY;
 	}
 
@@ -1552,7 +1556,7 @@ void shell_vfprintf(const struct shell *sh, enum shell_vt100_color color,
 		return;
 	}
 
-	if (k_mutex_lock(&sh->ctx->wr_mtx, K_MSEC(SHELL_TX_MTX_TIMEOUT_MS)) != 0) {
+	if (k_mutex_lock(&sh->ctx->wr_mtx, SHELL_TX_MTX_TIMEOUT) != 0) {
 		return;
 	}
 
@@ -1692,7 +1696,7 @@ int shell_prompt_change(const struct shell *sh, const char *prompt)
 
 	size_t prompt_length = z_shell_strlen(prompt);
 
-	if (k_mutex_lock(&sh->ctx->wr_mtx, K_MSEC(SHELL_TX_MTX_TIMEOUT_MS)) != 0) {
+	if (k_mutex_lock(&sh->ctx->wr_mtx, SHELL_TX_MTX_TIMEOUT) != 0) {
 		return -EBUSY;
 	}
 
@@ -1715,7 +1719,7 @@ int shell_prompt_change(const struct shell *sh, const char *prompt)
 
 void shell_help(const struct shell *sh)
 {
-	if (k_mutex_lock(&sh->ctx->wr_mtx, K_MSEC(SHELL_TX_MTX_TIMEOUT_MS)) != 0) {
+	if (k_mutex_lock(&sh->ctx->wr_mtx, SHELL_TX_MTX_TIMEOUT) != 0) {
 		return;
 	}
 	shell_internal_help_print(sh);
@@ -1750,7 +1754,7 @@ int shell_execute_cmd(const struct shell *sh, const char *cmd)
 	sh->ctx->cmd_buff_len = cmd_len;
 	sh->ctx->cmd_buff_pos = cmd_len;
 
-	if (k_mutex_lock(&sh->ctx->wr_mtx, K_MSEC(SHELL_TX_MTX_TIMEOUT_MS)) != 0) {
+	if (k_mutex_lock(&sh->ctx->wr_mtx, SHELL_TX_MTX_TIMEOUT) != 0) {
 		return -ENOEXEC;
 	}
 	ret_val = execute(sh);