Fix deadlock in stdio.format calls in tostring metamethod (lune-org#288)

Author: dai

Cargo.lock

@@ -22,4 +22,5 @@ dunce = "1.0"
 once_cell = "1.17"
 path-clean = "1.0"
 pathdiff = "0.2"
+parking_lot = "0.12.3"
 semver = "1.0"

crates/lune-utils/Cargo.toml

@@ -1,11 +1,9 @@
-use std::{
-    collections::HashSet,
-    sync::{Arc, Mutex},
-};
+use std::{collections::HashSet, sync::Arc};
 
 use console::{colors_enabled as get_colors_enabled, set_colors_enabled};
 use mlua::prelude::*;
 use once_cell::sync::Lazy;
+use parking_lot::ReentrantMutex;
 
 mod basic;
 mod config;
@@ -20,15 +18,15 @@ pub use self::config::ValueFormatConfig;
 // NOTE: Since the setting for colors being enabled is global,
 // and these functions may be called in parallel, we use this global
 // lock to make sure that we don't mess up the colors for other threads.
-static COLORS_LOCK: Lazy<Arc<Mutex<()>>> = Lazy::new(|| Arc::new(Mutex::new(())));
+static COLORS_LOCK: Lazy<Arc<ReentrantMutex<()>>> = Lazy::new(|| Arc::new(ReentrantMutex::new(())));
 
 /**
     Formats a Lua value into a pretty string using the given config.
 */
 #[must_use]
 #[allow(clippy::missing_panics_doc)]
 pub fn pretty_format_value(value: &LuaValue, config: &ValueFormatConfig) -> String {
-    let _guard = COLORS_LOCK.lock().unwrap();
+    let _guard = COLORS_LOCK.lock();
 
     let were_colors_enabled = get_colors_enabled();
     set_colors_enabled(were_colors_enabled && config.colors_enabled);
@@ -48,7 +46,7 @@ pub fn pretty_format_value(value: &LuaValue, config: &ValueFormatConfig) -> Stri
 #[must_use]
 #[allow(clippy::missing_panics_doc)]
 pub fn pretty_format_multi_value(values: &LuaMultiValue, config: &ValueFormatConfig) -> String {
-    let _guard = COLORS_LOCK.lock().unwrap();
+    let _guard = COLORS_LOCK.lock();
 
     let were_colors_enabled = get_colors_enabled();
     set_colors_enabled(were_colors_enabled && config.colors_enabled);

crates/lune-utils/src/fmt/value/mod.rs

@@ -122,3 +122,23 @@ stdio.ewrite(typeof(errorMessage))
 assertContains("Should format errors similarly to userdata", stdio.format(errorMessage), "<LuaErr")
 assertContains("Should format errors with stack begins", stdio.format(errorMessage), "Stack Begin")
 assertContains("Should format errors with stack ends", stdio.format(errorMessage), "Stack End")
+
+-- Check that calling stdio.format in a __tostring metamethod by print doesn't cause a deadlock
+
+local inner = {}
+setmetatable(inner, {
+	__tostring = function()
+		return stdio.format(5)
+	end,
+})
+
+print(inner)
+
+local outer = {}
+setmetatable(outer, {
+	__tostring = function()
+		return stdio.format(inner)
+	end,
+})
+
+print(outer)