Detect active mono debugger automatically; remove debug_start_server

Author: ghorsington

README.md

@@ -157,7 +157,6 @@ All Doorstop arguments start with `--doorstop-` and always contain an argument.
 | `--doorstop-target-assembly string`               | Path to the assembly to load and execute.                                                            |
 | `--doorstop-mono-dll-search-path-override string` | Overrides default Mono DLL search path                                                               |
 | `--doorstop-mono-debug-enabled bool`              | If true, Mono debugger server will be enabled                                                        |
-| `--doorstop-mono-debug-start-server bool`         | Whether Doorstop should initialize the debugger server automatically                                 |
 | `--doorstop-mono-debug-suspend bool`              | Whether to suspend the game execution until the debugger is attached.                                |
 | `--doorstop-mono-debug-address string`            | The address to use for the Mono debugger server.                                                     |
 | `--doorstop-clr-corlib-dir string`                | Path to coreclr library that contains the CoreCLR runtime                                            |

assets/nix/run.sh

@@ -41,10 +41,6 @@ dll_search_path_override=""
 # If 1, Mono debugger server will be enabled
 debug_enable="0"
 
-# When debug_enabled is 1, this option specifies whether Doorstop should initialize the debugger server
-# If you experience crashes when starting the debugger on debug UnityPlayer builds, try setting this to 0
-debug_start_server="0"
-
 # When debug_enabled is 1, specifies the address to use for the debugger server
 debug_address="127.0.0.1:10000"
 
@@ -244,7 +240,6 @@ export DOORSTOP_TARGET_ASSEMBLY="$target_assembly"
 export DOORSTOP_IGNORE_DISABLED_ENV="$ignore_disable_switch"
 export DOORSTOP_MONO_DLL_SEARCH_PATH_OVERRIDE="$dll_search_path_override"
 export DOORSTOP_MONO_DEBUG_ENABLED="$debug_enable"
-export DOORSTOP_MONO_DEBUG_START_SERVER="$debug_start_server"
 export DOORSTOP_MONO_DEBUG_ADDRESS="$debug_address"
 export DOORSTOP_MONO_DEBUG_SUSPEND="$debug_suspend"
 export DOORSTOP_CLR_RUNTIME_CORECLR_PATH="$coreclr_path.$lib_extension"

assets/windows/doorstop_config.ini

@@ -29,10 +29,6 @@ dll_search_path_override=
 # If true, Mono debugger server will be enabled
 debug_enabled=false
 
-# When debug_enabled is true, this option specifies whether Doorstop should initialize the debugger server
-# If you experience crashes when starting the debugger on debug UnityPlayer builds, try setting this to false
-debug_start_server=true
-
 # When debug_enabled is true, specifies the address to use for the debugger server
 debug_address=127.0.0.1:10000
 

src/bootstrap.c

@@ -8,6 +8,8 @@
 #include "util/paths.h"
 #include "util/util.h"
 
+bool_t mono_debug_init_called = FALSE;
+
 void mono_doorstop_bootstrap(void *mono_domain) {
     if (getenv(TEXT("DOORSTOP_INITIALIZED"))) {
         LOG("DOORSTOP_INITIALIZED is set! Skipping!");
@@ -171,7 +173,13 @@ void *init_mono(const char *root_domain_name, const char *runtime_version) {
 
     void *domain = mono.jit_init_version(root_domain_name, runtime_version);
 
-    if (config.mono_debug_enabled && config.mono_debug_start_server) {
+    bool_t debugger_already_enabled = mono_debug_init_called;
+    if (mono.debug_enabled) {
+        debugger_already_enabled |= mono.debug_enabled();
+    }
+
+    if (config.mono_debug_enabled && !debugger_already_enabled) {
+        LOG("Detected mono debugger is not initialized; initialized it");
         mono.debug_init(MONO_DEBUG_FORMAT_MONO);
         mono.debug_domain_create(domain);
     }
@@ -350,4 +358,9 @@ void *hook_mono_image_open_from_data_with_name(void *data,
                                                      status, refonly, name);
     }
     return result;
+}
+
+void hook_mono_debug_init(MonoDebugFormat format) {
+    mono_debug_init_called = TRUE;
+    mono.debug_init(format);
 }

src/bootstrap.h

@@ -13,5 +13,6 @@ void *hook_mono_image_open_from_data_with_name(void *data,
                                                MonoImageOpenStatus *status,
                                                int refonly, const char *name);
 void hook_mono_jit_parse_options(int argc, char **argv);
+void hook_mono_debug_init(MonoDebugFormat format);
 
 #endif

src/config/common.c

@@ -24,7 +24,6 @@ void init_config_defaults() {
     config.ignore_disabled_env = FALSE;
     config.redirect_output_log = FALSE;
     config.mono_debug_enabled = FALSE;
-    config.mono_debug_start_server = TRUE;
     config.mono_debug_suspend = FALSE;
     config.mono_debug_address = NULL;
     config.target_assembly = NULL;

src/config/config.h

@@ -46,14 +46,6 @@ typedef struct {
      */
     bool_t mono_debug_enabled;
 
-    /**
-     * @brief Whether to manually intialize the mono debugger server.
-     *
-     * On debug Unity versions, Unity automatically initializes the debugger
-     * server. Use this option to disable the automatic server initialization.
-     */
-    bool_t mono_debug_start_server;
-
     /**
      * @brief Whether to enable the debugger in suspended state.
      *

src/nix/config.c

@@ -33,8 +33,6 @@ void load_config() {
     get_env_bool("DOORSTOP_REDIRECT_OUTPUT_LOG", &config.redirect_output_log);
     get_env_bool("DOORSTOP_IGNORE_DISABLED_ENV", &config.ignore_disabled_env);
     get_env_bool("DOORSTOP_MONO_DEBUG_ENABLED", &config.mono_debug_enabled);
-    get_env_bool("DOORSTOP_MONO_DEBUG_START_SERVER",
-                 &config.mono_debug_start_server);
     get_env_bool("DOORSTOP_MONO_DEBUG_SUSPEND", &config.mono_debug_suspend);
     try_get_env("DOORSTOP_MONO_DEBUG_ADDRESS", TEXT("127.0.0.1:10000"),
                 &config.mono_debug_address);

src/nix/entrypoint.c

@@ -38,6 +38,8 @@ void *dlsym_hook(void *handle, const char *name) {
                   capture_mono_path(res));
     REDIRECT_INIT("mono_jit_parse_options", load_mono_funcs,
                   hook_mono_jit_parse_options, capture_mono_path(res));
+    REDIRECT_INIT("mono_debug_init", load_mono_funcs, hook_mono_debug_init,
+                  capture_mono_path(res));
 
 #undef REDIRECT_INIT
     return res;

src/runtimes/mono.h

@@ -36,6 +36,7 @@ DEF_CALL(void *, assembly_load_from_full, void *image, const char *fname,
 DEF_CALL(void *, jit_parse_options, int argc, char **argv)
 DEF_CALL(void *, debug_init, MonoDebugFormat format)
 DEF_CALL(void *, debug_domain_create, void *domain)
+DEF_CALL(int, debug_enabled)
 #else
 
 #ifndef MONO_H