Clean up diagnosticserver socket on unexpected shutdown (#25976) (#26190)

* Clean up diagnosticserver socket/pipe on shutdown

* Refactor dbg transport pipe cleanup to be registered as signal handler from the vm side

* cleanup dead code

* Remove more dead code and fix windows builds

* Moving some ifdefs around

Author: sywhang

src/debug/ee/debugger.cpp

@@ -890,15 +890,6 @@ ShutdownTransport()
         g_pDbgTransport = NULL;
     }
 }
-
-void
-AbortTransport()
-{
-    if (g_pDbgTransport != NULL)
-    {
-        g_pDbgTransport->AbortConnection();
-    }
-}
 #endif // FEATURE_DBGIPC_TRANSPORT_VM
 
 
@@ -1895,6 +1886,16 @@ void NotifyDebuggerOfStartup()
 
 #endif // !FEATURE_PAL
 
+void Debugger::CleanupTransportSocket(void)
+{
+#if defined(FEATURE_PAL) && defined(FEATURE_DBGIPC_TRANSPORT_VM)
+    if (g_pDbgTransport != NULL)
+    {
+        g_pDbgTransport->AbortConnection();
+    }
+#endif // FEATURE_PAL && FEATURE_DBGIPC_TRANSPORT_VM
+}
+
 //---------------------------------------------------------------------------------------
 //
 // Initialize Left-Side debugger object
@@ -2047,9 +2048,6 @@ HRESULT Debugger::Startup(void)
             ShutdownTransport();
             ThrowHR(hr);
         }
-    #ifdef FEATURE_PAL
-        PAL_SetShutdownCallback(AbortTransport);
-    #endif // FEATURE_PAL
     #endif // FEATURE_DBGIPC_TRANSPORT_VM
 
         RaiseStartupNotification();

src/debug/ee/debugger.h

@@ -1852,6 +1852,8 @@ class Debugger : public DebugInterface
 
     HRESULT StartupPhase2(Thread * pThread);
 
+    void CleanupTransportSocket();
+
     void InitializeLazyDataIfNecessary();
 
     void LazyInit(); // will throw

src/vm/ceemain.cpp

@@ -596,6 +596,31 @@ do { \
 #define IfFailGoLog(EXPR) IfFailGotoLog(EXPR, ErrExit)
 #endif
 
+
+#ifndef CROSSGEN_COMPILE
+#ifdef FEATURE_PAL
+void EESocketCleanupHelper()
+{
+    CONTRACTL
+    {
+        GC_NOTRIGGER;
+        MODE_ANY;
+    } CONTRACTL_END;
+
+    // Close the debugger transport socket first
+    if (g_pDebugInterface != NULL)
+    {
+        g_pDebugInterface->CleanupTransportSocket();
+    }
+
+    // Close the diagnostic server socket.
+#ifdef FEATURE_PERFTRACING
+    DiagnosticServer::Shutdown();
+#endif // FEATURE_PERFTRACING
+}
+#endif // FEATURE_PAL
+#endif // CROSSGEN_COMPILE
+
 void EEStartupHelper(COINITIEE fFlags)
 {
     CONTRACTL
@@ -653,8 +678,13 @@ void EEStartupHelper(COINITIEE fFlags)
 #ifdef FEATURE_PERFTRACING
         // Initialize the event pipe.
         EventPipe::Initialize();
+
 #endif // FEATURE_PERFTRACING
 
+#ifdef FEATURE_PAL
+        PAL_SetShutdownCallback(EESocketCleanupHelper);
+#endif // FEATURE_PAL
+
 #ifdef FEATURE_GDBJIT
         // Initialize gdbjit
         NotifyGdb::Initialize();

src/vm/dbginterface.h

@@ -399,6 +399,9 @@ class DebugInterface
     virtual LONG FirstChanceSuspendHijackWorker(PCONTEXT pContext, PEXCEPTION_RECORD pExceptionRecord) = 0;
 #endif
 
+    // Helper method for cleaning up transport socket
+    virtual void CleanupTransportSocket(void) = 0;
+
 #endif // #ifndef DACCESS_COMPILE
 
 #ifdef DACCESS_COMPILE