internal/gocommand: send SIGQUIT to hanging go commands on posix

Existing debug information has not proven useful for understanding
hanging go commands. On posix systems we can send SIGQUIT, and peek at
stderr, to try to see what the go command is doing.

Tested locally by setting the timeout to 10ms.

Updates golang/go#54461

Change-Id: I1bc38d6da82b0dffb55081364b0af8f20a3afcfc
Reviewed-on: https://go-review.googlesource.com/c/tools/+/643915
LUCI-TryBot-Result: Go LUCI <[email protected]>
Reviewed-by: Alan Donovan <[email protected]>
Auto-Submit: Robert Findley <[email protected]>

Author: findleyr

internal/gocommand/invoke.go

@@ -179,7 +179,7 @@ type Invocation struct {
 	CleanEnv   bool
 	Env        []string
 	WorkingDir string
-	Logf       func(format string, args ...interface{})
+	Logf       func(format string, args ...any)
 }
 
 // Postcondition: both error results have same nilness.
@@ -388,7 +388,9 @@ func runCmdContext(ctx context.Context, cmd *exec.Cmd) (err error) {
 		case err := <-resChan:
 			return err
 		case <-timer.C:
-			HandleHangingGoCommand(startTime, cmd)
+			// HandleHangingGoCommand terminates this process.
+			// Pass off resChan in case we can collect the command error.
+			handleHangingGoCommand(startTime, cmd, resChan)
 		case <-ctx.Done():
 		}
 	} else {
@@ -413,32 +415,32 @@ func runCmdContext(ctx context.Context, cmd *exec.Cmd) (err error) {
 	}
 
 	// Didn't shut down in response to interrupt. Kill it hard.
-	// TODO(rfindley): per advice from bcmills@, it may be better to send SIGQUIT
-	// on certain platforms, such as unix.
 	if err := cmd.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) && debug {
 		log.Printf("error killing the Go command: %v", err)
 	}
 
 	return <-resChan
 }
 
-func HandleHangingGoCommand(start time.Time, cmd *exec.Cmd) {
+// handleHangingGoCommand outputs debugging information to help diagnose the
+// cause of a hanging Go command, and then exits with log.Fatalf.
+func handleHangingGoCommand(start time.Time, cmd *exec.Cmd, resChan chan error) {
 	switch runtime.GOOS {
 	case "linux", "darwin", "freebsd", "netbsd":
 		fmt.Fprintln(os.Stderr, `DETECTED A HANGING GO COMMAND
 
-The gopls test runner has detected a hanging go command. In order to debug
-this, the output of ps and lsof/fstat is printed below.
+			The gopls test runner has detected a hanging go command. In order to debug
+			this, the output of ps and lsof/fstat is printed below.
 
-See golang/go#54461 for more details.`)
+			See golang/go#54461 for more details.`)
 
 		fmt.Fprintln(os.Stderr, "\nps axo ppid,pid,command:")
 		fmt.Fprintln(os.Stderr, "-------------------------")
 		psCmd := exec.Command("ps", "axo", "ppid,pid,command")
 		psCmd.Stdout = os.Stderr
 		psCmd.Stderr = os.Stderr
 		if err := psCmd.Run(); err != nil {
-			panic(fmt.Sprintf("running ps: %v", err))
+			log.Printf("Handling hanging Go command: running ps: %v", err)
 		}
 
 		listFiles := "lsof"
@@ -452,10 +454,24 @@ See golang/go#54461 for more details.`)
 		listFilesCmd.Stdout = os.Stderr
 		listFilesCmd.Stderr = os.Stderr
 		if err := listFilesCmd.Run(); err != nil {
-			panic(fmt.Sprintf("running %s: %v", listFiles, err))
+			log.Printf("Handling hanging Go command: running %s: %v", listFiles, err)
+		}
+		// Try to extract information about the slow go process by issuing a SIGQUIT.
+		if err := cmd.Process.Signal(sigStuckProcess); err == nil {
+			select {
+			case err := <-resChan:
+				stderr := "not a bytes.Buffer"
+				if buf, _ := cmd.Stderr.(*bytes.Buffer); buf != nil {
+					stderr = buf.String()
+				}
+				log.Printf("Quit hanging go command:\n\terr:%v\n\tstderr:\n%v\n\n", err, stderr)
+			case <-time.After(5 * time.Second):
+			}
+		} else {
+			log.Printf("Sending signal %d to hanging go command: %v", sigStuckProcess, err)
 		}
 	}
-	panic(fmt.Sprintf("detected hanging go command (golang/go#54461); waited %s\n\tcommand:%s\n\tpid:%d", time.Since(start), cmd, cmd.Process.Pid))
+	log.Fatalf("detected hanging go command (golang/go#54461); waited %s\n\tcommand:%s\n\tpid:%d", time.Since(start), cmd, cmd.Process.Pid)
 }
 
 func cmdDebugStr(cmd *exec.Cmd) string {

internal/gocommand/invoke_notunix.go

@@ -0,0 +1,13 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !unix
+
+package gocommand
+
+import "os"
+
+// sigStuckProcess is the signal to send to kill a hanging subprocess.
+// On Unix we send SIGQUIT, but on non-Unix we only have os.Kill.
+var sigStuckProcess = os.Kill

internal/gocommand/invoke_unix.go

@@ -0,0 +1,13 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build unix
+
+package gocommand
+
+import "syscall"
+
+// Sigstuckprocess is the signal to send to kill a hanging subprocess.
+// Send SIGQUIT to get a stack trace.
+var sigStuckProcess = syscall.SIGQUIT