Revert "[supervisor] revert recent changes to reduce frequency of supervisor run error with unexpected exit code (#20325)"

This reverts commit 3f066ce.

Author: mustard-mh

components/supervisor/cmd/init.go

@@ -77,25 +77,59 @@ var initCmd = &cobra.Command{
 		}
 
 		supervisorDone := make(chan struct{})
+		handledByReaper := make(chan int)
+		handleSupervisorExit := func(exitCode int) {
+			if exitCode == 0 {
+				return
+			}
+			logs := extractFailureFromRun()
+			if shared.IsExpectedShutdown(exitCode) {
+				log.Fatal(logs)
+			} else {
+				log.WithError(fmt.Errorf(logs)).Fatal("supervisor run error with unexpected exit code")
+			}
+		}
 		go func() {
 			defer close(supervisorDone)
 
 			err := runCommand.Wait()
-			if err != nil && !(strings.Contains(err.Error(), "signal: ") || strings.Contains(err.Error(), "no child processes")) {
+			if err == nil {
+				return
+			}
+			// exited by reaper
+			if strings.Contains(err.Error(), "no child processes") {
+				ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+				defer cancel()
+				select {
+				case <-ctx.Done(): // timeout
+				case exitCode := <-handledByReaper:
+					handleSupervisorExit(exitCode)
+				}
+			} else if !(strings.Contains(err.Error(), "signal: ")) {
 				if eerr, ok := err.(*exec.ExitError); ok && eerr.ExitCode() != 0 {
-					logs := extractFailureFromRun()
-					if shared.IsExpectedShutdown(eerr.ExitCode()) {
-						log.Fatal(logs)
-					} else {
-						log.WithError(fmt.Errorf(logs)).Fatal("supervisor run error with unexpected exit code")
-					}
+					handleSupervisorExit(eerr.ExitCode())
 				}
 				log.WithError(err).Error("supervisor run error")
 				return
 			}
 		}()
 		// start the reaper to clean up zombie processes
-		reaper.Reap()
+		reaperChan := make(chan reaper.Status, 10)
+		reaper.Start(reaper.Config{
+			Pid:              -1,
+			Options:          0,
+			DisablePid1Check: false,
+			StatusChannel:    reaperChan,
+		})
+		go func() {
+			for status := range reaperChan {
+				if status.Pid != runCommand.Process.Pid {
+					continue
+				}
+				exitCode := status.WaitStatus.ExitStatus()
+				handledByReaper <- exitCode
+			}
+		}()
 
 		select {
 		case <-supervisorDone:

components/supervisor/go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/gitpod-io/gitpod/ide-metrics-api v0.0.0-00010101000000-000000000000
 	github.com/gitpod-io/gitpod/supervisor/api v0.0.0-00010101000000-000000000000
 	github.com/gitpod-io/gitpod/ws-daemon/api v0.0.0-00010101000000-000000000000
+	github.com/gitpod-io/go-reaper v0.0.0-20241024192051-78d04cc2e25f
 	github.com/golang/mock v1.6.0
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
@@ -29,7 +30,7 @@ require (
 	github.com/prometheus/common v0.42.0
 	github.com/prometheus/procfs v0.10.1
 	github.com/prometheus/pushgateway v1.5.1
-	github.com/ramr/go-reaper v0.2.1
+	github.com/ramr/go-reaper v0.2.2
 	github.com/sirupsen/logrus v1.9.3
 	github.com/soheilhy/cmux v0.1.5
 	github.com/spf13/cobra v1.4.0