Reapply "[supervisor] set pod failure reason when supervisor is reaped" (#20327)

* Revert "[supervisor] revert recent changes to reduce frequency of `supervisor run error with unexpected exit code` (#20325)"

This reverts commit 3f066ce.

* Ignore expected case: pod kill / process kill by supervisor

* Debug commit

* fix

* Revert "Debug commit"

This reverts commit 1473e3d.

Author: mustard-mh

components/supervisor/cmd/init.go

@@ -15,6 +15,7 @@ import (
 	"os/signal"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -77,31 +78,71 @@ var initCmd = &cobra.Command{
 		}
 
 		supervisorDone := make(chan struct{})
+		handledByReaper := make(chan int)
+		// supervisor is expected to be killed when receiving signals
+		ignoreUnexpectedExitCode := atomic.Bool{}
+		handleSupervisorExit := func(exitCode int) {
+			if exitCode == 0 {
+				return
+			}
+			logs := extractFailureFromRun()
+			if shared.IsExpectedShutdown(exitCode) {
+				log.Fatal(logs)
+			} else {
+				if ignoreUnexpectedExitCode.Load() {
+					return
+				}
+				log.WithError(fmt.Errorf(logs)).Fatal("supervisor run error with unexpected exit code")
+			}
+		}
 		go func() {
 			defer close(supervisorDone)
 
 			err := runCommand.Wait()
-			if err != nil && !(strings.Contains(err.Error(), "signal: ") || strings.Contains(err.Error(), "no child processes")) {
+			if err == nil {
+				return
+			}
+			// exited by reaper
+			if strings.Contains(err.Error(), "no child processes") {
+				ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+				defer cancel()
+				select {
+				case <-ctx.Done(): // timeout
+				case exitCode := <-handledByReaper:
+					handleSupervisorExit(exitCode)
+				}
+			} else if !(strings.Contains(err.Error(), "signal: ")) {
 				if eerr, ok := err.(*exec.ExitError); ok && eerr.ExitCode() != 0 {
-					logs := extractFailureFromRun()
-					if shared.IsExpectedShutdown(eerr.ExitCode()) {
-						log.Fatal(logs)
-					} else {
-						log.WithError(fmt.Errorf(logs)).Fatal("supervisor run error with unexpected exit code")
-					}
+					handleSupervisorExit(eerr.ExitCode())
 				}
 				log.WithError(err).Error("supervisor run error")
 				return
 			}
 		}()
 		// start the reaper to clean up zombie processes
-		reaper.Reap()
+		reaperChan := make(chan reaper.Status, 10)
+		reaper.Start(reaper.Config{
+			Pid:              -1,
+			Options:          0,
+			DisablePid1Check: false,
+			StatusChannel:    reaperChan,
+		})
+		go func() {
+			for status := range reaperChan {
+				if status.Pid != runCommand.Process.Pid {
+					continue
+				}
+				exitCode := status.WaitStatus.ExitStatus()
+				handledByReaper <- exitCode
+			}
+		}()
 
 		select {
 		case <-supervisorDone:
 			// supervisor has ended - we're all done here
 			return
 		case <-sigInput:
+			ignoreUnexpectedExitCode.Store(true)
 			// we received a terminating signal - pass on to supervisor and wait for it to finish
 			ctx, cancel := context.WithTimeout(context.Background(), cfg.GetTerminationGracePeriod())
 			defer cancel()

components/supervisor/go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/gitpod-io/gitpod/ide-metrics-api v0.0.0-00010101000000-000000000000
 	github.com/gitpod-io/gitpod/supervisor/api v0.0.0-00010101000000-000000000000
 	github.com/gitpod-io/gitpod/ws-daemon/api v0.0.0-00010101000000-000000000000
+	github.com/gitpod-io/go-reaper v0.0.0-20241024192051-78d04cc2e25f
 	github.com/golang/mock v1.6.0
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
@@ -29,7 +30,7 @@ require (
 	github.com/prometheus/common v0.42.0
 	github.com/prometheus/procfs v0.10.1
 	github.com/prometheus/pushgateway v1.5.1
-	github.com/ramr/go-reaper v0.2.1
+	github.com/ramr/go-reaper v0.2.2
 	github.com/sirupsen/logrus v1.9.3
 	github.com/soheilhy/cmux v0.1.5
 	github.com/spf13/cobra v1.4.0