refactor: improve error visibility on devspace dev

Author: FabianKramm

cmd/dev.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/loft-sh/devspace/pkg/util/stringutil"
 	"io"
 	"os"
 	"os/exec"
@@ -758,7 +759,7 @@ func GetPaths(config *latest.Config) []string {
 		}
 	}
 
-	return removeDuplicates(paths)
+	return stringutil.RemoveDuplicates(paths)
 }
 
 func (cmd *DevCmd) loadConfig(configOptions *loader.ConfigOptions) (config.Config, error) {
@@ -823,25 +824,6 @@ func defaultStdStreams(stdout io.Writer, stderr io.Writer, stdin io.Reader) (io.
 	return stdout, stderr, stdin
 }
 
-func removeDuplicates(arr []string) []string {
-	newArr := []string{}
-	for _, v := range arr {
-		if !contains(newArr, v) {
-			newArr = append(newArr, v)
-		}
-	}
-	return newArr
-}
-
-func contains(haystack []string, needle string) bool {
-	for _, v := range haystack {
-		if v == needle {
-			return true
-		}
-	}
-	return false
-}
-
 func updateLastKubeContext(configLoader loader.ConfigLoader, client kubectl.Client, generatedConfig *generated.Config) error {
 	// Update generated if we deploy the application
 	if generatedConfig != nil {

pkg/devspace/hook/hook.go

@@ -143,16 +143,16 @@ func executeSingle(client kubectl.Client, config config.Config, dependencies []t
 			var hook Hook
 			if hookConfig.Container != nil {
 				if hookConfig.Upload != nil {
-					hook = NewRemoteHook(NewUploadHook())
+					hook = NewRemoteHook(NewUploadHook(), client, hookConfig.Container.Namespace)
 				} else if hookConfig.Download != nil {
-					hook = NewRemoteHook(NewDownloadHook())
+					hook = NewRemoteHook(NewDownloadHook(), client, hookConfig.Container.Namespace)
 				} else if hookConfig.Logs != nil {
 					// we use another waiting strategy here, because the pod might has finished already
-					hook = NewRemoteHookWithWaitingStrategy(NewLogsHook(hookWriter), targetselector.NewUntilNotWaitingStrategy(time.Second*2))
+					hook = NewRemoteHookWithWaitingStrategy(NewLogsHook(hookWriter), targetselector.NewUntilNotWaitingStrategy(time.Second*2, client, hookConfig.Container.Namespace))
 				} else if hookConfig.Wait != nil {
 					hook = NewWaitHook()
 				} else {
-					hook = NewRemoteHook(NewRemoteCommandHook(hookWriter, hookWriter))
+					hook = NewRemoteHook(NewRemoteCommandHook(hookWriter, hookWriter), client, hookConfig.Container.Namespace)
 				}
 			} else {
 				hook = NewLocalCommandHook(hookWriter, hookWriter)

pkg/devspace/hook/remote_hook.go

@@ -23,10 +23,10 @@ type RemoteHook interface {
 	ExecuteRemotely(hook *latest.HookConfig, podContainer *selector.SelectedPodContainer, client kubectl.Client, config config.Config, dependencies []types.Dependency, log logpkg.Logger) error
 }
 
-func NewRemoteHook(hook RemoteHook) Hook {
+func NewRemoteHook(hook RemoteHook, client kubectl.Client, namespace string) Hook {
 	return &remoteHook{
 		Hook:            hook,
-		WaitingStrategy: targetselector.NewUntilNewestRunningWaitingStrategy(time.Second * 2),
+		WaitingStrategy: targetselector.NewUntilNewestRunningWaitingStrategy(time.Second*2, client, namespace),
 	}
 }
 

pkg/devspace/services/logs.go

@@ -24,7 +24,7 @@ func (serviceClient *client) StartLogsWithWriter(options targetselector.Options,
 	} else {
 		options.FilterPod = nil
 		options.FilterContainer = nil
-		options.WaitingStrategy = targetselector.NewUntilNotWaitingStrategy(time.Second * 2)
+		options.WaitingStrategy = targetselector.NewUntilNotWaitingStrategy(time.Second*2, serviceClient.client, options.Namespace)
 	}
 
 	container, err := targetSelector.SelectSingleContainer(context.TODO(), options, serviceClient.log)

pkg/devspace/services/podreplace/persist.go

@@ -72,7 +72,7 @@ func persistPaths(podName string, replacePod *latest.ReplacePod, copiedPod *core
 			Name:    fmt.Sprintf("path-%d-init", i),
 			Image:   container.Image,
 			Command: []string{"sh"},
-			Args:    []string{"-c", fmt.Sprintf("if [ ! -d \"/devspace-persistence/.devspace/\" ] && [ -d \"%s\" ]; then cp -a %s/. /devspace-persistence/ && mkdir /devspace-persistence/.devspace ; fi", path.Clean(p.Path), path.Clean(p.Path))},
+			Args:    []string{"-c", fmt.Sprintf("set -x\nif [ ! -d \"/devspace-persistence/.devspace/\" ] && [ -d \"%s\" ]; then\n cp -a %s/. /devspace-persistence/ && mkdir /devspace-persistence/.devspace\nfi", path.Clean(p.Path), path.Clean(p.Path))},
 			VolumeMounts: []corev1.VolumeMount{
 				{
 					Name:      "devspace-persistence",

pkg/devspace/services/podreplace/replace.go

@@ -829,7 +829,7 @@ func findSingleReplacedPod(ctx context.Context, client kubectl.Client, replacePo
 	targetOptions := targetselector.NewEmptyOptions().ApplyConfigParameter(labelSelector, replacePod.Namespace, replacePod.ContainerName, "")
 	targetOptions.Timeout = 30
 	targetOptions.AllowPick = false
-	targetOptions.WaitingStrategy = targetselector.NewUntilNotTerminatingStrategy(0)
+	targetOptions.WaitingStrategy = targetselector.NewUntilNotTerminatingStrategy(0, client, targetOptions.Namespace)
 	targetOptions.SkipInitContainers = true
 	selected, err := targetselector.NewTargetSelector(client).SelectSingleContainer(ctx, targetOptions, log)
 	if err != nil {
@@ -902,7 +902,7 @@ func findSingleReplaceablePodParent(ctx context.Context, client kubectl.Client,
 	targetOptions := targetselector.NewEmptyOptions().ApplyConfigParameter(replacePod.LabelSelector, replacePod.Namespace, replacePod.ContainerName, "")
 	targetOptions.Timeout = int64(300)
 	targetOptions.AllowPick = false
-	targetOptions.WaitingStrategy = targetselector.NewUntilNotTerminatingStrategy(time.Second * 2)
+	targetOptions.WaitingStrategy = targetselector.NewUntilNotTerminatingStrategy(time.Second*2, client, targetOptions.Namespace)
 	targetOptions.SkipInitContainers = true
 	targetOptions.ImageSelector = []imageselector.ImageSelector{}
 	if replacePod.ImageSelector != "" {

pkg/devspace/services/port_forwarding.go

@@ -122,7 +122,7 @@ func (serviceClient *client) startForwarding(portForwarding *latest.PortForwardi
 
 		options.ImageSelector = append(options.ImageSelector, *imageSelector)
 	}
-	options.WaitingStrategy = targetselector.NewUntilNewestRunningWaitingStrategy(time.Second * 2)
+	options.WaitingStrategy = targetselector.NewUntilNewestRunningWaitingStrategy(time.Second*2, serviceClient.client, options.Namespace)
 	options.SkipInitContainers = true
 
 	// start port forwarding

pkg/devspace/services/reverse_port_forwarding.go

@@ -34,7 +34,7 @@ func (serviceClient *client) startReversePortForwarding(portForwarding *latest.P
 
 		options.ImageSelector = append(options.ImageSelector, *imageSelector)
 	}
-	options.WaitingStrategy = targetselector.NewUntilNewestRunningWaitingStrategy(time.Second * 2)
+	options.WaitingStrategy = targetselector.NewUntilNewestRunningWaitingStrategy(time.Second*2, serviceClient.client, options.Namespace)
 	options.SkipInitContainers = true
 
 	log.Info("Reverse-Port-Forwarding: Waiting for containers to start...")

pkg/devspace/services/sync.go

@@ -98,7 +98,7 @@ func (serviceClient *client) newSyncFn(idx int, syncConfig *latest.SyncConfig, i
 	return func() error {
 		targetOptions := targetselector.NewEmptyOptions().ApplyConfigParameter(syncConfig.LabelSelector, syncConfig.Namespace, syncConfig.ContainerName, "")
 		targetOptions.AllowPick = false
-		targetOptions.WaitingStrategy = targetselector.NewUntilNewestRunningWaitingStrategy(time.Second * 2)
+		targetOptions.WaitingStrategy = targetselector.NewUntilNewestRunningWaitingStrategy(time.Second*2, serviceClient.client, targetOptions.Namespace)
 
 		// set options
 		options := &synccontroller.Options{

pkg/devspace/services/targetselector/until_newest_running.go

@@ -1,6 +1,9 @@
 package targetselector
 
 import (
+	"context"
+	"github.com/loft-sh/devspace/pkg/util/stringutil"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sort"
 	"strings"
 	"sync"
@@ -13,10 +16,14 @@ import (
 )
 
 // NewUntilNewestRunningWaitingStrategy creates a new waiting strategy
-func NewUntilNewestRunningWaitingStrategy(initialDelay time.Duration) WaitingStrategy {
+func NewUntilNewestRunningWaitingStrategy(initialDelay time.Duration, client kubectl.Client, namespace string) WaitingStrategy {
 	return &untilNewestRunning{
 		initialDelay: time.Now().Add(initialDelay),
-		lastWarning:  time.Now().Add(initialDelay),
+		podInfoPrinter: &PodInfoPrinter{
+			lastWarning: time.Now().Add(initialDelay),
+			namespace:   namespace,
+			client:      client,
+		},
 	}
 }
 
@@ -25,27 +32,26 @@ func NewUntilNewestRunningWaitingStrategy(initialDelay time.Duration) WaitingStr
 type untilNewestRunning struct {
 	initialDelay time.Time
 
-	lastMutex   sync.Mutex
-	lastWarning time.Time
+	podInfoPrinter *PodInfoPrinter
 }
 
 func (u *untilNewestRunning) SelectPod(pods []*v1.Pod, log log.Logger) (bool, *v1.Pod, error) {
 	now := time.Now()
 	if now.Before(u.initialDelay) {
 		return false, nil, nil
 	} else if len(pods) == 0 {
-		u.printNotFoundWarning(log)
+		u.podInfoPrinter.PrintNotFoundWarning(log)
 		return false, nil, nil
 	}
 
 	sort.Slice(pods, func(i, j int) bool {
 		return selector.SortPodsByNewest(pods, i, j)
 	})
 	if HasPodProblem(pods[0]) {
-		u.printPodWarning(pods[0], log)
+		u.podInfoPrinter.PrintPodWarning(pods[0], log)
 		return false, nil, nil
 	} else if kubectl.GetPodStatus(pods[0]) != "Running" {
-		u.printPodInfo(pods[0], log)
+		u.podInfoPrinter.PrintPodInfo(pods[0], log)
 		return false, nil, nil
 	}
 
@@ -57,56 +63,148 @@ func (u *untilNewestRunning) SelectContainer(containers []*selector.SelectedPodC
 	if now.Before(u.initialDelay) {
 		return false, nil, nil
 	} else if len(containers) == 0 {
-		u.printNotFoundWarning(log)
+		u.podInfoPrinter.PrintNotFoundWarning(log)
 		return false, nil, nil
 	}
 
 	sort.Slice(containers, func(i, j int) bool {
 		return selector.SortContainersByNewest(containers, i, j)
 	})
 	if HasPodProblem(containers[0].Pod) {
-		u.printPodWarning(containers[0].Pod, log)
+		u.podInfoPrinter.PrintPodWarning(containers[0].Pod, log)
 		return false, nil, nil
 	} else if !IsContainerRunning(containers[0]) {
-		u.printPodInfo(containers[0].Pod, log)
+		u.podInfoPrinter.PrintPodInfo(containers[0].Pod, log)
 		return false, nil, nil
 	}
 
 	return true, containers[0], nil
 }
 
-func (u *untilNewestRunning) printPodInfo(pod *v1.Pod, log log.Logger) {
+type PodInfoPrinter struct {
+	lastMutex   sync.Mutex
+	lastWarning time.Time
+
+	namespace   string
+	client      kubectl.Client
+	shownEvents []string
+}
+
+func (u *PodInfoPrinter) PrintPodInfo(pod *v1.Pod, log log.Logger) {
 	u.lastMutex.Lock()
 	defer u.lastMutex.Unlock()
 
 	if time.Since(u.lastWarning) > time.Second*10 {
 		status := kubectl.GetPodStatus(pod)
-		log.Infof("DevSpace is waiting, because Pod %s/%s has status: %s", pod.Namespace, pod.Name, status)
+		log.Infof("DevSpace is waiting, because Pod %s has status: %s", pod.Name, status)
+
+		u.shownEvents = displayWarnings(relevantObjectsFromPod(pod), pod.Namespace, u.client, u.shownEvents, log)
 		u.lastWarning = time.Now()
 	}
 }
 
-func (u *untilNewestRunning) printNotFoundWarning(log log.Logger) {
+func (u *PodInfoPrinter) PrintNotFoundWarning(log log.Logger) {
 	u.lastMutex.Lock()
 	defer u.lastMutex.Unlock()
 
 	if time.Since(u.lastWarning) > time.Second*10 {
 		log.Warnf("DevSpace still couldn't find any Pods that match the selector. DevSpace will continue waiting, but this operation might timeout")
+
+		u.shownEvents = displayWarnings([]relevantObject{
+			{
+				Kind: "StatefulSet",
+			},
+			{
+				Kind: "Deployment",
+			},
+			{
+				Kind: "ReplicaSet",
+			},
+		}, u.namespace, u.client, u.shownEvents, log)
 		u.lastWarning = time.Now()
 	}
 }
 
-func (u *untilNewestRunning) printPodWarning(pod *v1.Pod, log log.Logger) {
+func (u *PodInfoPrinter) PrintPodWarning(pod *v1.Pod, log log.Logger) {
 	u.lastMutex.Lock()
 	defer u.lastMutex.Unlock()
 
 	if time.Since(u.lastWarning) > time.Second*10 {
 		status := kubectl.GetPodStatus(pod)
-		log.Warnf("Pod %s/%s has critical status: %s. DevSpace will continue waiting, but this operation might timeout", pod.Namespace, pod.Name, status)
+		log.Warnf("Pod %s has critical status: %s. DevSpace will continue waiting, but this operation might timeout", pod.Name, status)
 		u.lastWarning = time.Now()
 	}
 }
 
+type relevantObject struct {
+	Kind string
+	Name string
+	UID  string
+}
+
+func displayWarnings(relevantObjects []relevantObject, namespace string, client kubectl.Client, filter []string, log log.Logger) []string {
+	events, err := client.KubeClient().CoreV1().Events(namespace).List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		log.Debugf("Error retrieving pod events: %v", err)
+		return nil
+	}
+
+	sort.Slice(events.Items, func(i, j int) bool {
+		return events.Items[i].CreationTimestamp.Unix() > events.Items[j].CreationTimestamp.Unix()
+	})
+	for _, event := range events.Items {
+		if event.Type != "Warning" {
+			continue
+		} else if stringutil.Contains(filter, event.Name) {
+			continue
+		} else if !eventMatches(&event, relevantObjects) {
+			continue
+		}
+
+		log.Warnf("%s %s: %s (%s)", event.InvolvedObject.Kind, event.InvolvedObject.Name, event.Message, event.Reason)
+		filter = append(filter, event.Name)
+	}
+
+	return filter
+}
+
+func relevantObjectsFromPod(pod *v1.Pod) []relevantObject {
+	// search for persistent volume claims
+	objects := []relevantObject{
+		{
+			Kind: "Pod",
+			Name: pod.Name,
+			UID:  string(pod.UID),
+		},
+	}
+	for _, v := range pod.Spec.Volumes {
+		if v.PersistentVolumeClaim != nil {
+			objects = append(objects, relevantObject{
+				Kind: "PersistentVolumeClaim",
+				Name: v.PersistentVolumeClaim.ClaimName,
+			})
+		}
+
+	}
+	return objects
+}
+
+func eventMatches(event *v1.Event, objects []relevantObject) bool {
+	for _, o := range objects {
+		if o.Name != "" && event.InvolvedObject.Name != o.Name {
+			continue
+		} else if o.Kind != "" && event.InvolvedObject.Kind != o.Kind {
+			continue
+		} else if o.UID != "" && string(event.InvolvedObject.UID) != o.UID {
+			continue
+		}
+
+		return true
+	}
+
+	return false
+}
+
 func IsContainerRunning(container *selector.SelectedPodContainer) bool {
 	if container.Pod.DeletionTimestamp != nil {
 		return false