test(e2e): enhance cluster state diagnostics on timeout (cloudnative-pg#10305)

Add missing diagnostics to the cluster state dump printed when a cluster
fails to become ready within the timeout. The existing dump only showed
instance pods (filtered by label) and minimal PVC info, making it
impossible to diagnose scheduling and volume provisioning failures.

Add a section listing all pods in the namespace with their phase, node
assignment, conditions, and container waiting states, so that initdb job
pods stuck in Pending with no node are visible.

Enhance PVC details with storage class, bound volume name, selected-node
annotation, and PVC conditions to surface provisioning errors.

Add namespace events sorted chronologically, capturing FailedScheduling,
ProvisioningFailed, and other events that explain why pods or volumes
are stuck.

Signed-off-by: Marco Nenciarini <marco.nenciarini@enterprisedb.com>
Signed-off-by: Armando Ruocco <armando.ruocco@enterprisedb.com>
Co-authored-by: Armando Ruocco <armando.ruocco@enterprisedb.com>

Author: mnencia

.wordlist-en-custom.txt

@@ -1058,10 +1058,10 @@ livenessProbe
 livenessProbeTimeout
 livenessprobe
 lm
+loadBalancerSourceRanges
 localeCType
 localeCollate
 localeProvider
-loadBalancerSourceRanges
 localhost
 localobjectreference
 locktype
@@ -1270,8 +1270,8 @@ postgresqlcnpgiov
 ppc
 pprof
 pre
-prefetched
 preferredDuringSchedulingIgnoredDuringExecution
+prefetched
 preload
 prepended
 primaryUpdateMethod

tests/utils/utils.go

@@ -23,15 +23,21 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"sort"
 	"text/tabwriter"
+	"time"
 
 	"github.com/cheynewallace/tabby"
 	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	eventsv1 "k8s.io/api/events/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
 	utils2 "github.com/cloudnative-pg/cloudnative-pg/pkg/utils"
 	"github.com/cloudnative-pg/cloudnative-pg/tests/utils/clusterutils"
+	"github.com/cloudnative-pg/cloudnative-pg/tests/utils/namespaces"
+	"github.com/cloudnative-pg/cloudnative-pg/tests/utils/pods"
 	"github.com/cloudnative-pg/cloudnative-pg/tests/utils/run"
 	"github.com/cloudnative-pg/cloudnative-pg/tests/utils/storage"
 )
@@ -68,7 +74,8 @@ func PrintClusterResources(ctx context.Context, crudClient client.Client, namesp
 	clusterInfo.AddLine("Ready pod number: ", utils2.CountReadyPods(podList.Items))
 	clusterInfo.AddLine()
 	clusterInfo.AddHeader("Items", "Values")
-	for _, pod := range podList.Items {
+	for i := range podList.Items {
+		pod := &podList.Items[i]
 		clusterInfo.AddLine("Pod name", pod.Name)
 		clusterInfo.AddLine("Pod phase", pod.Status.Phase)
 		if cluster.Status.InstancesReportedState != nil {
@@ -89,29 +96,38 @@ func PrintClusterResources(ctx context.Context, crudClient client.Client, namesp
 	_ = crudClient.List(
 		ctx, jobList, client.InNamespace(namespace),
 	)
-	for _, job := range jobList.Items {
+	for i := range jobList.Items {
+		job := &jobList.Items[i]
 		clusterInfo.AddLine("Job name", job.Name)
 		clusterInfo.AddLine("Job status", fmt.Sprintf("%#v", job.Status))
 	}
 
+	allPodList, _ := pods.List(ctx, crudClient, namespace)
+	clusterInfo.AddLine()
+	clusterInfo.AddLine("All namespace pods:")
+	clusterInfo.AddLine()
+	clusterInfo.AddHeader("Items", "Values")
+	for i := range allPodList.Items {
+		printPodDiagnostics(clusterInfo, &allPodList.Items[i])
+	}
+
 	pvcList, _ := storage.GetPVCList(ctx, crudClient, cluster.GetNamespace())
 	clusterInfo.AddLine()
 	clusterInfo.AddLine("Cluster PVC information: (dumping all pvc under the namespace)")
 	clusterInfo.AddLine("Available Cluster PVCCount", cluster.Status.PVCCount)
 	clusterInfo.AddLine()
 	clusterInfo.AddHeader("Items", "Values")
-	for _, pvc := range pvcList.Items {
-		clusterInfo.AddLine("PVC name", pvc.Name)
-		clusterInfo.AddLine("PVC phase", pvc.Status.Phase)
-		clusterInfo.AddLine("---", "---")
+	for i := range pvcList.Items {
+		printPVCDiagnostics(clusterInfo, &pvcList.Items[i])
 	}
 
 	snapshotList, _ := storage.GetSnapshotList(ctx, crudClient, cluster.Namespace)
 	clusterInfo.AddLine()
 	clusterInfo.AddLine("Cluster Snapshot information: (dumping all snapshot under the namespace)")
 	clusterInfo.AddLine()
 	clusterInfo.AddHeader("Items", "Values")
-	for _, snapshot := range snapshotList.Items {
+	for i := range snapshotList.Items {
+		snapshot := &snapshotList.Items[i]
 		clusterInfo.AddLine("Snapshot name", snapshot.Name)
 		if snapshot.Status.ReadyToUse != nil {
 			clusterInfo.AddLine("Snapshot ready to use", *snapshot.Status.ReadyToUse)
@@ -121,12 +137,102 @@ func PrintClusterResources(ctx context.Context, crudClient client.Client, namesp
 		clusterInfo.AddLine("---", "---")
 	}
 
+	eventList, _ := namespaces.GetEventList(ctx, crudClient, namespace)
+	printNamespaceEvents(clusterInfo, eventList)
+
 	// do not remove, this is needed to ensure that the writer cache is always flushed.
 	clusterInfo.Print()
 
 	return buffer.String()
 }
 
+func printPodDiagnostics(clusterInfo *tabby.Tabby, pod *corev1.Pod) {
+	clusterInfo.AddLine("Pod name", pod.Name)
+	clusterInfo.AddLine("Pod phase", pod.Status.Phase)
+	clusterInfo.AddLine("Pod node", pod.Spec.NodeName)
+	if pod.Status.Reason != "" {
+		clusterInfo.AddLine("Pod reason", pod.Status.Reason)
+	}
+	if pod.Status.Message != "" {
+		clusterInfo.AddLine("Pod message", pod.Status.Message)
+	}
+	for _, cond := range pod.Status.Conditions {
+		if cond.Status == corev1.ConditionFalse {
+			clusterInfo.AddLine(
+				fmt.Sprintf("Condition %s", cond.Type),
+				fmt.Sprintf("%s: %s", cond.Reason, cond.Message),
+			)
+		}
+	}
+	for _, cs := range pod.Status.InitContainerStatuses {
+		if cs.State.Waiting != nil {
+			clusterInfo.AddLine(
+				fmt.Sprintf("Init container %s waiting", cs.Name),
+				fmt.Sprintf("%s: %s", cs.State.Waiting.Reason, cs.State.Waiting.Message),
+			)
+		}
+	}
+	for _, cs := range pod.Status.ContainerStatuses {
+		if cs.State.Waiting != nil {
+			clusterInfo.AddLine(
+				fmt.Sprintf("Container %s waiting", cs.Name),
+				fmt.Sprintf("%s: %s", cs.State.Waiting.Reason, cs.State.Waiting.Message),
+			)
+		}
+	}
+	clusterInfo.AddLine("---", "---")
+}
+
+func printPVCDiagnostics(clusterInfo *tabby.Tabby, pvc *corev1.PersistentVolumeClaim) {
+	clusterInfo.AddLine("PVC name", pvc.Name)
+	clusterInfo.AddLine("PVC phase", pvc.Status.Phase)
+	if pvc.Spec.StorageClassName != nil {
+		clusterInfo.AddLine("PVC storage class", *pvc.Spec.StorageClassName)
+	}
+	if pvc.Spec.VolumeName != "" {
+		clusterInfo.AddLine("PVC volume", pvc.Spec.VolumeName)
+	}
+	if node, ok := pvc.Annotations["volume.kubernetes.io/selected-node"]; ok {
+		clusterInfo.AddLine("PVC selected node", node)
+	}
+	for _, cond := range pvc.Status.Conditions {
+		clusterInfo.AddLine(
+			fmt.Sprintf("PVC condition %s", cond.Type),
+			fmt.Sprintf("%s: %s", cond.Reason, cond.Message),
+		)
+	}
+	clusterInfo.AddLine("---", "---")
+}
+
+func printNamespaceEvents(clusterInfo *tabby.Tabby, eventList *eventsv1.EventList) {
+	if len(eventList.Items) == 0 {
+		return
+	}
+	eventTimeOf := func(ev *eventsv1.Event) time.Time {
+		if ev.EventTime.Time != (time.Time{}) {
+			return ev.EventTime.Time
+		}
+		return ev.CreationTimestamp.Time
+	}
+	sort.Slice(eventList.Items, func(i, j int) bool {
+		return eventTimeOf(&eventList.Items[i]).Before(eventTimeOf(&eventList.Items[j]))
+	})
+	clusterInfo.AddLine()
+	clusterInfo.AddLine("Namespace events:")
+	clusterInfo.AddLine()
+	clusterInfo.AddHeader("Time", "Type", "Reason", "Object", "Message")
+	for i := range eventList.Items {
+		ev := &eventList.Items[i]
+		clusterInfo.AddLine(
+			eventTimeOf(ev).Format(time.RFC3339),
+			ev.Type,
+			ev.Reason,
+			fmt.Sprintf("%s/%s", ev.Regarding.Kind, ev.Regarding.Name),
+			ev.Note,
+		)
+	}
+}
+
 // ForgeArchiveWalOnMinio instead of using `switchWalCmd` to generate a real WAL archive, directly forges a WAL archive
 // file on Minio by copying and renaming an existing WAL archive file for the sake of more control of testing. To make
 // sure the forged one won't be a real WAL archive, we let the sequence in newWALName to be big enough so that it can't