Merge pull request #8800 from sbueringer/pr-dump-kube-system-pods

🌱 test/e2e: also dump workload cluster kube-system pods

Author: k8s-ci-robot

controlplane/kubeadm/internal/controllers/status.go

@@ -66,8 +66,8 @@ func (r *KubeadmControlPlaneReconciler) updateStatus(ctx context.Context, kcp *c
 		return nil
 	}
 
-	machinesWithHealthAPIServer := ownedMachines.Filter(collections.HealthyAPIServer())
-	lowestVersion := machinesWithHealthAPIServer.LowestVersion()
+	machinesWithHealthyAPIServer := ownedMachines.Filter(collections.HealthyAPIServer())
+	lowestVersion := machinesWithHealthyAPIServer.LowestVersion()
 	if lowestVersion != nil {
 		kcp.Status.Version = lowestVersion
 	}

docs/book/src/developer/logging.md

@@ -176,28 +176,28 @@ In the Log browser the following queries can be used to browse logs by controlle
 Will return logs from the `capi-controller-manager` which are parsed in json. Passing the query through the json parser allows filtering by key-value pairs that are part of nested json objects. For example `.cluster.name` becomes `cluster_name`.
 
 ```
-{app="capi-controller-manager"} | json | cluster_name="my-cluster"
+{app="capi-controller-manager"} | json | Cluster_name="my-cluster"
 ```
 Will return logs from the `capi-controller-manager` that are associated with the Cluster `my-cluster`.
 
 ```
-{app="capi-controller-manager"} | json | cluster_name="my-cluster" | v <= 2
+{app="capi-controller-manager"} | json | Cluster_name="my-cluster" | v <= 2
 ```
 Will return logs from the `capi-controller-manager` that are associated with the Cluster `my-cluster` with log level <= 2.
 
 ```
-{app="capi-controller-manager"} | json | cluster_name="my-cluster" reconcileID="6f6ad971-bdb6-4fa3-b803-xxxxxxxxxxxx"
+{app="capi-controller-manager"} | json | Cluster_name="my-cluster" reconcileID="6f6ad971-bdb6-4fa3-b803-xxxxxxxxxxxx"
 ```
 
 Will return logs from the `capi-controller-manager`, associated with the Cluster `my-cluster` and the Reconcile ID `6f6ad971-bdb6-4fa3-b803-xxxxxxxxxxxx`. Each reconcile loop will have a unique Reconcile ID.
 
 ```
-{app="capi-controller-manager"} | json | cluster_name="my-cluster" reconcileID="6f6ad971-bdb6-4fa3-b803-ef81c5c8f9d0" controller="cluster" | line_format "{{ .msg }}"
+{app="capi-controller-manager"} | json | Cluster_name="my-cluster" reconcileID="6f6ad971-bdb6-4fa3-b803-ef81c5c8f9d0" controller="cluster" | line_format "{{ .msg }}"
 ```
 Will return logs from the `capi-controller-manager`, associated with the Cluster `my-cluster` and the Reconcile ID `6f6ad971-bdb6-4fa3-b803-xxxxxxxxxxxx` it further selects only those logs which come from the Cluster controller. It will then format the logs so only the message is displayed.
 
 ```
-{app=~"capd-controller-manager|capi-kubeadm-bootstrap-controller-manager|capi-kubeadm-control-plane-controller-manager"} | json | cluster_name="my-cluster" machine_name="my-cluster-linux-worker-1" | line_format "{{.controller}} {{.msg}}"
+{app=~"capd-controller-manager|capi-kubeadm-bootstrap-controller-manager|capi-kubeadm-control-plane-controller-manager"} | json | Cluster_name="my-cluster" Machine_name="my-cluster-linux-worker-1" | line_format "{{.controller}} {{.msg}}"
 ```
 
 Will return the logs from four CAPI providers - the Core provider, Kubeadm Control Plane provider, Kubeadm Bootstrap provider and the Docker infrastructure provider. It filters by the cluster name and the machine name and then formats the log lines to show just the source controller and the message. This allows us to correlate logs and see actions taken by each of these four providers related to the machine `my-cluster-linux-worker-1`.

test/e2e/common.go

@@ -26,6 +26,7 @@ import (
 	"github.com/onsi/gomega/types"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/framework"
@@ -78,6 +79,14 @@ func dumpSpecResourcesAndCleanup(ctx context.Context, specName string, clusterPr
 		LogPath:   filepath.Join(artifactFolder, "clusters", clusterProxy.GetName(), "resources"),
 	})
 
+	// If the cluster still exists, dump kube-system pods in the workload cluster before deleting the cluster.
+	if err := clusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(cluster), &clusterv1.Cluster{}); err == nil {
+		framework.DumpKubeSystemPods(ctx, framework.DumpKubeSystemPodsInput{
+			Lister:  clusterProxy.GetWorkloadCluster(ctx, cluster.Namespace, cluster.Name).GetClient(),
+			LogPath: filepath.Join(artifactFolder, "clusters", cluster.Name, "resources"),
+		})
+	}
+
 	if !skipCleanup {
 		Byf("Deleting cluster %s", klog.KObj(cluster))
 		// While https://github.com/kubernetes-sigs/cluster-api/issues/2955 is addressed in future iterations, there is a chance

test/framework/alltypes_helpers.go

@@ -26,6 +26,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -49,7 +50,7 @@ type GetCAPIResourcesInput struct {
 // This list includes all the types belonging to CAPI providers.
 func GetCAPIResources(ctx context.Context, input GetCAPIResourcesInput) []*unstructured.Unstructured {
 	Expect(ctx).NotTo(BeNil(), "ctx is required for GetCAPIResources")
-	Expect(input.Lister).NotTo(BeNil(), "input.Deleter is required for GetCAPIResources")
+	Expect(input.Lister).NotTo(BeNil(), "input.Lister is required for GetCAPIResources")
 	Expect(input.Namespace).NotTo(BeEmpty(), "input.Namespace is required for GetCAPIResources")
 
 	types := getClusterAPITypes(ctx, input.Lister)
@@ -76,6 +77,34 @@ func GetCAPIResources(ctx context.Context, input GetCAPIResourcesInput) []*unstr
 	return objList
 }
 
+// GetKubeSystemPodsInput is the input for GetKubeSystemPods.
+type GetKubeSystemPodsInput struct {
+	Lister Lister
+}
+
+// GetKubeSystemPods reads all pods in the kube-system namespace.
+// Note: This function intentionally retrieves Pods as Unstructured, because we need the Pods
+// as Unstructured eventually.
+func GetKubeSystemPods(ctx context.Context, input GetKubeSystemPodsInput) []*unstructured.Unstructured {
+	Expect(ctx).NotTo(BeNil(), "ctx is required for GetKubeSystemPods")
+	Expect(input.Lister).NotTo(BeNil(), "input.Lister is required for GetKubeSystemPods")
+
+	podList := new(unstructured.UnstructuredList)
+	podList.SetAPIVersion(corev1.SchemeGroupVersion.String())
+	podList.SetKind("Pod")
+	if err := input.Lister.List(ctx, podList, client.InNamespace(metav1.NamespaceSystem)); err != nil {
+		Fail(fmt.Sprintf("failed to list Pods in kube-system: %v", err))
+	}
+
+	objList := []*unstructured.Unstructured{}
+	for i := range podList.Items {
+		obj := podList.Items[i]
+		objList = append(objList, &obj)
+	}
+
+	return objList
+}
+
 // getClusterAPITypes returns the list of TypeMeta to be considered for the move discovery phase.
 // This list includes all the types belonging to CAPI providers.
 func getClusterAPITypes(ctx context.Context, lister Lister) []metav1.TypeMeta {
@@ -115,7 +144,7 @@ type DumpAllResourcesInput struct {
 // This dump includes all the types belonging to CAPI providers.
 func DumpAllResources(ctx context.Context, input DumpAllResourcesInput) {
 	Expect(ctx).NotTo(BeNil(), "ctx is required for DumpAllResources")
-	Expect(input.Lister).NotTo(BeNil(), "input.Deleter is required for DumpAllResources")
+	Expect(input.Lister).NotTo(BeNil(), "input.Lister is required for DumpAllResources")
 	Expect(input.Namespace).NotTo(BeEmpty(), "input.Namespace is required for DumpAllResources")
 
 	resources := GetCAPIResources(ctx, GetCAPIResourcesInput{
@@ -129,6 +158,27 @@ func DumpAllResources(ctx context.Context, input DumpAllResourcesInput) {
 	}
 }
 
+// DumpKubeSystemPodsInput is the input for DumpKubeSystemPods.
+type DumpKubeSystemPodsInput struct {
+	Lister  Lister
+	LogPath string
+}
+
+// DumpKubeSystemPods dumps kube-system Pods to YAML.
+func DumpKubeSystemPods(ctx context.Context, input DumpKubeSystemPodsInput) {
+	Expect(ctx).NotTo(BeNil(), "ctx is required for DumpAllResources")
+	Expect(input.Lister).NotTo(BeNil(), "input.Lister is required for DumpAllResources")
+
+	resources := GetKubeSystemPods(ctx, GetKubeSystemPodsInput{
+		Lister: input.Lister,
+	})
+
+	for i := range resources {
+		r := resources[i]
+		dumpObject(r, input.LogPath)
+	}
+}
+
 func dumpObject(resource runtime.Object, logPath string) {
 	resourceYAML, err := yaml.Marshal(resource)
 	Expect(err).ToNot(HaveOccurred(), "Failed to marshal %s", resource.GetObjectKind().GroupVersionKind().String())