Print kpack debug info on e2e test failure

Print kpack pods logs, kpack images and kpack builds on e2e/smoke/crd
test failure

fixes #4072

Author: danail-branekov

tests/crds/crds_suite_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/google/uuid"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	buildv1alpha2 "github.com/pivotal/kpack/pkg/apis/build/v1alpha2"
 )
 
 func init() {
@@ -33,6 +34,9 @@ func TestCrds(t *testing.T) {
 		Hook: func(config *rest.Config, failure fail_handler.TestFailure) {
 			fail_handler.PrintKorifiLogs(config, "", failure.StartTime)
 			printBuildLogs(config)
+			fail_handler.PrintKpackLogs(config, failure.StartTime)
+			fail_handler.PrintAllObjects(config, testOrg.Name, &buildv1alpha2.BuildList{})
+			fail_handler.PrintAllObjects(config, testOrg.Name, &buildv1alpha2.ImageList{})
 		},
 	})
 	RegisterFailHandler(failHandler.Fail)

tests/e2e/e2e_suite_test.go

@@ -16,16 +16,16 @@ import (
 	"testing"
 	"time"
 
+	"code.cloudfoundry.org/go-loggregator/v8/rpc/loggregator_v2"
 	"code.cloudfoundry.org/korifi/api/payloads"
+	korifiv1alpha1 "code.cloudfoundry.org/korifi/controllers/api/v1alpha1"
 	"code.cloudfoundry.org/korifi/tests/helpers"
 	"code.cloudfoundry.org/korifi/tests/helpers/fail_handler"
-
-	"code.cloudfoundry.org/go-loggregator/v8/rpc/loggregator_v2"
-	korifiv1alpha1 "code.cloudfoundry.org/korifi/controllers/api/v1alpha1"
 	"github.com/go-resty/resty/v2"
 	"github.com/google/uuid"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	buildv1alpha2 "github.com/pivotal/kpack/pkg/apis/build/v1alpha2"
 	"go.yaml.in/yaml/v3"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -325,6 +325,9 @@ func TestE2E(t *testing.T) {
 			Matcher: fail_handler.Always,
 			Hook: func(config *rest.Config, failure fail_handler.TestFailure) {
 				fail_handler.PrintKorifiLogs(config, correlationId, failure.StartTime)
+				fail_handler.PrintKpackLogs(config, failure.StartTime)
+				fail_handler.PrintAllObjects(config, commonTestOrgGUID, &buildv1alpha2.BuildList{})
+				fail_handler.PrintAllObjects(config, commonTestOrgGUID, &buildv1alpha2.ImageList{})
 			},
 		},
 		fail_handler.Hook{

tests/helpers/fail_handler/objects.go

@@ -0,0 +1,97 @@
+package fail_handler
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+
+	"github.com/onsi/ginkgo/v2"
+	buildv1alpha2 "github.com/pivotal/kpack/pkg/apis/build/v1alpha2"
+	"go.yaml.in/yaml/v2"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	rest "k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func init() {
+	utilruntime.Must(clientgoscheme.AddToScheme(scheme.Scheme))
+	utilruntime.Must(corev1.AddToScheme(scheme.Scheme))
+	utilruntime.Must(buildv1alpha2.AddToScheme(scheme.Scheme))
+}
+
+func PrintObject(k8sClient client.Client, obj client.Object) error {
+	if err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(obj), obj); err != nil {
+		return fmt.Errorf("failed to get object %q: %v", client.ObjectKeyFromObject(obj), err)
+	}
+
+	fmt.Fprintf(ginkgo.GinkgoWriter, "\n\n========== %T %s/%s (skipping managed fields) ==========\n", obj, obj.GetNamespace(), obj.GetName())
+	obj.SetManagedFields([]metav1.ManagedFieldsEntry{})
+	objBytes, err := yaml.Marshal(obj)
+	if err != nil {
+		return fmt.Errorf("failed marshalling object %v: %v", obj, err)
+	}
+	fmt.Fprintln(ginkgo.GinkgoWriter, string(objBytes))
+	return nil
+}
+
+func PrintAllObjects(config *rest.Config, orgGUID string, objs client.ObjectList) {
+	k8sClient, err := createK8sClient(config)
+	if err != nil {
+		fmt.Fprintf(ginkgo.GinkgoWriter, "failed to create k8s client: %v\n", err)
+		return
+
+	}
+
+	objects, err := collectObjects(k8sClient, orgGUID, objs)
+	if err != nil {
+		fmt.Fprintf(ginkgo.GinkgoWriter, "failed to list objects %T: %v\n", objs, err)
+		return
+	}
+
+	if len(objects) == 0 {
+		fmt.Fprintf(ginkgo.GinkgoWriter, "no objects found of type %T\n", objs)
+		return
+	}
+
+	for _, o := range objects {
+		if err := PrintObject(k8sClient, o); err != nil {
+			fmt.Fprintf(ginkgo.GinkgoWriter, "failed to print object %s/%s: %v\n", o.GetNamespace(), o.GetName(), err)
+		}
+	}
+}
+
+func collectObjects(k8sClient client.Client, orgGUID string, objectList client.ObjectList) ([]client.Object, error) {
+	spaceNamespaces := &corev1.NamespaceList{}
+	if err := k8sClient.List(context.Background(), spaceNamespaces, client.MatchingLabels{
+		"korifi.cloudfoundry.org/org-guid": orgGUID,
+	}); err != nil {
+		return nil, fmt.Errorf("failed to list %T: %v", objectList, err)
+	}
+
+	objects := []client.Object{}
+	for _, ns := range spaceNamespaces.Items {
+		if err := k8sClient.List(context.Background(), objectList, client.InNamespace(ns.Name)); err != nil {
+			return nil, fmt.Errorf("failed to list %T: %v", objectList, err)
+		}
+
+		s := reflect.ValueOf(objectList).Elem().FieldByName("Items")
+		for i := 0; i < s.Len(); i++ {
+			objects = append(objects, s.Index(i).Addr().Interface().(client.Object))
+		}
+	}
+
+	return objects, nil
+}
+
+func createK8sClient(config *rest.Config) (client.Client, error) {
+	k8sClient, err := client.New(config, client.Options{Scheme: scheme.Scheme})
+	if err != nil {
+		return nil, fmt.Errorf("failed to create k8s client: %v", err)
+	}
+
+	return k8sClient, nil
+}

tests/helpers/fail_handler/pods.go

@@ -218,6 +218,23 @@ func PrintKorifiLogs(config *rest.Config, correlationId string, since time.Time)
 	})
 }
 
+func PrintKpackLogs(config *rest.Config, since time.Time) {
+	PrintPodsLogs(config, []PodDescriptor{
+		{
+			Namespace:  "kpack",
+			LabelKey:   "app",
+			LabelValue: "kpack-controller",
+			Since:      tools.PtrTo(metav1.NewTime(since)),
+		},
+		{
+			Namespace:  "kpack",
+			LabelKey:   "app",
+			LabelValue: "kpack-webhook",
+			Since:      tools.PtrTo(metav1.NewTime(since)),
+		},
+	})
+}
+
 func PrintAllBuildLogs(config *rest.Config, namespace string) {
 	PrintPodsLogs(config, []PodDescriptor{
 		{

tests/smoke/suite_test.go

@@ -1,7 +1,6 @@
 package smoke_test
 
 import (
-	"context"
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
@@ -20,7 +19,7 @@ import (
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gexec"
 	"github.com/onsi/gomega/types"
-	"gopkg.in/yaml.v2"
+	buildv1alpha2 "github.com/pivotal/kpack/pkg/apis/build/v1alpha2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -34,6 +33,7 @@ type SmokeTestSharedData struct {
 	CfAdmin          string `json:"cf_admin"`
 	RootNamespace    string `json:"root_namespace"`
 	OrgName          string `json:"org_name"`
+	OrgGUID          string `json:"org_guid"`
 	BrokerOrgName    string `json:"broker_org_name"`
 	SpaceName        string `json:"space_name"`
 	AppsDomain       string `json:"apps_domain"`
@@ -53,6 +53,9 @@ func TestSmoke(t *testing.T) {
 				printCfApp(config)
 				fail_handler.PrintKorifiLogs(config, "", failure.StartTime)
 				printBuildLogs(config)
+				fail_handler.PrintKpackLogs(config, failure.StartTime)
+				fail_handler.PrintAllObjects(config, sharedData.OrgGUID, &buildv1alpha2.BuildList{})
+				fail_handler.PrintAllObjects(config, sharedData.OrgGUID, &buildv1alpha2.ImageList{})
 			},
 		}).Fail)
 
@@ -95,6 +98,9 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	sharedData.BrokerURL = helpers.GetInClusterURL(getAppGUID(brokerAppName))
 
 	Expect(helpers.Cf("create-org", sharedData.OrgName)).To(Exit(0))
+	sharedData.OrgGUID, err = sessionOutput(helpers.Cf("org", sharedData.OrgName, "--guid"))
+	Expect(err).NotTo(HaveOccurred())
+
 	Expect(helpers.Cf("create-space", "-o", sharedData.OrgName, sharedData.SpaceName)).To(Exit(0))
 	Expect(helpers.Cf("target", "-o", sharedData.OrgName, "-s", sharedData.SpaceName)).To(Exit(0))
 
@@ -110,6 +116,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
 
 	Expect(korifiv1alpha1.AddToScheme(scheme.Scheme)).To(Succeed())
+	Expect(buildv1alpha2.AddToScheme(scheme.Scheme)).To(Succeed())
 })
 
 var _ = SynchronizedAfterSuite(func() {
@@ -192,7 +199,7 @@ func printCfApp(config *rest.Config) {
 		return
 	}
 
-	if err := printObject(k8sClient, &korifiv1alpha1.CFApp{
+	if err := fail_handler.PrintObject(k8sClient, &korifiv1alpha1.CFApp{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      cfAppGUID,
 			Namespace: cfAppNamespace,
@@ -203,21 +210,6 @@ func printCfApp(config *rest.Config) {
 	}
 }
 
-func printObject(k8sClient client.Client, obj client.Object) error {
-	if err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(obj), obj); err != nil {
-		return fmt.Errorf("failed to get object %q: %v\n", client.ObjectKeyFromObject(obj), err)
-	}
-
-	fmt.Fprintf(GinkgoWriter, "\n\n========== %T %s/%s (skipping managed fields) ==========\n", obj, obj.GetNamespace(), obj.GetName())
-	obj.SetManagedFields([]metav1.ManagedFieldsEntry{})
-	objBytes, err := yaml.Marshal(obj)
-	if err != nil {
-		return fmt.Errorf("failed marshalling object %v: %v\n", obj, err)
-	}
-	fmt.Fprintln(GinkgoWriter, string(objBytes))
-	return nil
-}
-
 func printBuildLogs(config *rest.Config) {
 	spaceGUID, err := sessionOutput(helpers.Cf("space", sharedData.SpaceName, "--guid").Wait())
 	if err != nil {