Collect more info for debugging kubernetes#124136

Author: carlory

test/integration/volume/persistent_volumes_test.go

@@ -1058,6 +1058,25 @@ func TestPersistentVolumeProvisionMultiPVCs(t *testing.T) {
 	defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
 	defer testClient.StorageV1().StorageClasses().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
 
+	// Watch all events in the namespace, and save them to artifacts for debugging.
+	// TODO: This is a temporary solution to debug flaky tests `panic: test timed out after 10m0s`.
+	// We should remove this once https://github.com/kubernetes/kubernetes/issues/124136 is fixed.
+	go func() {
+		w, err := testClient.EventsV1().Events(ns.Name).Watch(tCtx, metav1.ListOptions{})
+		if err != nil {
+			return
+		}
+		for {
+			select {
+			case event := <-w.ResultChan():
+				reportToArtifacts(t.Name()+"-events.text", event.Object)
+			case <-tCtx.Done():
+				w.Stop()
+				return
+			}
+		}
+	}()
+
 	storageClass := storage.StorageClass{
 		TypeMeta: metav1.TypeMeta{
 			Kind: "StorageClass",
@@ -1090,7 +1109,7 @@ func TestPersistentVolumeProvisionMultiPVCs(t *testing.T) {
 
 	// Wait until the controller provisions and binds all of them
 	for i := 0; i < objCount; i++ {
-		waitForAnyPersistentVolumeClaimPhase(watchPVC, v1.ClaimBound)
+		waitForAnyPersistentVolumeClaimPhaseAndReportIt(t, watchPVC, v1.ClaimBound)
 		klog.V(1).Infof("%d claims bound", i+1)
 	}
 	klog.V(2).Infof("TestPersistentVolumeProvisionMultiPVCs: claims are bound")
@@ -1474,6 +1493,21 @@ func waitForAnyPersistentVolumeClaimPhase(w watch.Interface, phase v1.Persistent
 	}
 }
 
+func waitForAnyPersistentVolumeClaimPhaseAndReportIt(t *testing.T, w watch.Interface, phase v1.PersistentVolumeClaimPhase) {
+	for {
+		event := <-w.ResultChan()
+		reportToArtifacts(t.Name()+"-watched-pvcs.text", event)
+		claim, ok := event.Object.(*v1.PersistentVolumeClaim)
+		if !ok {
+			continue
+		}
+		if claim.Status.Phase == phase {
+			klog.V(2).Infof("claim %q is %s", claim.Name, phase)
+			break
+		}
+	}
+}
+
 func waitForPersistentVolumeClaimStorageClass(t *testing.T, claimName, scName string, w watch.Interface, duration time.Duration) (*v1.PersistentVolumeClaim, bool) {
 	stopTimer := time.NewTimer(duration)
 	defer stopTimer.Stop()

test/integration/volume/util_test.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volume
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	"k8s.io/klog/v2"
+)
+
+func reportToArtifacts(filename string, obj any) {
+	if os.Getenv("ARTIFACTS") == "" {
+		return
+	}
+	path := filepath.Join(os.Getenv("ARTIFACTS"), filename)
+	file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		klog.Error("Error opening file:", err)
+		return
+	}
+	defer func() {
+		if err := file.Close(); err != nil {
+			klog.Error("Error closing file:", err)
+		}
+	}()
+
+	content, err := json.Marshal(obj)
+	if err != nil {
+		klog.Error("Error marshalling to json:", err)
+		return
+	}
+	content = append(content, '\n')
+	_, err = file.Write(content)
+	if err != nil {
+		klog.Error("Error writing to file:", err)
+		return
+	}
+}