Skip to content

Commit 78b532c

Browse files
xiaojieyxiaojiey
committed
Collect must-gather
1 parent dbefbbe commit 78b532c

File tree

1 file changed

+93
-73
lines changed

1 file changed

+93
-73
lines changed

‎tests/e2e/framework/common.go‎

Lines changed: 93 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import (
88
"io"
99
"log"
1010
"os"
11+
"os/exec"
12+
"path/filepath"
1113
"regexp"
1214
"strconv"
1315
"strings"
@@ -123,6 +125,88 @@ func (f *Framework) cleanUpFromYAMLFile(p *string) error {
123125
return nil
124126
}
125127

128+
// CollectMustGather collects must-gather data from the cluster for debugging test failures.
129+
// It attempts to determine the must-gather image from the CSV and runs oc adm must-gather.
130+
// If the CSV cannot be found or the must-gather image is not available, it falls back to
131+
// using a default image or skips collection.
132+
// The output is saved to ARTIFACT_DIR if set, otherwise to the current directory.
133+
func (f *Framework) CollectMustGather(testName string) {
134+
timestamp := time.Now().Format("20060102-150405")
135+
outputDirName := fmt.Sprintf("must-gather-%s-%s", testName, timestamp)
136+
137+
// Use ARTIFACT_DIR if set (for CI), otherwise use current directory
138+
baseDir := os.Getenv("ARTIFACT_DIR")
139+
var outputDir string
140+
if baseDir != "" {
141+
outputDir = filepath.Join(baseDir, outputDirName)
142+
log.Printf("Collecting must-gather data to artifact directory: %s\n", outputDir)
143+
} else {
144+
outputDir = outputDirName
145+
log.Printf("Collecting must-gather data to current directory: %s\n", outputDir)
146+
}
147+
148+
// Try to get the must-gather image from the CSV
149+
mustGatherImage := ""
150+
151+
// List all CSVs in the operator namespace
152+
csvList := &unstructured.UnstructuredList{}
153+
csvList.SetGroupVersionKind(metav1.GroupVersionKind{
154+
Group: "operators.coreos.com",
155+
Version: "v1alpha1",
156+
Kind: "ClusterServiceVersionList",
157+
})
158+
159+
if err := f.Client.List(context.TODO(), csvList, client.InNamespace(f.OperatorNamespace)); err != nil {
160+
log.Printf("Warning: Failed to list CSVs: %v\n", err)
161+
} else {
162+
// Find the compliance-operator CSV
163+
for _, csv := range csvList.Items {
164+
if strings.Contains(csv.GetName(), "compliance-operator") {
165+
// Extract must-gather image from relatedImages
166+
relatedImages, found, err := unstructured.NestedSlice(csv.Object, "spec", "relatedImages")
167+
if err == nil && found {
168+
for _, img := range relatedImages {
169+
imgMap, ok := img.(map[string]interface{})
170+
if !ok {
171+
continue
172+
}
173+
name, _, _ := unstructured.NestedString(imgMap, "name")
174+
if name == "must-gather" {
175+
image, _, _ := unstructured.NestedString(imgMap, "image")
176+
mustGatherImage = image
177+
break
178+
}
179+
}
180+
}
181+
break
182+
}
183+
}
184+
}
185+
186+
// If we couldn't find the image from CSV, use the default
187+
if mustGatherImage == "" {
188+
mustGatherImage = "ghcr.io/complianceascode/must-gather-ocp:latest"
189+
log.Printf("Using default must-gather image: %s\n", mustGatherImage)
190+
} else {
191+
log.Printf("Using must-gather image from CSV: %s\n", mustGatherImage)
192+
}
193+
194+
// Run oc adm must-gather
195+
log.Printf("Running: oc adm must-gather --image=%s --dest-dir=%s\n", mustGatherImage, outputDir)
196+
197+
cmd := exec.Command("oc", "adm", "must-gather", fmt.Sprintf("--image=%s", mustGatherImage), fmt.Sprintf("--dest-dir=%s", outputDir))
198+
cmd.Stdout = os.Stdout
199+
cmd.Stderr = os.Stderr
200+
201+
if err := cmd.Run(); err != nil {
202+
log.Printf("Warning: Failed to collect must-gather data: %v\n", err)
203+
log.Printf("Must-gather data was not collected, but test will continue\n")
204+
} else {
205+
log.Printf("Successfully collected must-gather data to %s\n", outputDir)
206+
log.Printf("You can find the must-gather archive in the current directory\n")
207+
}
208+
}
209+
126210
func (f *Framework) PrintROSADebugInfo(t *testing.T) {
127211
// List cluster claims
128212
clusterClaimList := clusterv1alpha1.ClusterClaimList{}
@@ -591,83 +675,19 @@ func (f *Framework) WaitForProfileBundleStatus(name string, status compv1alpha1.
591675
return false, nil
592676
})
593677
if timeouterr != nil {
594-
// Log detailed debug information when ProfileBundle fails to reach the expected status
595-
log.Printf("DEBUG: ProfileBundle %s failed to reach state %s\n", name, status)
596-
log.Printf("DEBUG: Current status: %s\n", pb.Status.DataStreamStatus)
597-
log.Printf("DEBUG: ContentImage: %s\n", pb.Spec.ContentImage)
598-
log.Printf("DEBUG: ContentFile: %s\n", pb.Spec.ContentFile)
678+
// Log basic information about the failure
679+
log.Printf("ProfileBundle %s failed to reach state %s\n", name, status)
680+
log.Printf("Current status: %s\n", pb.Status.DataStreamStatus)
681+
log.Printf("ContentImage: %s\n", pb.Spec.ContentImage)
682+
log.Printf("ContentFile: %s\n", pb.Spec.ContentFile)
599683

600684
if pb.Status.ErrorMessage != "" {
601-
log.Printf("DEBUG: ErrorMessage: %s\n", pb.Status.ErrorMessage)
602-
}
603-
604-
if len(pb.Status.Conditions) > 0 {
605-
log.Printf("DEBUG: Conditions:\n")
606-
for _, condition := range pb.Status.Conditions {
607-
log.Printf(" - Type: %s, Status: %s, Reason: %s, Message: %s\n",
608-
condition.Type, condition.Status, condition.Reason, condition.Message)
609-
}
610-
} else {
611-
log.Printf("DEBUG: No conditions found\n")
685+
log.Printf("ErrorMessage: %s\n", pb.Status.ErrorMessage)
612686
}
613687

614-
// Get and display parser pod information
615-
log.Printf("DEBUG: Fetching parser pod information...\n")
616-
pods, err := f.KubeClient.CoreV1().Pods(f.OperatorNamespace).List(context.TODO(), metav1.ListOptions{
617-
LabelSelector: fmt.Sprintf("profile-bundle=%s", name),
618-
})
619-
if err != nil {
620-
log.Printf("DEBUG: Failed to list parser pods: %v\n", err)
621-
} else if len(pods.Items) == 0 {
622-
log.Printf("DEBUG: No parser pods found for profile-bundle=%s\n", name)
623-
} else {
624-
log.Printf("DEBUG: Found %d parser pod(s):\n", len(pods.Items))
625-
for _, pod := range pods.Items {
626-
log.Printf(" - Pod: %s, Phase: %s, Reason: %s, Message: %s\n",
627-
pod.Name, pod.Status.Phase, pod.Status.Reason, pod.Status.Message)
628-
629-
// Show container statuses
630-
for _, containerStatus := range pod.Status.ContainerStatuses {
631-
log.Printf(" Container: %s, Ready: %v, RestartCount: %d\n",
632-
containerStatus.Name, containerStatus.Ready, containerStatus.RestartCount)
633-
if containerStatus.State.Waiting != nil {
634-
log.Printf(" Waiting: %s - %s\n",
635-
containerStatus.State.Waiting.Reason, containerStatus.State.Waiting.Message)
636-
}
637-
if containerStatus.State.Terminated != nil {
638-
log.Printf(" Terminated: %s (exit code %d) - %s\n",
639-
containerStatus.State.Terminated.Reason,
640-
containerStatus.State.Terminated.ExitCode,
641-
containerStatus.State.Terminated.Message)
642-
}
643-
}
644-
645-
// Get pod logs
646-
log.Printf("DEBUG: Fetching logs for pod %s...\n", pod.Name)
647-
logOpts := &corev1.PodLogOptions{
648-
TailLines: int64Ptr(50),
649-
}
650-
req := f.KubeClient.CoreV1().Pods(f.OperatorNamespace).GetLogs(pod.Name, logOpts)
651-
podLogs, err := req.Stream(context.TODO())
652-
if err != nil {
653-
log.Printf("DEBUG: Failed to get logs for pod %s: %v\n", pod.Name, err)
654-
} else {
655-
defer podLogs.Close()
656-
buf := new(bytes.Buffer)
657-
_, err = buf.ReadFrom(podLogs)
658-
if err != nil {
659-
log.Printf("DEBUG: Failed to read logs: %v\n", err)
660-
} else {
661-
logContent := buf.String()
662-
if logContent != "" {
663-
log.Printf("DEBUG: Last 50 lines of logs from %s:\n%s\n", pod.Name, logContent)
664-
} else {
665-
log.Printf("DEBUG: No logs available from pod %s\n", pod.Name)
666-
}
667-
}
668-
}
669-
}
670-
}
688+
// Collect must-gather data for comprehensive debugging
689+
log.Printf("Collecting must-gather data for detailed debugging information...\n")
690+
f.CollectMustGather(fmt.Sprintf("ProfileBundle-%s", name))
671691

672692
// Include error details in the returned error
673693
errMsg := fmt.Sprintf("ProfileBundle %s failed to reach state %s (current: %s)",

0 commit comments

Comments
 (0)