Skip to content

Commit 3e33b37

Browse files
committed
Added lastLogOutput, lastPodName, and pollCount tracking variables
1 parent 644ed37 commit 3e33b37

File tree

1 file changed

+31
-11
lines changed

1 file changed

+31
-11
lines changed

internal/integration_tests/standalone.go

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -519,9 +519,13 @@ func waitForPodsTerminated(t *testing.T, namespace string, timeout time.Duration
519519

520520
// waitForBackupPodCompletion polls until a pod matching podNameSubstring appears in the namespace
521521
// and its logs contain successMessage. Returns the pod name and full log output.
522+
// Periodically logs the pod's actual output for diagnostics.
522523
func waitForBackupPodCompletion(t *testing.T, namespace, podNameSubstring, successMessage string, timeout time.Duration) (string, string, error) {
523524
deadline := time.Now().Add(timeout)
524525
var podFound bool
526+
var lastLogOutput string
527+
var lastPodName string
528+
pollCount := 0
525529

526530
for time.Now().Before(deadline) {
527531
pods, err := Clientset.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{})
@@ -535,30 +539,46 @@ func waitForBackupPodCompletion(t *testing.T, namespace, podNameSubstring, succe
535539
for _, pod := range pods.Items {
536540
if strings.Contains(pod.Name, podNameSubstring) {
537541
podFound = true
542+
lastPodName = pod.Name
538543
t.Logf("Found backup pod: %s (status: %s)", pod.Name, pod.Status.Phase)
539544

540545
logOutput, logsErr := kubectlLogs(t, pod.Name, namespace)
541546
if logsErr != nil {
542547
break
543548
}
549+
lastLogOutput = logOutput
544550

545551
if strings.Contains(logOutput, successMessage) {
546552
return pod.Name, logOutput, nil
547553
}
548-
t.Logf("Backup pod '%s' not yet completed, waiting...", podNameSubstring)
554+
555+
// Log pod output periodically (every 4th poll ~2 min) for diagnostics
556+
if pollCount%4 == 0 {
557+
preview := logOutput
558+
if len(preview) > 500 {
559+
preview = preview[:500] + "..."
560+
}
561+
t.Logf("Backup pod '%s' not yet completed. Current logs:\n%s", podNameSubstring, preview)
562+
} else {
563+
t.Logf("Backup pod '%s' not yet completed, waiting...", podNameSubstring)
564+
}
549565
break
550566
}
551567
}
552568

553569
if !podFound {
554570
t.Logf("Backup pod matching '%s' not yet created, waiting...", podNameSubstring)
555571
}
572+
pollCount++
556573
time.Sleep(30 * time.Second)
557574
}
558575

559576
if !podFound {
560577
return "", "", fmt.Errorf("no backup pod matching '%s' found after %s", podNameSubstring, timeout)
561578
}
579+
580+
// Log full pod output on timeout for post-failure diagnostics
581+
t.Logf("TIMEOUT: Backup pod '%s' did not complete within %s. Final pod logs:\n%s", lastPodName, timeout, lastLogOutput)
562582
return "", "", fmt.Errorf("backup pod matching '%s' did not complete within %s", podNameSubstring, timeout)
563583
}
564584

@@ -948,8 +968,8 @@ func TestAggregateBackupWithWildcard(t *testing.T, standaloneReleaseName model.R
948968
{"uninstall", initialBackupReleaseName.String(), "--wait", "--timeout", "3m", "--namespace", namespace},
949969
}, false)
950970

951-
// Step 2: Create second backup to establish backup chains (needed for aggregation)
952-
t.Log("Step 2: Creating second backup to establish backup chains")
971+
// Step 2: Create DIFF backup to establish backup chain (needed for aggregation)
972+
t.Log("Step 2: Creating DIFF backup to establish backup chain")
953973
secondBackupReleaseName := model.NewReleaseName("wildcard-second-backup-" + TestRunIdentifier)
954974

955975
t.Cleanup(func() {
@@ -968,24 +988,24 @@ func TestAggregateBackupWithWildcard(t *testing.T, standaloneReleaseName model.R
968988
SecretName: "gcpcred",
969989
SecretKeyName: "credentials",
970990
KeepBackupFiles: true,
971-
Type: "DIFF", // Differential backup to create chain
991+
Type: "DIFF",
972992
Verbose: true,
973993
}
974994
secondHelmValues.ConsistencyCheck.Enable = false
975995

976-
t.Logf("Installing second backup chart to create backup chains")
996+
t.Logf("Installing DIFF backup chart to create backup chain")
977997
_, err = helmClient.Install(t, secondBackupReleaseName.String(), namespace, secondHelmValues)
978998
if err != nil {
979-
return fmt.Errorf("failed to install second backup helm chart: %v", err)
999+
return fmt.Errorf("failed to install DIFF backup helm chart: %v", err)
9801000
}
9811001

982-
_, _, secondPollErr := waitForBackupPodCompletion(t, namespace, "wildcard-second-backup", "Cloud backup completed successfully", 8*time.Minute)
1002+
_, _, secondPollErr := waitForBackupPodCompletion(t, namespace, "wildcard-second-backup", "Cloud backup completed successfully", 12*time.Minute)
9831003
if secondPollErr != nil {
984-
return fmt.Errorf("second backup did not complete: %v", secondPollErr)
1004+
return fmt.Errorf("DIFF backup did not complete: %v", secondPollErr)
9851005
}
9861006

9871007
// Uninstall step 2 CronJob to prevent interference with the aggregate backup
988-
t.Log("Uninstalling second backup CronJob before proceeding to step 3")
1008+
t.Log("Uninstalling DIFF backup CronJob before proceeding to step 3")
9891009
_ = runAll(t, "helm", [][]string{
9901010
{"uninstall", secondBackupReleaseName.String(), "--wait", "--timeout", "3m", "--namespace", namespace},
9911011
}, false)
@@ -1014,7 +1034,7 @@ func TestAggregateBackupWithWildcard(t *testing.T, standaloneReleaseName model.R
10141034
AggregateBackup: model.AggregateBackup{
10151035
Enabled: true,
10161036
Database: "*",
1017-
FromPath: fmt.Sprintf("gs://%s/", bucketName),
1037+
FromPath: "",
10181038
KeepOldBackup: false,
10191039
ParallelRecovery: false,
10201040
Verbose: true,
@@ -1028,7 +1048,7 @@ func TestAggregateBackupWithWildcard(t *testing.T, standaloneReleaseName model.R
10281048
return fmt.Errorf("failed to install aggregate backup helm chart: %v", err)
10291049
}
10301050

1031-
// Step 4: Poll for aggregate backup completion
1051+
// Step 4: Poll for aggregate backup completion and verify logs
10321052
t.Log("Step 4: Waiting for aggregate backup with wildcard to complete")
10331053

10341054
deadline := time.Now().Add(8 * time.Minute)

0 commit comments

Comments
 (0)