backup: update Backup CR status to Failed when log backup task not found (#6630) (#6637)

Co-authored-by: RidRisR <[email protected]>

Author: ti-chi-bot

pkg/backup/backup/backup_tracker.go

@@ -453,7 +453,15 @@ func (bt *backupTracker) SyncLogBackupState(backup *v1alpha1.Backup) (bool, erro
 			// After stop, key not existing is normal
 			return true, nil
 		}
-		// For other commands, key not existing is an error
+		// For other commands, key not existing is an error - update status to Failed
+		klog.Errorf("log backup %s/%s info key not found in etcd, marking as failed", ns, name)
+		bt.statusUpdater.Update(backup, &v1alpha1.BackupCondition{
+			Command: command,
+			Type:    v1alpha1.BackupFailed,
+			Status:  corev1.ConditionTrue,
+			Reason:  "LogBackupTaskNotFound",
+			Message: "Log backup task not found in TiKV, the task may have been deleted or cluster state is inconsistent",
+		}, nil)
 		return false, fmt.Errorf("log backup key not found")
 	}
 

tests/e2e/br/br.go

@@ -770,6 +770,120 @@ var _ = ginkgo.Describe("Backup and Restore", func() {
 		// 	k8se2e.ExpectNoError(err)
 		// 	k8se2e.ExpectEqual(cleaned, true, "storage should be cleaned")
 		// })
+
+		ginkgo.It("log backup task deleted externally should fail with LogBackupTaskNotFound", func() {
+			backupClusterName := "log-backup-not-found"
+			backupVersion := utilimage.TiDBLatest
+			enableTLS := false
+			skipCA := false
+			backupName := backupClusterName
+			typ := strings.ToLower(typeBR)
+
+			ns := f.Namespace.Name
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			ginkgo.By("Create log-backup.enable TiDB cluster")
+			err := createLogBackupEnableTidbCluster(f, backupClusterName, backupVersion, enableTLS, skipCA)
+			framework.ExpectNoError(err)
+
+			ginkgo.By("Wait for backup TiDB cluster ready")
+			err = utiltidbcluster.WaitForTCConditionReady(f.ExtClient, ns, backupClusterName, tidbReadyTimeout, 0)
+			framework.ExpectNoError(err)
+
+			ginkgo.By("Create RBAC for log backup")
+			err = createRBAC(f)
+			framework.ExpectNoError(err)
+
+			ginkgo.By("Start log backup")
+			backup, err := createBackupAndWaitForComplete(f, backupName, backupClusterName, typ, func(backup *v1alpha1.Backup) {
+				backup.Spec.CleanPolicy = v1alpha1.CleanPolicyTypeDelete
+				backup.Spec.Mode = v1alpha1.BackupModeLog
+			})
+			framework.ExpectNoError(err)
+			framework.ExpectNotEqual(backup.Status.CommitTs, "")
+			framework.ExpectEqual(backup.Status.Phase, v1alpha1.BackupRunning)
+
+			ginkgo.By("Stop log backup task via BR command (bypass operator)")
+			err = stopLogBackupTaskViaBR(f, backupClusterName, backupName)
+			framework.ExpectNoError(err)
+
+			ginkgo.By("Wait for backup to fail with LogBackupTaskNotFound")
+			err = brutil.WaitForLogBackupFailedWithReason(f.ExtClient, ns, backupName, "LogBackupTaskNotFound", backupCompleteTimeout)
+			framework.ExpectNoError(err)
+
+			ginkgo.By("Verify backup can be deleted")
+			err = deleteBackup(f, backupName)
+			framework.ExpectNoError(err)
+
+			ginkgo.By("Check if all backup files in storage is deleted")
+			cleaned, err := f.Storage.IsDataCleaned(ctx, ns, backup.Spec.S3.Prefix)
+			framework.ExpectNoError(err)
+			framework.ExpectEqual(cleaned, true, "storage should be cleaned")
+		})
+
+		ginkgo.It("log backup should wait and start when cluster becomes ready", func() {
+			backupClusterName := "log-backup-wait-cluster"
+			backupVersion := utilimage.TiDBLatest
+			enableTLS := false
+			skipCA := false
+			backupName := backupClusterName
+			typ := strings.ToLower(typeBR)
+
+			ns := f.Namespace.Name
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			ginkgo.By("Create RBAC for log backup first")
+			err := createRBAC(f)
+			framework.ExpectNoError(err)
+
+			ginkgo.By("Create log backup CR before cluster exists")
+			backupFolder := time.Now().Format(time.RFC3339)
+			cfg := f.Storage.Config(ns, backupFolder)
+			s := brutil.GetSecret(ns, backupName, "")
+			_, err = f.ClientSet.CoreV1().Secrets(ns).Create(context.TODO(), s, metav1.CreateOptions{})
+			framework.ExpectNoError(err)
+
+			backup := brutil.GetBackup(ns, backupName, backupClusterName, typ, cfg)
+			backup.Spec.CleanPolicy = v1alpha1.CleanPolicyTypeDelete
+			backup.Spec.Mode = v1alpha1.BackupModeLog
+			_, err = f.ExtClient.PingcapV1alpha1().Backups(ns).Create(context.TODO(), backup, metav1.CreateOptions{})
+			framework.ExpectNoError(err)
+
+			ginkgo.By("Verify backup is waiting (cluster not found)")
+			// Wait a few seconds to let controller attempt reconcile
+			time.Sleep(10 * time.Second)
+			_, err = f.ExtClient.PingcapV1alpha1().Backups(ns).Get(context.TODO(), backupName, metav1.GetOptions{})
+			framework.ExpectNoError(err)
+
+			ginkgo.By("Create log-backup.enable TiDB cluster")
+			err = createLogBackupEnableTidbCluster(f, backupClusterName, backupVersion, enableTLS, skipCA)
+			framework.ExpectNoError(err)
+
+			ginkgo.By("Wait for TiDB cluster ready")
+			err = utiltidbcluster.WaitForTCConditionReady(f.ExtClient, ns, backupClusterName, tidbReadyTimeout, 0)
+			framework.ExpectNoError(err)
+
+			ginkgo.By("Wait for backup to become Running")
+			err = brutil.WaitForBackupComplete(f.ExtClient, ns, backupName, backupCompleteTimeout)
+			framework.ExpectNoError(err)
+
+			ginkgo.By("Verify CommitTs is set")
+			backup, err = f.ExtClient.PingcapV1alpha1().Backups(ns).Get(context.TODO(), backupName, metav1.GetOptions{})
+			framework.ExpectNoError(err)
+			framework.ExpectNotEqual(backup.Status.CommitTs, "", "CommitTs should be set")
+			framework.ExpectEqual(backup.Status.Phase, v1alpha1.BackupRunning)
+
+			ginkgo.By("Delete backup")
+			err = deleteBackup(f, backupName)
+			framework.ExpectNoError(err)
+
+			ginkgo.By("Check if all backup files in storage is deleted")
+			cleaned, err := f.Storage.IsDataCleaned(ctx, ns, backup.Spec.S3.Prefix)
+			framework.ExpectNoError(err)
+			framework.ExpectEqual(cleaned, true, "storage should be cleaned")
+		})
 	})
 
 	// the following cases may encounter errors after restarting the backup pod:
@@ -1836,3 +1950,57 @@ func createCompactBackupAndWaitForComplete(f *e2eframework.Framework, name, tcNa
 	}
 	return f.ExtClient.PingcapV1alpha1().CompactBackups(ns).Get(context.TODO(), name, metav1.GetOptions{})
 }
+
+// stopLogBackupTaskViaBR stops a log backup task directly via BR command, bypassing the operator.
+// This is used to simulate external deletion of the log backup task in TiKV.
+func stopLogBackupTaskViaBR(f *e2eframework.Framework, clusterName, taskName string) error {
+	ns := f.Namespace.Name
+	pdAddr := fmt.Sprintf("%s-pd:2379", clusterName)
+
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "br-cli-" + taskName,
+			Namespace: ns,
+		},
+		Spec: v1.PodSpec{
+			RestartPolicy: v1.RestartPolicyNever,
+			Containers: []v1.Container{
+				{
+					Name:    "br",
+					Image:   fmt.Sprintf("pingcap/br:%s", utilimage.TiDBLatest),
+					Command: []string{"/br", "log", "stop", "--task-name=" + taskName, "--pd=" + pdAddr},
+				},
+			},
+		},
+	}
+
+	// Create the pod
+	_, err := f.ClientSet.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to create BR CLI pod: %v", err)
+	}
+
+	// Wait for pod to complete
+	err = wait.PollImmediate(time.Second*2, time.Minute*5, func() (bool, error) {
+		p, err := f.ClientSet.CoreV1().Pods(ns).Get(context.TODO(), pod.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		if p.Status.Phase == v1.PodSucceeded {
+			return true, nil
+		}
+		if p.Status.Phase == v1.PodFailed {
+			return false, fmt.Errorf("BR CLI pod failed")
+		}
+		return false, nil
+	})
+
+	// Clean up pod regardless of success/failure
+	deleteErr := f.ClientSet.CoreV1().Pods(ns).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
+	if deleteErr != nil {
+		// Log but don't fail on cleanup error
+		fmt.Printf("Warning: failed to delete BR CLI pod: %v\n", deleteErr)
+	}
+
+	return err
+}

tests/e2e/br/framework/br/wait.go

@@ -181,6 +181,40 @@ func WaitForBackupFailed(c versioned.Interface, ns, name string, timeout time.Du
 	return nil
 }
 
+// WaitForLogBackupFailedWithReason will poll and wait until timeout or log backup fails with expected reason
+func WaitForLogBackupFailedWithReason(c versioned.Interface, ns, name, expectedReason string, timeout time.Duration) error {
+	if err := wait.PollImmediate(poll, timeout, func() (bool, error) {
+		b, err := c.PingcapV1alpha1().Backups(ns).Get(context.TODO(), name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+
+		// Check log backup subcommand status for failure
+		if b.Spec.Mode == v1alpha1.BackupModeLog {
+			if v1alpha1.IsLogBackupSubCommandOntheCondition(b, v1alpha1.BackupFailed) {
+				reason, _ := v1alpha1.GetLogSubcommandConditionInfo(b)
+				if reason == expectedReason {
+					return true, nil
+				}
+				return false, fmt.Errorf("log backup failed with unexpected reason: %s, expected: %s", reason, expectedReason)
+			}
+		}
+
+		// Also check top-level conditions
+		for _, cond := range b.Status.Conditions {
+			if cond.Type == v1alpha1.BackupFailed && cond.Status == corev1.ConditionTrue {
+				if cond.Reason == expectedReason {
+					return true, nil
+				}
+			}
+		}
+		return false, nil
+	}); err != nil {
+		return fmt.Errorf("can't wait for log backup failed with reason %s: %v", expectedReason, err)
+	}
+	return nil
+}
+
 // WaitForRestoreComplete will poll and wait until timeout or restore complete condition is true
 func WaitForRestoreComplete(c versioned.Interface, ns, name string, timeout time.Duration) error {
 	if err := wait.PollImmediate(poll, timeout, func() (bool, error) {