K8SPSMDB-971: Ensure restore status is updated (#1311)

Co-authored-by: Viacheslav Sarzhan <[email protected]>

Author: egegunes

pkg/controller/perconaservermongodbrestore/perconaservermongodbrestore_controller.go

@@ -13,6 +13,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -236,7 +237,14 @@ func (r *ReconcilePerconaServerMongoDBRestore) getBackup(ctx context.Context, cr
 }
 
 func (r *ReconcilePerconaServerMongoDBRestore) updateStatus(ctx context.Context, cr *psmdbv1.PerconaServerMongoDBRestore) error {
-	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+	var backoff = wait.Backoff{
+		Steps:    5,
+		Duration: 500 * time.Millisecond,
+		Factor:   5.0,
+		Jitter:   0.1,
+	}
+
+	err := retry.OnError(backoff, func(error) bool { return true }, func() error {
 		c := &psmdbv1.PerconaServerMongoDBRestore{}
 
 		err := r.client.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: cr.Namespace}, c)
@@ -246,7 +254,23 @@ func (r *ReconcilePerconaServerMongoDBRestore) updateStatus(ctx context.Context,
 
 		c.Status = cr.Status
 
-		return r.client.Status().Update(ctx, c)
+		err = r.client.Status().Update(ctx, c)
+		if err != nil {
+			return err
+		}
+
+		// ensure status is updated
+		c = &psmdbv1.PerconaServerMongoDBRestore{}
+		err = r.client.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: cr.Namespace}, c)
+		if err != nil {
+			return err
+		}
+
+		if c.Status.State != cr.Status.State {
+			return errors.New("status not updated")
+		}
+
+		return nil
 	})
 
 	if k8serrors.IsNotFound(err) {

pkg/controller/perconaservermongodbrestore/physical.go

@@ -276,7 +276,7 @@ func (r *ReconcilePerconaServerMongoDBRestore) reconcilePhysicalRestore(ctx cont
 		log.V(1).Info("Check restore status", "command", command, "pod", pod.Name)
 
 		if err := r.clientcmd.Exec(ctx, &pod, "mongod", command, nil, stdoutBuf, stderrBuf, false); err != nil {
-			return errors.Wrap(err, "describe restore")
+			return errors.Wrapf(err, "describe restore stderr: %s stdout: %s", stderrBuf.String(), stdoutBuf.String())
 		}
 
 		return nil