K8SPSMDB 1472 fixes on sts updates and df execution flows when pvc resizing

pkg/controller/perconaservermongodb/metrics_client.go

@@ -5,10 +5,14 @@ import (
 	"context"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/retry"
 
+	"github.com/percona/percona-server-mongodb-operator/pkg/naming"
 	"github.com/percona/percona-server-mongodb-operator/pkg/psmdb/config"
 )
 
@@ -25,6 +29,16 @@ func (r *ReconcilePerconaServerMongoDB) getPVCUsageFromMetrics(
 	pod *corev1.Pod,
 	pvcName string,
 ) (*PVCUsage, error) {
+	if pod == nil {
+		return nil, errors.New("pod is nil")
+	}
+
+	backoff := wait.Backoff{
+		Steps:    5,
+		Duration: 5 * time.Second,
+		Factor:   2.0,
+	}
+
 	// Execute df command in the mongod container to get disk usage
 	// df -B1 /data/db outputs in bytes
 	// Example output:
@@ -33,9 +47,18 @@ func (r *ReconcilePerconaServerMongoDB) getPVCUsageFromMetrics(
 	var stdout, stderr bytes.Buffer
 	command := []string{"df", "-B1", config.MongodContainerDataDir}
 
-	err := r.clientcmd.Exec(ctx, pod, "mongod", command, nil, &stdout, &stderr, false)
+	err := retry.OnError(backoff, func(err error) bool { return true }, func() error {
+		stdout.Reset()
+		stderr.Reset()
+
+		err := r.clientcmd.Exec(ctx, pod, naming.ComponentMongod, command, nil, &stdout, &stderr, false)
+		if err != nil {
+			return errors.Wrapf(err, "failed to execute df in pod %s: %s", pod.Name, stderr.String())
+		}
+		return nil
+	})
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to execute df in pod %s: %s", pod.Name, stderr.String())
+		return nil, errors.Wrap(err, "wait for df execution")
 	}
 
 	lines := strings.Split(strings.TrimSpace(stdout.String()), "\n")

pkg/controller/perconaservermongodb/metrics_client_test.go

@@ -145,6 +145,24 @@ func TestGetPVCUsageFromMetrics(t *testing.T) {
 					Name:      "test-pod-0",
 					Namespace: "test-namespace",
 				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: "mongod",
+						},
+					},
+				},
+				Status: corev1.PodStatus{
+					Phase: corev1.PodRunning,
+					ContainerStatuses: []corev1.ContainerStatus{
+						{
+							Name: "mongod",
+							State: corev1.ContainerState{
+								Running: &corev1.ContainerStateRunning{},
+							},
+						},
+					},
+				},
 			}
 
 			result, err := r.getPVCUsageFromMetrics(ctx, pod, tt.pvcName)

pkg/controller/perconaservermongodb/statefulset.go

@@ -54,15 +54,14 @@ func (r *ReconcilePerconaServerMongoDB) reconcileStatefulSet(ctx context.Context
 		return sfs, nil
 	}
 
-	if err := r.reconcilePVCs(ctx, cr, sfs, ls, volumeSpec); err != nil {
-		return nil, errors.Wrapf(err, "reconcile PVCs for %s", sfs.Name)
-	}
-
-	// (non-blocking)
 	if err := r.reconcileStorageAutoscaling(ctx, cr, sfs, volumeSpec, ls); err != nil {
 		log.Error(err, "failed to reconcile storage autoscaling", "statefulset", sfs.Name)
 	}
 
+	if err := r.reconcilePVCs(ctx, cr, sfs, ls, volumeSpec); err != nil {
+		return nil, errors.Wrapf(err, "reconcile PVCs for %s", sfs.Name)
+	}
+
 	if _, ok := sfs.Annotations[api.AnnotationPVCResizeInProgress]; ok {
 		log.V(1).Info("PVC resize in progress, skipping reconciliation of statefulset", "name", sfs.Name)
 		return sfs, nil

pkg/controller/perconaservermongodb/volume_autoscaling.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/percona/percona-server-mongodb-operator/pkg/naming"
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -102,8 +103,8 @@ func (r *ReconcilePerconaServerMongoDB) checkAndResizePVC(
 ) error {
 	log := logf.FromContext(ctx).WithName("StorageAutoscaling").WithValues("pvc", pvc.Name)
 
-	if pod.Status.Phase != corev1.PodRunning {
-		log.V(1).Info("skipping PVC check: pod not running", "phase", pod.Status.Phase)
+	if !isContainerAndPodRunning(*pod, naming.ComponentMongod) {
+		log.V(1).Info("skipping PVC metrics check: container and pod not running", "phase", pod.Status.Phase)
 		return nil
 	}
 
@@ -140,9 +141,6 @@ func (r *ReconcilePerconaServerMongoDB) shouldTriggerResize(
 	config := cr.Spec.StorageAutoscaling()
 
 	if usage.UsagePercent < config.TriggerThresholdPercent {
-		log.V(1).Info("usage below threshold",
-			"usage", usage.UsagePercent,
-			"threshold", config.TriggerThresholdPercent)
 		return false
 	}
 
@@ -201,12 +199,11 @@ func (r *ReconcilePerconaServerMongoDB) triggerResize(
 ) error {
 	log := logf.FromContext(ctx).WithName("StorageAutoscaling").WithValues("pvc", pvc.Name)
 
-	patch := client.MergeFrom(cr.DeepCopy())
+	orig := cr.DeepCopy()
 
-	// We are modifying cr directly through the pointer. So the original cr object does get the storage size updated.
 	volumeSpec.PersistentVolumeClaim.Resources.Requests[corev1.ResourceStorage] = newSize
 
-	if err := r.client.Patch(ctx, cr.DeepCopy(), patch); err != nil {
+	if err := r.client.Patch(ctx, cr.DeepCopy(), client.MergeFrom(orig)); err != nil {
 		return errors.Wrap(err, "patch CR with new storage size")
 	}