fix: imperative restart should rollout the primary pod (cloudnative-pg#7122)

This patch updates the operator to create a new Pod for the primary
instance when an imperative restart is requested.

Previously, if `primaryUpdateMethod` was set to `restart`, the operator
would recreate the replica Pods but only perform an in-place restart of
the primary Pod.

This approach could lead to inconsistencies between the definitions of
the primary and the replica Pods, as only the replicas were fully
recreated.

Closes cloudnative-pg#7120

Signed-off-by: Leonardo Cecchi <leonardo.cecchi@enterprisedb.com>
Signed-off-by: Armando Ruocco <armando.ruocco@enterprisedb.com>
Co-authored-by: Armando Ruocco <armando.ruocco@enterprisedb.com>

Author: leonardoce

internal/controller/cluster_upgrade.go

@@ -352,11 +352,11 @@ func isPodNeedingRollout(
 	}
 
 	checkers := map[string]rolloutChecker{
-		"pod has missing PVCs":                 checkHasMissingPVCs,
-		"pod has PVC requiring resizing":       checkHasResizingPVC,
-		"pod projected volume is outdated":     checkProjectedVolumeIsOutdated,
-		"pod image is outdated":                checkPodImageIsOutdated,
-		"cluster has newer restart annotation": checkClusterHasNewerRestartAnnotation,
+		"pod has missing PVCs":                     checkHasMissingPVCs,
+		"pod has PVC requiring resizing":           checkHasResizingPVC,
+		"pod projected volume is outdated":         checkProjectedVolumeIsOutdated,
+		"pod image is outdated":                    checkPodImageIsOutdated,
+		"cluster has different restart annotation": checkClusterHasDifferentRestartAnnotation,
 	}
 
 	podRollout := applyCheckers(checkers)
@@ -544,19 +544,15 @@ func checkHasMissingPVCs(pod *corev1.Pod, cluster *apiv1.Cluster) (rollout, erro
 	return rollout{}, nil
 }
 
-func checkClusterHasNewerRestartAnnotation(pod *corev1.Pod, cluster *apiv1.Cluster) (rollout, error) {
-	// check if pod needs to be restarted because of some config requiring it
-	// or if the cluster have been explicitly restarted
-	// If the cluster has been restarted and we are working with a Pod
-	// which has not been restarted yet, or restarted at a different
-	// time, let's restart it.
+func checkClusterHasDifferentRestartAnnotation(pod *corev1.Pod, cluster *apiv1.Cluster) (rollout, error) {
+	// If the pod restart value doesn't match with the one contained in the cluster, restart the pod.
 	if clusterRestart, ok := cluster.Annotations[utils.ClusterRestartAnnotationName]; ok {
 		podRestart := pod.Annotations[utils.ClusterRestartAnnotationName]
 		if clusterRestart != podRestart {
 			return rollout{
 				required:     true,
 				reason:       "cluster has been explicitly restarted via annotation",
-				canBeInPlace: true,
+				canBeInPlace: false,
 			}, nil
 		}
 	}

internal/controller/cluster_upgrade_test.go

@@ -104,7 +104,7 @@ var _ = Describe("Pod upgrade", Ordered, func() {
 		rollout := isInstanceNeedingRollout(ctx, status, &clusterRestart)
 		Expect(rollout.required).To(BeTrue())
 		Expect(rollout.reason).To(Equal("cluster has been explicitly restarted via annotation"))
-		Expect(rollout.canBeInPlace).To(BeTrue())
+		Expect(rollout.canBeInPlace).To(BeFalse())
 		Expect(rollout.needsChangeOperandImage).To(BeFalse())
 		Expect(rollout.needsChangeOperatorImage).To(BeFalse())