Update pattern of handling minor PostgreSQL version upgrades

This modifies how PostgreSQL minor version upgrades are applied across a cluster to help
minimize downtime. The Operator will have each replica in a cluster updated sequentially to the requested minor version before attempting to update the primary.

When `autofail` is enabled, the cluster will promote one of the replicas, which at that point
will already be upgraded to the updated version of PostgreSQL.

Author: flamingdumpster

apis/cr/v1/common.go

@@ -58,7 +58,11 @@ type PgContainerResources struct {
 	LimitsCPU      string `json:"limitscpu"`
 }
 
+// CompletedStatus - 
 const CompletedStatus = "completed"
+// InProgressStatus - 
+const InProgressStatus = "in progress"
+// SubmittedStatus - 
 const SubmittedStatus = "submitted"
 
 // JobCompletedStatus ....

apiserver/failoverservice/failoverimpl.go

@@ -261,10 +261,10 @@ func preferredNode(nodes []string, targetNode string) bool {
 
 func checkAutofail(cluster *crv1.Pgcluster) error {
 	var err error
-	labels := cluster.ObjectMeta.Labels
-	failLabel := labels[config.LABEL_AUTOFAIL]
-	if failLabel == "true" {
+	
+	if util.IsAutofailEnabled(cluster) {
 		return errors.New("autofail flag is set to true, manual failover requires autofail to be set to false, use pgo update to disable autofail.")
 	}
+	
 	return err
 }

apiserver/upgradeservice/upgradeimpl.go

@@ -78,7 +78,7 @@ func CreateUpgrade(request *msgs.CreateUpgradeRequest, ns string) msgs.CreateUpg
 		spec.Status = "requested"
 		spec.Parameters = make(map[string]string)
 		spec.Parameters[config.LABEL_PG_CLUSTER] = clusterName
-		spec.Name = clusterName + "-minor-upgrade"
+		spec.Name = clusterName + "-" + config.LABEL_MINOR_UPGRADE
 		spec.Namespace = ns
 		labels := make(map[string]string)
 		labels[config.LABEL_PG_CLUSTER] = clusterName
@@ -104,7 +104,7 @@ func CreateUpgrade(request *msgs.CreateUpgradeRequest, ns string) msgs.CreateUpg
 			}
 		}
 
-		//validate the cluster name
+		//validate the cluster name and ensure autofail is turned off for each cluster.
 		cl := crv1.Pgcluster{}
 		found, err = kubeapi.Getpgcluster(apiserver.RESTClient,
 			&cl, clusterName, ns)
@@ -113,7 +113,7 @@ func CreateUpgrade(request *msgs.CreateUpgradeRequest, ns string) msgs.CreateUpg
 			response.Status.Msg = clusterName + " is not a valid pgcluster"
 			return response
 		}
-
+		
 		//figure out what version we are upgrading to
 		imageToUpgradeTo := apiserver.Pgo.Cluster.CCPImageTag
 		if request.CCPImageTag != "" {

config/labels.go

@@ -60,6 +60,13 @@ const LABEL_DELETE_BACKUPS = "delete-backups"
 const LABEL_IS_REPLICA = "is-replica"
 const LABEL_IS_BACKUP = "is-backup"
 
+const LABEL_MINOR_UPGRADE = "minor-upgrade"
+const LABEL_UPGRADE_IN_PROGRESS = "upgrade-in-progress"
+const LABEL_UPGRADE_COMPLETED = "upgrade-complete"
+const LABEL_UPGRADE_REPLICA = "upgrade-replicas"
+const LABEL_UPGRADE_PRIMARY = "upgrade-primary"
+const LABEL_UPGRADE_BACKREST = "upgrade-backrest"
+
 const LABEL_BACKREST = "pgo-backrest"
 const LABEL_BACKREST_JOB = "pgo-backrest-job"
 const LABEL_BACKREST_RESTORE = "pgo-backrest-restore"

controller/podcontroller.go

@@ -115,6 +115,16 @@ func (c *PodController) onUpdate(oldObj, newObj interface{}) {
 		return
 	}
 
+	// check here if cluster has an upgrade in progress flag set. 
+	clusterInMinorUpgrade := pgcluster.Labels[config.LABEL_MINOR_UPGRADE] == config.LABEL_UPGRADE_IN_PROGRESS 
+	// log.Debugf("Cluster: %s Minor Upgrade: %s ", clusterName, clusterInMinorUpgrade)
+
+	// have a pod coming back up from upgrade and is ready - time to kick off the next pod.
+	if clusterInMinorUpgrade && isUpgradedPostgresPod(newpod, oldpod) {
+		upgradeTaskName := clusterName + "-" + config.LABEL_MINOR_UPGRADE
+		clusteroperator.ProcessNextUpgradeItem(c.PodClientset, c.PodClient, clusterName, upgradeTaskName, newpod.ObjectMeta.Namespace)
+	}
+
 	//handle the case when a pg database pod is updated
 	if isPostgresPod(newpod) {
 		//only check the status of primary pods
@@ -145,7 +155,8 @@ func (c *PodController) checkReadyStatus(oldpod, newpod *apiv1.Pod, cluster *crv
 		log.Debug("the pod was updated and the service names were changed in this pod update, not going to check the ReadyStatus")
 		return
 	}
-	//handle the case of a database pod going to Ready that has
+
+	//handle the case of a database pod going to Not Ready that has
 	//autofail enabled
 	autofailEnabled := c.checkAutofailLabel(newpod, newpod.ObjectMeta.Namespace)
 	clusterName := newpod.ObjectMeta.Labels[config.LABEL_PG_CLUSTER]
@@ -304,10 +315,6 @@ func isPostgresPod(newpod *apiv1.Pod) bool {
 		log.Debugf("postgres-operator-pod found [%s]", newpod.Name)
 		return false
 	}
-	if newpod.ObjectMeta.Labels[config.LABEL_PGO_BACKREST_REPO] == "true" {
-		log.Debugf("pgo-backrest-repo pod found [%s]", newpod.Name)
-		return false
-	}
 	if newpod.ObjectMeta.Labels[config.LABEL_PGPOOL_POD] == "true" {
 		log.Debugf("pgpool pod found [%s]", newpod.Name)
 		return false
@@ -318,6 +325,75 @@ func isPostgresPod(newpod *apiv1.Pod) bool {
 	}
 	return true
 }
+// isUpgradedPostgresPod - determines if the pod is one that could be getting a minor upgrade
+// differs from above check in that the backrest repo pod is upgradeable. 
+func isUpgradedPostgresPod(newpod *apiv1.Pod, oldPod *apiv1.Pod) bool {
+
+
+	clusterName := newpod.ObjectMeta.Labels[config.LABEL_PG_CLUSTER]
+	replicaServiceName := clusterName + "-replica"
+
+	var podIsReady bool
+	for _, v := range newpod.Status.ContainerStatuses {
+		if v.Name == "database" {
+			podIsReady = v.Ready
+		}
+	}
+
+	var oldPodStatus bool 
+	for _, v := range oldPod.Status.ContainerStatuses {
+		if v.Name == "database" {
+			oldPodStatus = v.Ready
+		}
+	}
+
+	log.Debugf("[isUpgradedPostgesPod] oldstatus: %s newstatus: %s ", oldPodStatus, podIsReady)
+
+	// only care about pods that have changed from !ready to ready
+	if podIsReady && !oldPodStatus {
+
+		// eliminate anything we don't care about - it will be most things
+		if newpod.ObjectMeta.Labels[config.LABEL_JOB_NAME] != "" {
+			log.Debugf("job pod found [%s]", newpod.Name)
+			return false
+		}
+
+		if newpod.ObjectMeta.Labels[config.LABEL_NAME] == "postgres-operator" {
+			log.Debugf("postgres-operator-pod found [%s]", newpod.Name)
+			return false
+		}
+		if newpod.ObjectMeta.Labels[config.LABEL_PGPOOL_POD] == "true" {
+			log.Debugf("pgpool pod found [%s]", newpod.Name)
+			return false
+		}
+		if newpod.ObjectMeta.Labels[config.LABEL_PGBOUNCER] == "true" {
+			log.Debugf("pgbouncer pod found [%s]", newpod.Name)
+			return false
+		}
+
+		// look for specific pods that could have just gone through upgrade
+
+		if newpod.ObjectMeta.Labels[config.LABEL_PGO_BACKREST_REPO] == "true"  {
+			log.Debugf("Minor Upgrade: upgraded pgo-backrest-repo found %s", newpod.Name)
+			return true
+		}
+
+		// primary identified by service-name being same as cluster name
+		if newpod.ObjectMeta.Labels[config.LABEL_SERVICE_NAME] == clusterName {
+			log.Debugf("Minor Upgrade: upgraded primary found %s", newpod.Name)
+			return true
+		}
+
+		if newpod.ObjectMeta.Labels[config.LABEL_SERVICE_NAME] == replicaServiceName {
+			log.Debugf("Minor Upgrade: upgraded replica found %s", newpod.Name)
+			return true
+		}
+
+		// This indicates there is a pod we didn't account for - shouldn't be the case
+		log.Debugf(" **** Minor Upgrade: unexpected isUpgraded pod found: [%s] ****", newpod.Name)
+	}
+	return false
+}
 
 func publishClusterComplete(clusterName, namespace string, cluster *crv1.Pgcluster) error {
 	//capture the cluster creation event

hugo/content/Upgrade/_index.md

@@ -17,3 +17,23 @@ This section of the documentation shows specific steps required to upgrade diffe
 [Upgrade Postgres Operator from 3.5 to 4.1] ( {{< relref "upgrade/upgrade35to4.md" >}})
 
 [Upgrade Postgres Operator from 4.0.1 to 4.1.0] ( {{< relref "upgrade/upgrade40to41.md" >}})
+
+## Upgrading A Postgres Cluster
+
+Using the operator, it is possible to upgrade a postgres cluster in place.  When a pgo upgrade command is issued, and a --CCPImageTag is specified, the operator will upgrade each replica and the primary to the new CCPImageTag version. It is important to note that the postgres version of the new container should be compatible with the current running version. There is currently no version check done to ensure compatibility.
+
+The upgrade is accomplished by updating the CCPImageTag version in the deployment, which causes the old pod to be terminated and a new pod created with the updated deployment specification.
+
+When the upgrade starts, each replica is upgraded seqentially, waiting for the previous replica to go ready before updating the next. After the replicas complete, the primary is then upgraded to the new image. The upgrade process respects the _autofail_ and the _AutofailReplaceReplica_ settings as provided in the pgo.yaml or as a command line flag, if applicable.
+
+When the cluster is not in _autofail_ mode, the deployments simply create a new pod when updated, terminating the old one. When autofail is enabled and the primary deployment is updated, the cluster behaves as though the primary had failed and begins the failover process. See _Automatic Failover_ in the _Overview_ section for more details on this and replica replacement.
+
+At this time, the backrest-repo container is not upgraded during this upgrade as it is part of the postgres operator release and is updated with the operator.
+
+## Minor Upgrade Example
+
+In this example, we are upgrading a cluster from PostgreSQL 11.4 to 11.5 using the `crunchy-postgres:centos7-11.5-2.4.2` container:
+
+`pgo upgrade mycluster --ccp-image-tag=centos7-11.5-2.4.2`
+
+For more information, please see the `pgo upgrade` documentation [here.] ( {{< relref "operatorcli/cli/pgo_upgrade.md" >}})

operator/cluster/failoverlogic.go

@@ -170,14 +170,14 @@ func deletePrimary(clientset *kubernetes.Clientset, namespace, clusterName, pgou
 	//the primary will be the one with a pod that has a label
 	//that looks like service-name=clustername and is not a backrest job
 	selector := config.LABEL_SERVICE_NAME + "=" + clusterName + "," + config.LABEL_BACKREST_RESTORE + "!=true," + config.LABEL_BACKREST_JOB + "!=true"
-	pods, err := kubeapi.GetPods(clientset, selector, namespace)
-	if len(pods.Items) == 0 {
-		log.Errorf("no primary pod found when trying to delete primary %s", selector)
-		return errors.New("could not find primary pod")
-	}
-	if len(pods.Items) > 1 {
-		log.Errorf("more than 1 primary pod found when trying to delete primary %s", selector)
-		return errors.New("more than 1 primary pod found in delete primary logic")
+
+	// wait for single primary pod to exist.
+	pods, success := waitForSinglePrimary(clientset, selector, namespace)
+
+
+	if !success {
+		log.Errorf("Received false while waiting for single primary, count: ", len(pods.Items))
+		return errors.New("Couldn't isolate single primary pod")
 	}
 
 	//update the label to 'fenced' on the pod to fence off traffic from
@@ -192,7 +192,7 @@ func deletePrimary(clientset *kubernetes.Clientset, namespace, clusterName, pgou
 
 	//delete the deployment with pg-cluster=clusterName,primary=true
 	log.Debugf("deleting deployment %s", deploymentToDelete)
-	err = kubeapi.DeleteDeployment(clientset, deploymentToDelete, namespace)
+	err := kubeapi.DeleteDeployment(clientset, deploymentToDelete, namespace)
 
 	err = waitForDelete(deploymentToDelete, pod.Name, clientset, namespace)
 
@@ -232,14 +232,48 @@ func waitForDelete(deploymentToDelete, podName string, clientset *kubernetes.Cli
 			log.Error("error getting pod when evaluating old primary in failover %s %s", deploymentToDelete, podName)
 			return err
 		}
-		log.Debugf("waitinf for %s to delete", pod.Name)
+		log.Debugf("waiting for %s to delete", pod.Name)
 		time.Sleep(time.Second * time.Duration(9))
 	}
 
 	return errors.New(fmt.Sprintf("timeout waiting for %s %s to delete", deploymentToDelete, podName))
 
 }
 
+// waitForSinglePrimary - during failover, there can exist the possibility that while one pod is in the process of
+// terminating, the Deployment will be spinning up another pod - both will appear to be a primary even though the
+// terminating pod will not be accessible via the service. This method gets the primary and if both exists, it waits to
+// give the terminating pod a chance to complete. If a single primary is never isolated, it returns false with the count
+// of primaries found when it gave up. The number of tries and duration can be increased if needed - max wait time is
+// tries * duration.
+func waitForSinglePrimary(clientset *kubernetes.Clientset, selector, namespace string)  (*v1.PodList, bool) {
+
+	var tries = 5
+	var duration = 2 // seconds
+	var pods *v1.PodList
+
+	for i :=0; i < tries; i ++ {
+		
+		pods, _ = kubeapi.GetPods(clientset, selector, namespace)
+
+		if len(pods.Items) > 1 {
+			log.Errorf("more than 1 primary pod found when looking for primary %s", selector)
+			log.Debug("Waiting in case a pod is terminating...")
+		// return errors.New("more than 1 primary pod found in delete primary logic")
+		time.Sleep(time.Second * time.Duration(duration))
+		} else if len(pods.Items) == 0 { 
+			log.Errorf("No pods found for primary deployment")
+			return pods, false
+		} else {
+			log.Debug("Found single pod for primary deployment")
+			return pods, true
+		}
+	}	
+	
+	return pods, false
+}
+
+
 func publishPromoteEvent(identifier, namespace, username, clusterName, target string) {
 	topics := make([]string, 1)
 	topics[0] = events.EventTopicCluster