Orchestrate Repo & Primary DB Deployments

This commit updates cluster initialization logic to ensure the pgBackRest
repository is online and ready prior to initializing the primary database
container.  This ensures that the pgBackRest repository host is always
available and online as needed to fully initialize the database cluster,
e.g. to create the stanza, initial backup, replicas, etc.  This is
especially beneficial in environments where latency can lead to delays in
pulling and initializing the pgBackRest repository image, therefore
leading to database initialization (and therefore attempts to create the
stanza, etc.) prior to the repo coming online.

To ensure the repo is always initialized prior to initializing the database,
the primary deployment is now initially created with 0 replicas.  Then,
once the Operator detects that the repo host is ready (specifically via a
new repo init handler in the Pod controller), the primary database is
scaled to 1.  From here cluster creation then proceeds in the same manner
as before.

It should be noted that with this change the repo host also initially started
with 0 replicas, but then is immediately scaled up.  This is to ensure
that all Deployments and supporting resources (services, configMaps, PVCs,
etc.) have been created and are ready to go prior to creating any Pods. This
prevent potential race conditions, e.g. the repo host being created and
becoming ready prior the creation of the primary DB deployment.  With this
solution we can be sure the primary deployment always exists by the time
the repo host becomes ready and attempts to scale it to continue initialization
of the cluster.

Author: andrewlecuyer

apiserver/root.go

@@ -366,6 +366,7 @@ func UserIsPermittedInNamespace(username, requestedNS string) (iAccess, uAccess
 		requestedNS); err != nil && !errors.Is(err, ns.ErrNamespaceNotWatched) {
 		return
 	}
+	iAccess = true
 
 	//get the pgouser Secret for this username
 	userSecretName := "pgouser-" + username

conf/postgres-operator/pgo-backrest-repo-template.json

@@ -12,7 +12,7 @@
         }
     },
     "spec": {
-        "replicas": 1,
+        "replicas": {{.Replicas}},
         "selector": {
             "matchLabels": {
                     "name": "{{.Name}}",

config/labels.go

@@ -169,5 +169,5 @@ const LABEL_PGHA_SCOPE = "crunchy-pgha-scope"
 const LABEL_PGHA_CONFIGMAP = "pgha-config"
 const LABEL_PGHA_BACKUP_TYPE = "pgha-backup-type"
 const LABEL_PGHA_ROLE = "role"
-const LABEL_PGHA_ROLE_PRIMARY = "primary"
+const LABEL_PGHA_ROLE_PRIMARY = "master"
 const LABEL_PGHA_ROLE_REPLICA = "replica"

controller/pgcluster/pgclustercontroller.go

@@ -137,8 +137,25 @@ func (c *Controller) processNextItem() bool {
 
 	log.Debugf("pgcluster added: %s", cluster.ObjectMeta.Name)
 
+	// AddClusterBase creates all deployments for the cluster (in addition to various other supporting
+	// resources such as services, configMaps, secrets, etc.), but leaves them scaled to 0.  This
+	// ensures all deployments exist as needed to properly orchestrate initialization of the
+	// cluster, e.g. we need to ensure the primary DB deployment resource has been created before
+	// bringing the repo deployment online, since that in turn will bring the primary DB online.
 	clusteroperator.AddClusterBase(c.PgclusterClientset, c.PgclusterClient, &cluster, cluster.ObjectMeta.Namespace)
 
+	// Now scale the repo deployment only to ensure it is initialized prior to the primary DB.
+	// Once the repo is ready, the primary datbase deployment will then also be scaled to 1.
+	clusterInfo, err := clusteroperator.ScaleClusterDeployments(c.PgclusterClientset,
+		cluster, 1, false, false, true, false)
+	if err != nil {
+		log.Error(err)
+		c.Queue.AddRateLimited(key)
+	}
+
+	log.Debugf("Scaled pgBackRest repo deployment %s to 1 to proceed with initializing "+
+		"cluster %s", clusterInfo.PrimaryDeployment, cluster.Name)
+
 	return true
 }
 

controller/pod/inithandler.go

@@ -39,7 +39,18 @@ import (
 // primary PG pod for a new or restored PG cluster reaches a ready status
 func (c *Controller) handleClusterInit(newPod *apiv1.Pod, cluster *crv1.Pgcluster) error {
 
-	clusterName := cluster.Name
+	clusterName := cluster.GetName()
+
+	// first check to see if the update is a repo pod.  If so, then call repo init handler and
+	// return since the other handlers are only applicable to PG pods
+	if isBackRestRepoPod(newPod) {
+		log.Debugf("Pod Controller: calling pgBackRest repo init for cluster %s", clusterName)
+		if err := c.handleBackRestRepoInit(cluster); err != nil {
+			log.Error(err)
+			return err
+		}
+		return nil
+	}
 
 	// handle common tasks for initializing a cluster, whether due to bootstap or reinitialization
 	// following a restore, or if a regular or standby cluster
@@ -69,6 +80,23 @@ func (c *Controller) handleClusterInit(newPod *apiv1.Pod, cluster *crv1.Pgcluste
 	}
 }
 
+// handleBackRestRepoInit handles cluster initialization tasks that must be executed once
+// as a result of an update to a cluster's pgBackRest repostiroy pod
+func (c *Controller) handleBackRestRepoInit(cluster *crv1.Pgcluster) error {
+
+	clusterInfo, err := clusteroperator.ScaleClusterDeployments(c.PodClientset, *cluster, 1,
+		true, false, false, false)
+	if err != nil {
+		log.Error(err)
+		return err
+	}
+
+	log.Debugf("Pod Controller: scaled primary deployment %s to 1 to proceed with initializing "+
+		"cluster %s", clusterInfo.PrimaryDeployment, cluster.Name)
+
+	return nil
+}
+
 // handleCommonInit is resposible for handling common initilization tasks for a PG cluster
 // regardless of the specific type of cluster (e.g. regualar or standby) or the reason the
 // cluster is being initialized (initial bootstrap or restore)
@@ -169,7 +197,7 @@ func (c *Controller) handleStandbyInit(cluster *crv1.Pgcluster) error {
 	//
 
 	// now scale any replicas deployments to 1
-	clusteroperator.ScaleClusterDeployments(c.PodClientset, *cluster, 1, false, true, false)
+	clusteroperator.ScaleClusterDeployments(c.PodClientset, *cluster, 1, false, true, false, false)
 
 	// Proceed with stanza-creation of this is not a standby cluster, or if its
 	// a standby cluster that does not have "s3" storage only enabled.

controller/pod/podcontroller.go

@@ -84,14 +84,33 @@ func (c *Controller) onUpdate(oldObj, newObj interface{}) {
 	// Lookup the pgcluster CR for PG cluster associated with this Pod.  Since a 'pg-cluster'
 	// label was found on updated Pod, this lookup should always succeed.
 	clusterName := newPodLabels[config.LABEL_PG_CLUSTER]
+	namespace := newPod.ObjectMeta.Namespace
 	cluster := crv1.Pgcluster{}
-	_, err := kubeapi.Getpgcluster(c.PodClient, &cluster, clusterName,
-		newPod.ObjectMeta.Namespace)
+	_, err := kubeapi.Getpgcluster(c.PodClient, &cluster, clusterName, namespace)
 	if err != nil {
 		log.Error(err.Error())
 		return
 	}
 
+	// For the following upgrade and cluster initialization scenarios we only care about updates
+	// where the database container within the pod is becoming ready.  We can therefore return
+	// at this point if this condition is false.
+	if cluster.Status.State != crv1.PgclusterStateInitialized &&
+		(isDBContainerBecomingReady(oldPod, newPod) ||
+			isBackRestRepoBecomingReady(oldPod, newPod)) {
+		if err := c.handleClusterInit(newPod, &cluster); err != nil {
+			log.Error(err)
+			return
+		}
+		return
+	}
+
+	// the handlers called below are only applicable to PG pods when the cluster is
+	// in an initialized status
+	if cluster.Status.State != crv1.PgclusterStateInitialized || !isPostgresPod(newPod) {
+		return
+	}
+
 	// Handle the "role" label change from "replica" to "master" following a failover.  This
 	// logic is only triggered when the cluster has already been initialized, which implies
 	// a failover or switchover has occurred.
@@ -104,8 +123,7 @@ func (c *Controller) onUpdate(oldObj, newObj interface{}) {
 		}
 	}
 
-	if cluster.Status.State == crv1.PgclusterStateInitialized &&
-		isPromotedStandby(oldPod, newPod) {
+	if isPromotedStandby(oldPod, newPod) {
 		log.Debugf("Pod Controller: standby pod %s in namespace %s promoted, calling standby pod "+
 			"promotion handler", newPod.Name, newPod.Namespace)
 		if err := c.handleStandbyPromotion(newPod, cluster); err != nil {
@@ -114,32 +132,17 @@ func (c *Controller) onUpdate(oldObj, newObj interface{}) {
 		}
 	}
 
-	// For the following upgrade and cluster initialization scenarios we only care about updates
-	// where the database container within the pod is becoming ready.  We can therefore return
-	// at this point if this condition is false.
-	if !isDBContainerBecomingReady(oldPod, newPod) {
-		return
-	}
-
 	// First handle pod update as needed if the update was part of an ongoing upgrade
 	if cluster.Labels[config.LABEL_MINOR_UPGRADE] == config.LABEL_UPGRADE_IN_PROGRESS {
-		log.Debugf("Pod Controller: upgrade pod %s now ready, calling pod upgrade "+
-			"handler", newPod.Name, newPod.Namespace)
+		log.Debugf("Pod Controller: upgrade pod %s (namespace %s) now ready, calling pod upgrade "+
+			"handler", newPod.Name, namespace)
 		if err := c.handleUpgradePodUpdate(newPod, &cluster); err != nil {
 			log.Error(err)
 			return
 		}
 	}
 
-	// Handle postgresql pod updates as needed for cluster initialization
-	if cluster.Status.State != crv1.PgclusterStateInitialized && isPostgresPrimaryPod(newPod) {
-		log.Debugf("Pod Controller: pg pod %s now ready in an unintialized cluster, calling "+
-			"cluster init handler", newPod.Name, newPod.Namespace)
-		if err := c.handleClusterInit(newPod, &cluster); err != nil {
-			log.Error(err)
-			return
-		}
-	}
+	return
 }
 
 // onDelete is called when a pgcluster is deleted
@@ -166,25 +169,43 @@ func (c *Controller) AddPodEventHandler() {
 	log.Debugf("Pod Controller: added event handler to informer")
 }
 
-// isDBContainerBecomingReady checks to see if the Pod update shows that the Pod has
-// transitioned from an 'unready' status to a 'ready' status.
-func isDBContainerBecomingReady(oldPod, newPod *apiv1.Pod) bool {
-	if !isPostgresPod(newPod) {
+// isBackRestRepoBecomingReady checks to see if the Pod update shows that the BackRest
+// repo Pod has transitioned from an 'unready' status to a 'ready' status.
+func isBackRestRepoBecomingReady(oldPod, newPod *apiv1.Pod) bool {
+	if !isBackRestRepoPod(newPod) {
 		return false
 	}
-	var oldDatabaseStatus bool
-	// first see if the old version of the pod was not ready
+	return isContainerBecomingReady("database", oldPod, newPod)
+}
+
+// isBackRestRepoPod determines whether or not a pod is a pgBackRest repository Pod.  This is
+// determined by checking to see if the 'pgo-backrest-repo' label is present on the Pod (also,
+// this controller will only process pod with the 'vendor=crunchydata' label, so that label is
+// assumed to be present), specifically because this label will only be included on pgBackRest
+// repository Pods.
+func isBackRestRepoPod(newpod *apiv1.Pod) bool {
+
+	_, backrestRepoLabelExists := newpod.ObjectMeta.Labels[config.LABEL_PGO_BACKREST_REPO]
+
+	return backrestRepoLabelExists
+}
+
+// isContainerBecomingReady determines whether or not that container specified is moving
+// from an unready status to a ready status.
+func isContainerBecomingReady(containerName string, oldPod, newPod *apiv1.Pod) bool {
+	var oldContainerStatus bool
+	// first see if the old version of the container was not ready
 	for _, v := range oldPod.Status.ContainerStatuses {
-		if v.Name == "database" {
-			oldDatabaseStatus = v.Ready
+		if v.Name == containerName {
+			oldContainerStatus = v.Ready
 			break
 		}
 	}
-	// if the old version of the pod was not ready, now check if the
+	// if the old version of the container was not ready, now check if the
 	// new version is ready
-	if !oldDatabaseStatus {
+	if !oldContainerStatus {
 		for _, v := range newPod.Status.ContainerStatuses {
-			if v.Name == "database" {
+			if v.Name == containerName {
 				if v.Ready {
 					return true
 				}
@@ -194,19 +215,26 @@ func isDBContainerBecomingReady(oldPod, newPod *apiv1.Pod) bool {
 	return false
 }
 
-// isPostgresPrimaryPod determines whether or not the specific Pod provided is the primary database
-// Pod within a PG cluster.  This is done by checking to see if the "role" label for the Pod is set
-// to either "master", as set by Patroni to identify the current primary, or "promoted", as set by
-// Patroni when promoting a replica to be the new primary.
-func isPostgresPrimaryPod(newPod *apiv1.Pod) bool {
+// isDBContainerBecomingReady checks to see if the Pod update shows that the Pod has
+// transitioned from an 'unready' status to a 'ready' status.
+func isDBContainerBecomingReady(oldPod, newPod *apiv1.Pod) bool {
 	if !isPostgresPod(newPod) {
 		return false
 	}
-	if newPod.ObjectMeta.Labels[config.LABEL_PGHA_ROLE] == "master" ||
-		newPod.ObjectMeta.Labels[config.LABEL_PGHA_ROLE] == "promoted" {
-		return true
-	}
-	return false
+	return isContainerBecomingReady("database", oldPod, newPod)
+}
+
+// isPostgresPod determines whether or not a pod is a PostreSQL Pod, specifically either the
+// primary or a replica pod within a PG cluster.  This is determined by checking to see if the
+// 'pgo-pg-database' label is present on the Pod (also, this controller will only process pod with
+// the 'vendor=crunchydata' label, so that label is assumed to be present), specifically because
+// this label will only be included on primary and replica PostgreSQL database pods (and will be
+// present as soon as the deployment and pod is created).
+func isPostgresPod(newpod *apiv1.Pod) bool {
+
+	_, pgDatabaseLabelExists := newpod.ObjectMeta.Labels[config.LABEL_PG_DATABASE]
+
+	return pgDatabaseLabelExists
 }
 
 // isPromotedPostgresPod determines if the Pod update is the result of the promotion of the pod
@@ -225,11 +253,11 @@ func isPromotedPostgresPod(oldPod, newPod *apiv1.Pod) bool {
 	return false
 }
 
-// isPromotedPostgresPod determines if the Pod update is the result of the promotion of the pod
+// isPromotedStandby determines if the Pod update is the result of the promotion of the standby pod
 // from a replica to the primary within a PG cluster.  This is determined by comparing the 'role'
 // label from the old Pod to the 'role' label in the New pod, specifically to determine if the
-// label has changed from "promoted" to "master" (this is the label change that will be performed
-// by Patroni when promoting a pod).
+// label has changed from "standby_leader" to "master" (this is the label change that will be
+// performed by Patroni when promoting a pod).
 func isPromotedStandby(oldPod, newPod *apiv1.Pod) bool {
 	if !isPostgresPod(newPod) {
 		return false
@@ -243,16 +271,3 @@ func isPromotedStandby(oldPod, newPod *apiv1.Pod) bool {
 	}
 	return false
 }
-
-// isPostgresPod determines whether or not a pod is a PostreSQL Pod, specifically either the
-// primary or a replica pod within a PG cluster.  This is determined by checking to see if the
-// 'pgo-pg-database' label is present on the Pod (also, this controller will only process pod with
-// the 'vendor=crunchydata' label, so that label is assumed to be present), specifically because
-// this label will only be included on primary and replica PostgreSQL database pods (and will be
-// present as soon as the deployment and pod is created).
-func isPostgresPod(newpod *apiv1.Pod) bool {
-
-	_, pgDatabaseLabelExists := newpod.ObjectMeta.Labels[config.LABEL_PG_DATABASE]
-
-	return pgDatabaseLabelExists
-}

controller/pod/promotionhandler.go

@@ -85,7 +85,7 @@ func (c *Controller) handleStartupInit(cluster crv1.Pgcluster) error {
 	}
 
 	// now scale any replicas deployments to 1
-	clusteroperator.ScaleClusterDeployments(c.PodClientset, cluster, 1, false, true, false)
+	clusteroperator.ScaleClusterDeployments(c.PodClientset, cluster, 1, false, true, false, false)
 
 	return nil
 }

installers/ansible/roles/pgo-operator/files/pgo-configs/pgo-backrest-repo-template.json

@@ -12,7 +12,7 @@
         }
     },
     "spec": {
-        "replicas": 1,
+        "replicas": {{.Replicas}},
         "selector": {
             "matchLabels": {
                     "name": "{{.Name}}",

operator/backrest/repo.go

@@ -55,6 +55,7 @@ type RepoDeploymentTemplateFields struct {
 	PodAntiAffinity           string
 	PodAntiAffinityLabelName  string
 	PodAntiAffinityLabelValue string
+	Replicas                  int
 }
 
 type RepoServiceTemplateFields struct {
@@ -63,7 +64,8 @@ type RepoServiceTemplateFields struct {
 	Port        string
 }
 
-func CreateRepoDeployment(clientset *kubernetes.Clientset, namespace string, cluster *crv1.Pgcluster, createPVC bool) error {
+func CreateRepoDeployment(clientset *kubernetes.Clientset, namespace string, cluster *crv1.Pgcluster, createPVC bool,
+	replicas int) error {
 
 	var b bytes.Buffer
 
@@ -119,6 +121,7 @@ func CreateRepoDeployment(clientset *kubernetes.Clientset, namespace string, clu
 		Name:                  serviceName,
 		ClusterName:           cluster.Name,
 		SecurityContext:       util.GetPodSecurityContext(cluster.Spec.BackrestStorage.GetSupplementalGroups()),
+		Replicas:              replicas,
 		PodAntiAffinity: operator.GetPodAntiAffinity(cluster,
 			crv1.PodAntiAffinityDeploymentPgBackRest, cluster.Spec.PodAntiAffinity.PgBackRest),
 		PodAntiAffinityLabelName: config.LABEL_POD_ANTI_AFFINITY,

operator/cluster/clone.go

@@ -306,7 +306,8 @@ func cloneStep2(clientset *kubernetes.Clientset, client *rest.RESTClient, restCo
 	}
 
 	// create the deployment without creating the PVC given we've already done that
-	if err := backrest.CreateRepoDeployment(clientset, namespace, &targetPgcluster, false); err != nil {
+	if err := backrest.CreateRepoDeployment(clientset, namespace, &targetPgcluster, false,
+		1); err != nil {
 		log.Error(err)
 		// publish a failure event
 		errorMessage := fmt.Sprintf("Could not create new pgbackrest repo: %s", err.Error())