Existence check for primary Deployment before scaling

During the cluster initialization process, there is a step where
the Deployment object representing the primary is explicitly
scaled from 0 to 1 Pods. However, there is a case that may
trigger the Deployment scaling before the Deployment is created.
As such, we can add a guard to check that the Deployment is
actually created before proceeding with the scaling operation.

This patch also modifies a debug line around the cluster
Deployment creation, as the debug output came after the
point at which the code may have exited due to error.

Author: jkatz

internal/controller/pod/inithandler.go

@@ -18,6 +18,7 @@ limitations under the License.
 import (
 	"fmt"
 	"strconv"
+	"time"
 
 	"github.com/crunchydata/postgres-operator/internal/config"
 	"github.com/crunchydata/postgres-operator/internal/controller"
@@ -28,12 +29,14 @@ import (
 	taskoperator "github.com/crunchydata/postgres-operator/internal/operator/task"
 	"github.com/crunchydata/postgres-operator/internal/util"
 	crv1 "github.com/crunchydata/postgres-operator/pkg/apis/crunchydata.com/v1"
+
+	log "github.com/sirupsen/logrus"
 	apiv1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/types"
-
-	log "github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 // handleClusterInit is responsible for proceeding with initialization of the PG cluster once the
@@ -85,6 +88,16 @@ func (c *Controller) handleBackRestRepoInit(newPod *apiv1.Pod, cluster *crv1.Pgc
 		return nil
 	}
 
+	// first: a sanity check that there exists a primary deployment to scale. this
+	// is to attempt to avoid any silent failures in the deployment scaling
+	// function.
+	//
+	// If we do encounter an error, we will proceed in case the deployment becomes
+	// available after.
+	if err := c.waitForPrimaryDeployment(cluster); err != nil {
+		log.Warn(err)
+	}
+
 	clusterInfo, err := clusteroperator.ScaleClusterDeployments(c.Client, *cluster, 1,
 		true, false, false, false)
 	if err != nil {
@@ -288,3 +301,33 @@ func (c *Controller) labelPostgresPodAndDeployment(newpod *apiv1.Pod) {
 	}
 
 }
+
+// waitForPrimaryDeployment checks to see that a primary deployment is
+// available. It does not check readiness, only that the deployment exists. This
+// used before scaling to ensure scaling does not fail silently
+func (c *Controller) waitForPrimaryDeployment(cluster *crv1.Pgcluster) error {
+	primaryDeploymentName := cluster.Annotations[config.ANNOTATION_CURRENT_PRIMARY]
+	options := metav1.ListOptions{
+		LabelSelector: fields.AndSelectors(
+			fields.OneTermEqualSelector(config.LABEL_PG_CLUSTER, cluster.Name),
+			fields.OneTermEqualSelector(config.LABEL_PG_DATABASE, config.LABEL_TRUE),
+			fields.OneTermEqualSelector(config.LABEL_DEPLOYMENT_NAME, primaryDeploymentName),
+		).String(),
+	}
+
+	// start polling to see if the primary deployment is created
+	if err := wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) {
+		// check to see if the deployment exists
+		d, err := c.Client.AppsV1().Deployments(cluster.Namespace).List(options)
+
+		if err != nil {
+			log.Warnf("could not find primary deployment for scaling: %s", err)
+		}
+
+		return err == nil && len(d.Items) > 0, nil
+	}); err != nil {
+		return fmt.Errorf("primary deployment lookup timeout reached for %q", primaryDeploymentName)
+	}
+
+	return nil
+}

internal/operator/cluster/clusterlogic.go

@@ -141,15 +141,15 @@ func addClusterDeployments(clientset kubeapi.Interface,
 	deploymentFields := getClusterDeploymentFields(clientset, cl,
 		dataVolume, walVolume, tablespaceVolumes)
 
+	if operator.CRUNCHY_DEBUG {
+		_ = config.DeploymentTemplate.Execute(os.Stdout, deploymentFields)
+	}
+
 	var primaryDoc bytes.Buffer
 	if err := config.DeploymentTemplate.Execute(&primaryDoc, deploymentFields); err != nil {
 		return err
 	}
 
-	if operator.CRUNCHY_DEBUG {
-		config.DeploymentTemplate.Execute(os.Stdout, deploymentFields)
-	}
-
 	deployment := &appsv1.Deployment{}
 	if err := json.Unmarshal(primaryDoc.Bytes(), deployment); err != nil {
 		return err