Proper Detection of Previously Processed pgcluster

The pgcluster controller has been updated to properly detect whether or
not a specific pgcluster custom resource was previously processed via
the controller's 'onAdd' function.  Specifically, the 'onAdd' function
will now no longer place a pgcluster in a 'processed' status and
attempt to call 'AddClusterBase' if it determines that the a previous
call to 'AddClusterBase' completed successfully.  This is indicated by
either a value of 'completed' in the /spec/status' field of the
pgcluster (which indicates that a previous call to 'AddClusterBase'
successfully created the resources needed to bootstrap the PG cluster),
or a value of 'pgcluster Initialized' in the  '/status/state' field
(which indicates that the cluster has been fully initialized).

In addition, the check for an existing Kubernetes Deployment matching
the name of the pgcluster, as well as the check to determine whether
or not a pgcluster custom resource is in a 'processed' status prior to
enqueuing it from the pgcluster controller's 'onAdd' function, have
been removed, since both are no longer valid methods for determining
whether or not to process a pgcluster.  For instance, a Deployment
matching the pgcluster name (i.e. the original primary for the cluster)
might no longer exist if it was since scaled down and removed, while
the 'processed' status does not properly indicate whether or not the
various Kubernetes resources (e.g. Deployments) for the initial primary
utilized to bootstrap the cluster have been created, nor does it
properly indicate whether or not the PG cluster has been fully
initialized.

Issue: [ch9574]

Author: andrewlecuyer

controller/pgcluster/pgclustercontroller.go

@@ -48,22 +48,11 @@ type Controller struct {
 
 // onAdd is called when a pgcluster is added
 func (c *Controller) onAdd(obj interface{}) {
-	cluster := obj.(*crv1.Pgcluster)
-	log.Debugf("[pgcluster Controller] ns %s onAdd %s", cluster.ObjectMeta.Namespace, cluster.ObjectMeta.SelfLink)
-
-	//handle the case when the operator restarts and don't
-	//process already processed pgclusters
-	if cluster.Status.State == crv1.PgclusterStateProcessed {
-		log.Debug("pgcluster " + cluster.ObjectMeta.Name + " already processed")
-		return
-	}
-
 	key, err := cache.MetaNamespaceKeyFunc(obj)
 	if err == nil {
 		log.Debugf("cluster putting key in queue %s", key)
 		c.Queue.Add(key)
 	}
-
 }
 
 // RunWorker is a long-running function that will continually call the
@@ -106,25 +95,23 @@ func (c *Controller) processNextItem() bool {
 	// parallel.
 	defer c.Queue.Done(key)
 
-	// Invoke the method containing the business logic
-	// in this case, the de-dupe logic is to test whether a cluster
-	// deployment exists , if so, then we don't create another
-	_, found, err := kubeapi.GetDeployment(c.PgclusterClientset, keyResourceName, keyNamespace)
-
-	if found {
-		log.Debugf("cluster add - dep already found, not creating again")
-		c.Queue.Forget(key)
-		return true
-	}
-
 	//get the pgcluster
 	cluster := crv1.Pgcluster{}
-	found, err = kubeapi.Getpgcluster(c.PgclusterClient, &cluster, keyResourceName, keyNamespace)
+	found, err := kubeapi.Getpgcluster(c.PgclusterClient, &cluster, keyResourceName, keyNamespace)
 	if !found {
 		log.Debugf("cluster add - pgcluster not found, this is invalid")
 		c.Queue.Forget(key) // NB(cbandy): This should probably be a retry.
 		return true
 	}
+	log.Debugf("[pgcluster Controller] ns %s onAdd %s", cluster.ObjectMeta.Namespace, cluster.ObjectMeta.SelfLink)
+
+	if cluster.Spec.Status == crv1.CompletedStatus ||
+		cluster.Status.State == crv1.PgclusterStateInitialized {
+		log.Debugf("pgcluster Contoller: onAdd event received for cluster %s but "+
+			"will not process because it either has a 'completed' status or is currently in an "+
+			"'initialized' state", cluster.GetName())
+		return true
+	}
 
 	addIdentifier(&cluster)
 

operator/cluster/cluster.go

@@ -63,13 +63,6 @@ const (
 func AddClusterBase(clientset *kubernetes.Clientset, client *rest.RESTClient, cl *crv1.Pgcluster, namespace string) {
 	var err error
 
-	if cl.Spec.Status == crv1.CompletedStatus {
-		errorMsg := "crv1 pgcluster " + cl.Spec.ClusterName + " is already marked complete, will not recreate"
-		log.Warn(errorMsg)
-		publishClusterCreateFailure(cl, errorMsg)
-		return
-	}
-
 	dataVolume, walVolume, tablespaceVolumes, err := pvc.CreateMissingPostgreSQLVolumes(
 		clientset, cl, namespace, cl.Annotations[config.ANNOTATION_CURRENT_PRIMARY], cl.Spec.PrimaryStorage)
 	if err != nil {