Upgrade Sidecar Containers & Cleanup Metrics Secrets (#1046)

* The 'crunchy-prometheus' ConfigMap is now deleted
when uninstall the metrics stack using the Ansible
installer (specifically when specifying the
'uninstall-mertics' tag).

* Add 'pgo_disable_eventing' to the default inventory
file for the Ansible installer.

* The available sidecar containers for the postgres
pod (i.e. 'crunchy-collect and 'crunchy-pgbadger')
are now patched and upgraded along with the
primary and replica postgres containers during a
minor upgrade.

Author: andrewlecuyer

ansible/inventory

@@ -77,6 +77,9 @@ pgo_client_version='v4.1.0'
 #pgo_disable_tls='false'
 #pgo_apiserver_port=8443
 
+# PGO Event Settings
+#pgo_disable_eventing='false'
+
 # Set to 'true' to assign the cluster-admin role to the PGO service account.  Needed for
 # OCP installs to enable dynamic namespace creation (see the PGO docs for more details).
 #pgo_cluster_admin='false'

ansible/roles/pgo-metrics/tasks/cleanup.yml

@@ -79,3 +79,11 @@
   no_log: false
   tags:
   - uninstall-metrics
+
+- name: Delete Prometheus ConfigMap
+  shell: |
+    {{ kubectl_or_oc }} delete configmap crunchy-prometheus -n {{ metrics_namespace }}
+  ignore_errors: yes
+  no_log: false
+  tags:
+  - uninstall-metrics

config/labels.go

@@ -38,6 +38,7 @@ const LABEL_PGREMOVE = "pgremove"
 const LABEL_PVCNAME = "pvcname"
 const LABEL_COLLECT = "crunchy_collect"
 const LABEL_COLLECT_PG_USER = "ccp_monitoring"
+const LABEL_COLLECT_CCPIMAGE = "crunchy-collect"
 const LABEL_ARCHIVE = "archive"
 const LABEL_ARCHIVE_TIMEOUT = "archive-timeout"
 const LABEL_CUSTOM_CONFIG = "custom-config"
@@ -83,6 +84,7 @@ const LABEL_BACKREST_OPTS = "backrest-opts"
 const LABEL_BACKREST_PITR_TARGET = "backrest-pitr-target"
 const LABEL_BACKREST_STORAGE_TYPE = "backrest-storage-type"
 const LABEL_BADGER = "crunchy-pgbadger"
+const LABEL_BADGER_CCPIMAGE = "crunchy-pgbadger"
 const LABEL_BACKUP_TYPE_BASEBACKUP = "pgbasebackup"
 const LABEL_BACKUP_TYPE_BACKREST = "pgbackrest"
 const LABEL_BACKUP_TYPE_PGDUMP = "pgdump"

controller/podcontroller.go

@@ -122,7 +122,7 @@ func (c *PodController) onUpdate(oldObj, newObj interface{}) {
 	// have a pod coming back up from upgrade and is ready - time to kick off the next pod.
 	if clusterInMinorUpgrade && isUpgradedPostgresPod(newpod, oldpod) {
 		upgradeTaskName := clusterName + "-" + config.LABEL_MINOR_UPGRADE
-		clusteroperator.ProcessNextUpgradeItem(c.PodClientset, c.PodClient, clusterName, upgradeTaskName, newpod.ObjectMeta.Namespace)
+		clusteroperator.ProcessNextUpgradeItem(c.PodClientset, c.PodClient, pgcluster, upgradeTaskName, newpod.ObjectMeta.Namespace)
 	}
 
 	//handle the case when a pg database pod is updated

kubeapi/deployment.go

@@ -106,21 +106,22 @@ type ThingSpec struct {
 }
 
 // PatchDeployment patches a deployment
-func PatchDeployment(clientset *kubernetes.Clientset, name, namespace, jsonpath, patchvalue string) error {
+func PatchDeployment(clientset *kubernetes.Clientset, name, namespace string, pathValueMap map[string]string) error {
 	var patchBytes []byte
 	var err error
 
-	things := make([]ThingSpec, 1)
-	things[0].Op = "replace"
-	things[0].Path = jsonpath
-	things[0].Value = patchvalue
+	things := make([]ThingSpec, 0)
+	for path, val := range pathValueMap {
+		things = append(things, ThingSpec{"replace", path, val})
+    }
 
 	patchBytes, err = json.Marshal(things)
 	if err != nil {
 		log.Error("error in converting patch " + err.Error())
 		return err
 	}
 
+	log.Debug("about to patch deployment using" + string(patchBytes))
 	_, err = clientset.AppsV1().Deployments(namespace).Patch(name, types.JSONPatchType, patchBytes)
 	if err != nil {
 		log.Error(err)

operator/cluster/upgrade.go

@@ -17,6 +17,7 @@ package cluster
 
 import (
 	"strings"
+	"strconv"
 	"time"
 
 	crv1 "github.com/crunchydata/postgres-operator/apis/cr/v1"
@@ -100,7 +101,7 @@ func AddUpgrade(clientset *kubernetes.Clientset, restclient *rest.RESTClient, up
 	publishMinorUpgradeStartedEvent(&currentTask, &cl, namespace)
 
 	// start the upgrade
-	ProcessNextUpgradeItem(clientset, restclient, cl.Spec.Name, currentTask.Spec.Name, namespace)
+	ProcessNextUpgradeItem(clientset, restclient, cl, currentTask.Spec.Name, namespace)
 
 }
 
@@ -109,7 +110,7 @@ func AddUpgrade(clientset *kubernetes.Clientset, restclient *rest.RESTClient, up
 // If more than one replica is in the list, they are done one at a time, once per call
 // with an item getting removed from the list each time. This method should get called
 // after the pod goes ready from the previous item, which is handled by the pod controller.
-func ProcessNextUpgradeItem(clientset *kubernetes.Clientset, restclient *rest.RESTClient, clusterName, upgradeTaskName, namespace string) {
+func ProcessNextUpgradeItem(clientset *kubernetes.Clientset, restclient *rest.RESTClient, cluster crv1.Pgcluster, upgradeTaskName, namespace string) {
 
 	log.Debug("Upgrade: ProcessNextUpgradeItem.... ", upgradeTaskName)
 
@@ -122,15 +123,6 @@ func ProcessNextUpgradeItem(clientset *kubernetes.Clientset, restclient *rest.RE
 		log.Error(err)
 	}
 
-	cluster := crv1.Pgcluster{}
-	// get the cluster
-	_, err = kubeapi.Getpgcluster(restclient, &cluster, clusterName, namespace)
-	if err != nil {
-		log.Error("cound not find pgcluster for minor upgrade")
-		log.Error(err)
-		return // make this failed upgrade with error?
-	}
-
 	replicaTargets := upgradeTask.Spec.Parameters[config.LABEL_UPGRADE_REPLICA]
 	backrestTargetName := upgradeTask.Spec.Parameters[config.LABEL_UPGRADE_BACKREST]
 	primaryTargetName := upgradeTask.Spec.Parameters[config.LABEL_UPGRADE_PRIMARY]
@@ -155,8 +147,15 @@ func ProcessNextUpgradeItem(clientset *kubernetes.Clientset, restclient *rest.RE
 		//the new image tag.
 
 		log.Debug("About to patch replica: ", replicaTargetName)
-		err = kubeapi.PatchDeployment(clientset, replicaTargetName, namespace, "/spec/template/spec/containers/0/image",
-			operator.Pgo.Cluster.CCPImagePrefix+"/"+cluster.Spec.CCPImage+":"+upgradeTask.Spec.Parameters["CCPImageTag"])
+		
+		patchMap := make(map[string]string)
+		patchMap["/spec/template/spec/containers/0/image"] =
+			operator.Pgo.Cluster.CCPImagePrefix+"/"+cluster.Spec.CCPImage+":"+upgradeTask.Spec.Parameters["CCPImageTag"]
+
+		addSidecarsToPgUpgradePatch(patchMap, cluster, operator.Pgo.Cluster.CCPImagePrefix, 
+			upgradeTask.Spec.Parameters["CCPImageTag"])
+
+		err = kubeapi.PatchDeployment(clientset, replicaTargetName, namespace, patchMap)
 		if err != nil {
 			log.Error(err)
 			log.Error("error in doing replica minor upgrade")
@@ -176,8 +175,12 @@ func ProcessNextUpgradeItem(clientset *kubernetes.Clientset, restclient *rest.RE
 		log.Debug("Minor Upgrade: backrest")
 
 		log.Debug("About to patch backrest: ", backrestTargetName)
-		err = kubeapi.PatchDeployment(clientset, backrestTargetName, namespace, "/spec/template/spec/containers/0/image",
-			operator.Pgo.Cluster.CCPImagePrefix+"/pgo-backrest-repo:"+upgradeTask.Spec.Parameters["CCPImageTag"])
+		
+		patchMap := make(map[string]string)
+		patchMap["/spec/template/spec/containers/0/image"] =
+			operator.Pgo.Cluster.CCPImagePrefix+"/pgo-backrest-repo:"+upgradeTask.Spec.Parameters["CCPImageTag"]
+
+		err = kubeapi.PatchDeployment(clientset, backrestTargetName, namespace, patchMap)
 		if err != nil {
 			log.Error(err)
 			log.Error("error in doing backrest minor upgrade")
@@ -199,13 +202,20 @@ func ProcessNextUpgradeItem(clientset *kubernetes.Clientset, restclient *rest.RE
 		// If we don't do this now, then one of the replicas will come back up with the old container image.
 		if autoFailEnabled {
 			upgradedImageTag := upgradeTask.Spec.Parameters["CCPImageTag"]
-			updateClusterCCPImage(restclient, upgradedImageTag, clusterName, namespace)
+			updateClusterCCPImage(restclient, upgradedImageTag, cluster.Spec.Name, namespace)
 		}
 
 		log.Debug("Minor Upgrade: primary")
 		log.Debug("About to patch primary: ", primaryTargetName)
-		err = kubeapi.PatchDeployment(clientset, primaryTargetName, namespace, "/spec/template/spec/containers/0/image",
-			operator.Pgo.Cluster.CCPImagePrefix+"/"+cluster.Spec.CCPImage+":"+upgradeTask.Spec.Parameters["CCPImageTag"])
+		
+		patchMap := make(map[string]string)
+		patchMap["/spec/template/spec/containers/0/image"] =
+			operator.Pgo.Cluster.CCPImagePrefix+"/"+cluster.Spec.CCPImage+":"+upgradeTask.Spec.Parameters["CCPImageTag"]
+		
+		addSidecarsToPgUpgradePatch(patchMap, cluster, operator.Pgo.Cluster.CCPImagePrefix, 
+			upgradeTask.Spec.Parameters["CCPImageTag"])
+
+		err = kubeapi.PatchDeployment(clientset, primaryTargetName, namespace, patchMap)
 		if err != nil {
 			log.Error(err)
 			log.Error("error in doing primary minor upgrade")
@@ -222,12 +232,32 @@ func ProcessNextUpgradeItem(clientset *kubernetes.Clientset, restclient *rest.RE
 
 	} else {
 		// No other deployments  left to upgrade, complete the upgrade
-		completeUpgrade(clientset, restclient, &upgradeTask, autoFailEnabled, clusterName, namespace)
+		completeUpgrade(clientset, restclient, &upgradeTask, autoFailEnabled, cluster.Spec.Name, namespace)
 
 		publishMinorUpgradeCompleteEvent(&upgradeTask, &cluster, namespace)
 	}
 }
 
+func addSidecarsToPgUpgradePatch(patchMap map[string]string, cluster crv1.Pgcluster, ccpImagePrefix, 
+	ccpImageTag string) {
+
+	collectEnabled, _ := strconv.ParseBool(cluster.Labels[config.LABEL_COLLECT])
+	badgerEnabled, _ := strconv.ParseBool(cluster.Labels[config.LABEL_BADGER])
+
+	if collectEnabled && badgerEnabled {
+		patchMap["/spec/template/spec/containers/1/image"] =
+		ccpImagePrefix+"/"+config.LABEL_COLLECT_CCPIMAGE+":"+ccpImageTag
+		patchMap["/spec/template/spec/containers/2/image"] =
+			ccpImagePrefix+"/"+config.LABEL_BADGER_CCPIMAGE+":"+ccpImageTag
+	} else if collectEnabled && !badgerEnabled {
+		patchMap["/spec/template/spec/containers/1/image"] =
+			ccpImagePrefix+"/"+config.LABEL_COLLECT_CCPIMAGE+":"+ccpImageTag
+	} else if badgerEnabled && !collectEnabled {
+		patchMap["/spec/template/spec/containers/1/image"] =
+			ccpImagePrefix+"/"+config.LABEL_BADGER+":"+ccpImageTag
+	}
+}
+
 // completeUpgrade - makes any finishing changes required to complete the upgrade and
 // does final updates to the task and cluster.
 func completeUpgrade(clientset *kubernetes.Clientset, restclient *rest.RESTClient, upgradeTask *crv1.Pgtask, autoFail bool, clusterName, namespace string) {