Allow CPU/Memory resources to be set one-off for pgBackRest repositories

There are cases where it may be helpful to control the container
resource utilization for the pgBackRest repository Deployment that comes
with every PostgreSQL cluster that is managed by the PostgreSQL Operato
.

This modifies the pgcluster CRD to store the resource values pertaining
to the pgBackRest repository. This also adds new flags to the PostgreSQL
Operator client to manage the CPU / memory resources, including:

- `pgo create cluster --pgbackrest-cpu`
- `pgo create cluster --pgbackrest-memory`
- `pgo update cluster --pgbackrest-cpu`
- `pgo update cluster --pgbackrest-memory`

This also fixes a bug with rotating the pgbouncer system account
password, where all pgBouncer Pods would be checked for propagation
vs. just the pgBouncer Pods for the specific cluster. Also there are
some general code cleanups.

Author: Jonathan S. Katz

apis/crunchydata.com/v1/cluster.go

@@ -81,6 +81,9 @@ type PgclusterSpec struct {
 	// https://www.postgresql.org/docs/current/kernel-resources.html#LINUX-MEMORY-OVERCOMMIT
 	// https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#how-pods-with-resource-limits-are-run
 	Resources v1.ResourceList `json:"resources"`
+	// BackrestResources, if specified, contains the container request resources
+	// for the pgBackRest Deployment for this PostgreSQL cluster
+	BackrestResources v1.ResourceList `json:"backrestResources"`
 	// PgBouncerResources, if specified, contains the container request resources
 	// for any pgBouncer Deployments that are part of a PostgreSQL cluster
 	PgBouncerResources v1.ResourceList          `json:"pgBouncerResources"`

apis/crunchydata.com/v1/zz_generated.deepcopy.go

@@ -571,6 +571,20 @@ func CreateCluster(request *msgs.CreateClusterRequest, ns, pgouser string) msgs.
 		return resp
 	}
 
+	// similarly, if any of the pgBackRest repo CPU / Memory values have been set,
+	// evaluate those as well
+	if err := apiserver.ValidateQuantity(request.BackrestCPURequest); err != nil {
+		resp.Status.Code = msgs.Error
+		resp.Status.Msg = fmt.Sprintf(apiserver.ErrMessageCPURequest, request.BackrestCPURequest, err.Error())
+		return resp
+	}
+
+	if err := apiserver.ValidateQuantity(request.BackrestMemoryRequest); err != nil {
+		resp.Status.Code = msgs.Error
+		resp.Status.Msg = fmt.Sprintf(apiserver.ErrMessageMemoryRequest, request.BackrestMemoryRequest, err.Error())
+		return resp
+	}
+
 	// similarly, if any of the pgBouncer CPU / Memory values have been set,
 	// evaluate those as well
 	if err := apiserver.ValidateQuantity(request.PgBouncerCPURequest); err != nil {
@@ -990,6 +1004,7 @@ func validateConfigPolicies(clusterName, PoliciesFlag, ns string) error {
 func getClusterParams(request *msgs.CreateClusterRequest, name string, userLabelsMap map[string]string, ns string) *crv1.Pgcluster {
 
 	spec := crv1.PgclusterSpec{
+		BackrestResources:  v1.ResourceList{},
 		PgBouncerResources: v1.ResourceList{},
 		Resources:          v1.ResourceList{},
 	}
@@ -1012,6 +1027,20 @@ func getClusterParams(request *msgs.CreateClusterRequest, name string, userLabel
 		spec.Resources[v1.ResourceMemory] = quantity
 	}
 
+	// similarly, if there are any overriding pgBackRest repository container
+	// resource request values, set them here
+	if request.BackrestCPURequest != "" {
+		// as this was already validated, we can ignore the error
+		quantity, _ := resource.ParseQuantity(request.BackrestCPURequest)
+		spec.BackrestResources[v1.ResourceCPU] = quantity
+	}
+
+	if request.BackrestMemoryRequest != "" {
+		// as this was already validated, we can ignore the error
+		quantity, _ := resource.ParseQuantity(request.BackrestMemoryRequest)
+		spec.BackrestResources[v1.ResourceMemory] = quantity
+	}
+
 	// similarly, if there are any overriding pgBouncer container resource request
 	// values, set them here
 	if request.PgBouncerCPURequest != "" {
@@ -1503,6 +1532,20 @@ func UpdateCluster(request *msgs.UpdateClusterRequest) msgs.UpdateClusterRespons
 		return response
 	}
 
+	// similarly, if any of the pgBackRest repo CPU / Memory values have been set,
+	// evaluate those as well
+	if err := apiserver.ValidateQuantity(request.BackrestCPURequest); err != nil {
+		response.Status.Code = msgs.Error
+		response.Status.Msg = fmt.Sprintf(apiserver.ErrMessageCPURequest, request.BackrestCPURequest, err.Error())
+		return response
+	}
+
+	if err := apiserver.ValidateQuantity(request.BackrestMemoryRequest); err != nil {
+		response.Status.Code = msgs.Error
+		response.Status.Msg = fmt.Sprintf(apiserver.ErrMessageMemoryRequest, request.BackrestMemoryRequest, err.Error())
+		return response
+	}
+
 	// validate the storage type for each specified tablespace actually exists.
 	// if a PVCSize is passed in, also validate that it follows the Kubernetes
 	// format
@@ -1608,6 +1651,23 @@ func UpdateCluster(request *msgs.UpdateClusterRequest) msgs.UpdateClusterRespons
 			cluster.Spec.Resources[v1.ResourceMemory] = quantity
 		}
 
+		// ensure there is a value for BackrestResources
+		if cluster.Spec.BackrestResources == nil {
+			cluster.Spec.BackrestResources = v1.ResourceList{}
+		}
+
+		// if the pgBackRest repository CPU or memory values have been modified,
+		// update the values in the cluster CRD
+		if request.BackrestCPURequest != "" {
+			quantity, _ := resource.ParseQuantity(request.BackrestCPURequest)
+			cluster.Spec.BackrestResources[v1.ResourceCPU] = quantity
+		}
+
+		if request.BackrestMemoryRequest != "" {
+			quantity, _ := resource.ParseQuantity(request.BackrestMemoryRequest)
+			cluster.Spec.BackrestResources[v1.ResourceMemory] = quantity
+		}
+
 		// extract the parameters for the TablespaceMounts and put them in the
 		// format that is required by the pgcluster CRD
 		for _, tablespace := range request.Tablespaces {

apiserver/clusterservice/clusterimpl.go

@@ -122,6 +122,13 @@ type CreateClusterRequest struct {
 	// be requested for deploying pgBouncer instances. Defaults to the server
 	// specified default
 	PgBouncerMemoryRequest string
+	// BackrestCPURequest, if specified, is the value of how much CPU should be
+	// requested the pgBackRest repository. Defaults to not being requested
+	BackrestCPURequest string
+	// BackrestMemoryRequest, if specified, is the value of how much RAM should
+	// be requested for the pgBackRest repository. Defaults to the server
+	// specified default
+	BackrestMemoryRequest string
 	// BackrestStorageConfig sets the storage configuration to use for the
 	// pgBackRest local repository. This overrides the value in pgo.yaml, though
 	// the value of BackrestPVCSize can override the PVC size set in this
@@ -292,6 +299,12 @@ type UpdateClusterRequest struct {
 	Namespace     string
 	AllFlag       bool
 	Autofail      UpdateClusterAutofailStatus
+	// BackrestCPURequest, if specified, is the value of how much CPU should be
+	// requested the pgBackRest repository.
+	BackrestCPURequest string
+	// BackrestMemoryRequest, if specified, is the value of how much RAM should
+	// be requested for the pgBackRest repository.
+	BackrestMemoryRequest string
 	// CPURequest is the value of how much CPU should be requested for deploying
 	// the PostgreSQL cluster
 	CPURequest string

apiservermsgs/clustermsgs.go

@@ -24,6 +24,7 @@ import (
 	crv1 "github.com/crunchydata/postgres-operator/apis/crunchydata.com/v1"
 	"github.com/crunchydata/postgres-operator/config"
 	"github.com/crunchydata/postgres-operator/kubeapi"
+	backrestoperator "github.com/crunchydata/postgres-operator/operator/backrest"
 	clusteroperator "github.com/crunchydata/postgres-operator/operator/cluster"
 	informers "github.com/crunchydata/postgres-operator/pkg/generated/informers/externalversions/crunchydata.com/v1"
 	"github.com/crunchydata/postgres-operator/util"
@@ -186,6 +187,16 @@ func (c *Controller) onUpdate(oldObj, newObj interface{}) {
 		}
 	}
 
+	// see if any of the pgBackRest repository resource values have changed, and
+	// if so, update them
+	if oldcluster.Spec.BackrestResources[v1.ResourceCPU] != newcluster.Spec.BackrestResources[v1.ResourceCPU] ||
+		oldcluster.Spec.BackrestResources[v1.ResourceMemory] != newcluster.Spec.BackrestResources[v1.ResourceMemory] {
+		if err := backrestoperator.UpdateResources(c.PgclusterClientset, c.PgclusterConfig, newcluster); err != nil {
+			log.Error(err)
+			return
+		}
+	}
+
 	// if we are not in a standby state, check to see if the tablespaces have
 	// differed, and if so, add the additional volumes to the primary and replicas
 	if !reflect.DeepEqual(oldcluster.Spec.TablespaceMounts, newcluster.Spec.TablespaceMounts) {

controller/pgcluster/pgclustercontroller.go

@@ -33,6 +33,8 @@ pgo create cluster [flags]
       --password-length int                   If no password is supplied, sets the length of the automatically generated password. Defaults to the value set on the server.
       --password-replication string           The password to use for the PostgreSQL replication user.
       --password-superuser string             The password to use for the PostgreSQL superuser.
+      --pgbackrest-cpu string                 Set the number of millicores to request for CPU for the pgBackRest repository. Defaults to being unset.
+      --pgbackrest-memory string              Set the amount of Memory to request for the pgBackRest repository. Defaults to server value (48Mi).
       --pgbackrest-pvc-size string            The size of the PVC capacity for the pgBackRest repository. Overrides the value set in the storage class. This is ignored if the storage type of "local" is not used. Must follow the standard Kubernetes format, e.g. "10.1Gi"
       --pgbackrest-repo-path string           The pgBackRest repository path that should be utilized instead of the default. Required for standby
                                               clusters to define the location of an existing pgBackRest repository.

docs/content/pgo-client/reference/pgo_create_cluster.md

@@ -21,27 +21,29 @@ pgo update cluster [flags]
 ### Options
 
 ```
-      --all                  all resources.
-      --cpu string           Set the number of millicores to request for the CPU, e.g. "100m" or "0.1". Overrides the value in "resources-config"
-      --disable-autofail     Disables autofail capabitilies in the cluster.
-      --disable-standby      Disables standby mode if enabled in the cluster(s) specified.
-      --enable-autofail      Enables autofail capabitilies in the cluster.
-  -h, --help                 help for cluster
-      --memory string        Set the amount of RAM to request, e.g. 1GiB. Overrides the value in "resources-config"
-      --no-prompt            No command line confirmation.
-      --promote-standby      Enables standby mode in the cluster(s) specified.
-  -s, --selector string      The selector to use for cluster filtering.
-      --shutdown             Shutdown the database cluster if it is currently running.
-      --startup              Restart the database cluster if it is currently shutdown.
-      --tablespace strings   Add a PostgreSQL tablespace on the cluster, e.g. "name=ts1:storageconfig=nfsstorage". The format is a key/value map that is delimited by "=" and separated by ":". The following parameters are available:
-                             
-                             - name (required): the name of the PostgreSQL tablespace
-                             - storageconfig (required): the storage configuration to use, as specified in the list available in the "pgo-config" ConfigMap (aka "pgo.yaml")
-                             - pvcsize: the size of the PVC capacity, which overrides the value set in the specified storageconfig. Follows the Kubernetes quantity format.
-                             
-                             For example, to create a tablespace with the NFS storage configuration with a PVC of size 10GiB:
-                             
-                             --tablespace=name=ts1:storageconfig=nfsstorage:pvcsize=10Gi
+      --all                        all resources.
+      --cpu string                 Set the number of millicores to request for the CPU, e.g. "100m" or "0.1". Overrides the value in "resources-config"
+      --disable-autofail           Disables autofail capabitilies in the cluster.
+      --disable-standby            Disables standby mode if enabled in the cluster(s) specified.
+      --enable-autofail            Enables autofail capabitilies in the cluster.
+  -h, --help                       help for cluster
+      --memory string              Set the amount of RAM to request, e.g. 1GiB. Overrides the value in "resources-config"
+      --no-prompt                  No command line confirmation.
+      --pgbackrest-cpu string      Set the number of millicores to request for CPU for the pgBackRest repository.
+      --pgbackrest-memory string   Set the amount of Memory to request for the pgBackRest repository.
+      --promote-standby            Enables standby mode in the cluster(s) specified.
+  -s, --selector string            The selector to use for cluster filtering.
+      --shutdown                   Shutdown the database cluster if it is currently running.
+      --startup                    Restart the database cluster if it is currently shutdown.
+      --tablespace strings         Add a PostgreSQL tablespace on the cluster, e.g. "name=ts1:storageconfig=nfsstorage". The format is a key/value map that is delimited by "=" and separated by ":". The following parameters are available:
+                                   
+                                   - name (required): the name of the PostgreSQL tablespace
+                                   - storageconfig (required): the storage configuration to use, as specified in the list available in the "pgo-config" ConfigMap (aka "pgo.yaml")
+                                   - pvcsize: the size of the PVC capacity, which overrides the value set in the specified storageconfig. Follows the Kubernetes quantity format.
+                                   
+                                   For example, to create a tablespace with the NFS storage configuration with a PVC of size 10GiB:
+                                   
+                                   --tablespace=name=ts1:storageconfig=nfsstorage:pvcsize=10Gi
 ```
 
 ### Options inherited from parent commands
@@ -61,4 +63,4 @@ pgo update cluster [flags]
 
 * [pgo update](/pgo-client/reference/pgo_update/)	 - Update a pgouser, pgorole, or cluster
 
-###### Auto generated by spf13/cobra on 28-Mar-2020
+###### Auto generated by spf13/cobra on 6-Apr-2020

docs/content/pgo-client/reference/pgo_update_cluster.md

@@ -27,15 +27,14 @@ import (
 	"github.com/crunchydata/postgres-operator/operator"
 	"github.com/crunchydata/postgres-operator/operator/pvc"
 	"github.com/crunchydata/postgres-operator/util"
+
 	log "github.com/sirupsen/logrus"
-	v1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 )
 
-const BackrestRepoServiceName = "%s-backrest-shared-repo"
-const BackrestRepoPVCName = "%s-pgbr-repo"
-
 type RepoDeploymentTemplateFields struct {
 	SecurityContext           string
 	PGOImagePrefix            string
@@ -68,8 +67,8 @@ func CreateRepoDeployment(clientset *kubernetes.Clientset, namespace string, clu
 
 	var b bytes.Buffer
 
-	repoName := fmt.Sprintf(BackrestRepoPVCName, cluster.Name)
-	serviceName := fmt.Sprintf(BackrestRepoServiceName, cluster.Name)
+	repoName := fmt.Sprintf(util.BackrestRepoPVCName, cluster.Name)
+	serviceName := fmt.Sprintf(util.BackrestRepoServiceName, cluster.Name)
 
 	//create backrest repo service
 	serviceFields := RepoServiceTemplateFields{
@@ -103,7 +102,7 @@ func CreateRepoDeployment(clientset *kubernetes.Clientset, namespace string, clu
 	fields := RepoDeploymentTemplateFields{
 		PGOImagePrefix:        operator.Pgo.Pgo.PGOImagePrefix,
 		PGOImageTag:           operator.Pgo.Pgo.PGOImageTag,
-		ContainerResources:    "",
+		ContainerResources:    operator.GetResourcesJSON(cluster.Spec.BackrestResources),
 		BackrestRepoClaimName: repoName,
 		SshdSecretsName:       "pgo-backrest-repo-config",
 		PGbackrestDBHost:      cluster.Name,
@@ -135,7 +134,7 @@ func CreateRepoDeployment(clientset *kubernetes.Clientset, namespace string, clu
 		config.PgoBackrestRepoTemplate.Execute(os.Stdout, fields)
 	}
 
-	deployment := v1.Deployment{}
+	deployment := appsv1.Deployment{}
 	err = json.Unmarshal(b.Bytes(), &deployment)
 	if err != nil {
 		log.Error("error unmarshalling backrest repo json into Deployment " + err.Error())
@@ -152,6 +151,52 @@ func CreateRepoDeployment(clientset *kubernetes.Clientset, namespace string, clu
 
 }
 
+// UpdateResources updates the pgBackRest repository Deployment to reflect any
+// resource updates
+func UpdateResources(clientset *kubernetes.Clientset, restConfig *rest.Config, cluster *crv1.Pgcluster) error {
+	// get a list of all of the instance deployments for the cluster
+	deployment, err := operator.GetBackrestDeployment(clientset, cluster)
+
+	if err != nil {
+		return err
+	}
+
+	// iterate through each PostgreSQL instance deployment and update the
+	// resource values for the pgBackRest repository container. This is the first
+	// container
+	requestResourceList := v1.ResourceList{}
+
+	// if there is a request resource list available already, use that one
+	// NOTE: this works as the "database" container is always first
+	if deployment.Spec.Template.Spec.Containers[0].Resources.Requests != nil {
+		requestResourceList = deployment.Spec.Template.Spec.Containers[0].Resources.Requests
+	}
+
+	// handle the CPU update
+	if request, ok := cluster.Spec.BackrestResources[v1.ResourceCPU]; ok {
+		requestResourceList[v1.ResourceCPU] = request
+	} else {
+		delete(requestResourceList, v1.ResourceCPU)
+	}
+
+	// handle the memory update
+	if request, ok := cluster.Spec.BackrestResources[v1.ResourceMemory]; ok {
+		requestResourceList[v1.ResourceMemory] = request
+	} else {
+		delete(requestResourceList, v1.ResourceMemory)
+	}
+
+	// update the requests resourcelist
+	deployment.Spec.Template.Spec.Containers[0].Resources.Requests = requestResourceList
+
+	// update the deployment with the new values
+	if err := kubeapi.UpdateDeployment(clientset, deployment); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func createService(clientset *kubernetes.Clientset, fields *RepoServiceTemplateFields, namespace string) error {
 	var err error
 
@@ -170,7 +215,7 @@ func createService(clientset *kubernetes.Clientset, fields *RepoServiceTemplateF
 			config.PgoBackrestRepoServiceTemplate.Execute(os.Stdout, fields)
 		}
 
-		s := corev1.Service{}
+		s := v1.Service{}
 		err = json.Unmarshal(b.Bytes(), &s)
 		if err != nil {
 			log.Error("error unmarshalling repo service json into repo Service " + err.Error())