Allow one-off specifications of CPU / Memory

Previously the only way to specify container resources (CPU/RAM)
for a PostgreSQL cluster was to customize the "resource config"
in the pgo.yaml file and reference it from the "pgo create cluster"
command.

This introduces the "--cpu" and "--memory" flags to the
"pgo create cluster" command to allow one to specify custom values
for the PostgreSQL cluster at cluster creation time.

Issue: [ch6658]
Issue: #1268

Author: Jonathan S. Katz

apiserver/clusterservice/clusterimpl.go

@@ -554,6 +554,23 @@ func CreateCluster(request *msgs.CreateClusterRequest, ns, pgouser string) msgs.
 		}
 	}
 
+	// evaluate if the CPU / Memory have been set to custom values
+	if request.CPURequest != "" {
+		if err := apiserver.ValidateQuantity(request.CPURequest); err != nil {
+			resp.Status.Code = msgs.Error
+			resp.Status.Msg = fmt.Sprintf(apiserver.ErrMessageCPURequest, request.CPURequest, err.Error())
+			return resp
+		}
+	}
+
+	if request.MemoryRequest != "" {
+		if err := apiserver.ValidateQuantity(request.MemoryRequest); err != nil {
+			resp.Status.Code = msgs.Error
+			resp.Status.Msg = fmt.Sprintf(apiserver.ErrMessageMemoryRequest, request.MemoryRequest, err.Error())
+			return resp
+		}
+	}
+
 	// validate the storage type for each specified tablespace actually exists.
 	// if a PVCSize is passed in, also validate that it follows the Kubernetes
 	// format
@@ -950,6 +967,19 @@ func getClusterParams(request *msgs.CreateClusterRequest, name string, userLabel
 		}
 	}
 
+	// if the request has overriding CPURequest and/or MemoryRequest parameters,
+	// these will take precedence over the values set by "ContainerResources"
+	// This will also overwrite the Limits
+	if request.CPURequest != "" {
+		spec.ContainerResources.RequestsCPU = request.CPURequest
+		spec.ContainerResources.LimitsCPU = ""
+	}
+
+	if request.MemoryRequest != "" {
+		spec.ContainerResources.RequestsMemory = request.MemoryRequest
+		spec.ContainerResources.LimitsMemory = ""
+	}
+
 	spec.PrimaryStorage, _ = apiserver.Pgo.GetStorageSpec(apiserver.Pgo.PrimaryStorage)
 	if request.StorageConfig != "" {
 		spec.PrimaryStorage, _ = apiserver.Pgo.GetStorageSpec(request.StorageConfig)

apiserver/common.go

@@ -28,14 +28,23 @@ import (
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+const (
+	// ErrMessageCPURequest provides a standard error message when a CPURequest
+	// is not specified to the Kubernetes sstandard
+	ErrMessageCPURequest = `could not parse CPU request "%s":%s (hint: try a value like "1" or "100m")`
+	// ErrMessageMemoryRequest provides a standard error message when a MemoryRequest
+	// is not specified to the Kubernetes sstandard
+	ErrMessageMemoryRequest = `could not parse memory request "%s":%s (hint: try a value like "1Gi")`
+	// ErrMessagePVCSize provides a standard error message when a PVCSize is not
+	// specified to the Kubernetes stnadard
+	ErrMessagePVCSize = `could not parse PVC size "%s": %s (hint: try a value like "1Gi")`
+)
+
 var (
 	backrestStorageTypes = []string{"local", "s3"}
 	// ErrDBContainerNotFound is an error that indicates that a "database" container
 	// could not be found in a specific pod
 	ErrDBContainerNotFound = errors.New("\"database\" container not found in pod")
-	// ErrMessagePVCSize provides a standard error message when a PVCSize is not
-	// specified to the Kubernetes stnadard
-	ErrMessagePVCSize = `could not parse PVC size "%s": %s (hint: try a value like "1Gi")`
 	// ErrStandbyNotAllowed contains the error message returned when an API call is not
 	// permitted because it involves a cluster that is in standby mode
 	ErrStandbyNotAllowed = errors.New("Action not permitted because standby mode is enabled")

apiserver/root.go

@@ -531,12 +531,6 @@ func validateWithKube() {
 
 // GetContainerResources ...
 func GetContainerResourcesJSON(resources *crv1.PgContainerResources) string {
-
-	//test for the case where no container resources are specified
-	if resources.RequestsMemory == "" || resources.RequestsCPU == "" ||
-		resources.LimitsMemory == "" || resources.LimitsCPU == "" {
-		return ""
-	}
 	fields := containerResourcesTemplateFields{}
 	fields.RequestsMemory = resources.RequestsMemory
 	fields.RequestsCPU = resources.RequestsCPU

apiservermsgs/clustermsgs.go

@@ -108,6 +108,12 @@ type CreateClusterRequest struct {
 	// CASecret is the name of the secret that contains the CA to use along with
 	// the TLS keypair for deploying a TLS-enabled PostgreSQL cluster
 	CASecret string
+	// CPURequest is the value of how much CPU should be requested for deploying
+	// the PostgreSQL cluster
+	CPURequest string
+	// MemoryRequest is the value of how much RAM should be requested for
+	// deploying the PostgreSQL cluster
+	MemoryRequest string
 }
 
 // CreateClusterDetail provides details about the PostgreSQL cluster that is

conf/postgres-operator/container-resources.json

@@ -1,10 +1,22 @@
-            "resources": {
-                "requests": {
-                    "memory": "{{.RequestsMemory}}",
-                    "cpu": "{{.RequestsCPU}}"
-                },
-                "limits": {
-                    "memory": "{{.LimitsMemory}}",
-                    "cpu": "{{.LimitsCPU}}"
-                }
-            },
+"resources": {
+  {{ if or .RequestsMemory .RequestsCPU }}
+  "requests": {
+    {{ if .RequestsCPU }}
+    "cpu": "{{.RequestsCPU}}"{{ if .RequestsMemory }},{{ end }}
+    {{ end }}
+    {{ if .RequestsMemory }}
+    "memory": "{{.RequestsMemory}}"
+    {{ end }}
+  }{{ if or .LimitsCPU .LimitsMemory }},{{ end }}
+  {{ end }}
+  {{ if or .LimitsCPU .LimitsMemory }}
+  "limits": {
+    {{ if .LimitsCPU }}
+    "cpu": "{{.LimitsCPU}}"{{ if .LimitsMemory }},{{ end }}
+    {{ end }}
+    {{ if .LimitsMemory }}
+    "memory": "{{.LimitsMemory}}"
+    {{ end }}
+  }
+  {{ end }}
+},

docs/content/pgo-client/reference/pgo_create_cluster.md

@@ -20,11 +20,13 @@ pgo create cluster [flags]
 ```
       --ccp-image string                      The CCPImage name to use for cluster creation. If specified, overrides the value crunchy-postgres.
   -c, --ccp-image-tag string                  The CCPImageTag to use for cluster creation. If specified, overrides the pgo.yaml setting.
+      --cpu string                            Set the number of millicores to request for the CPU, e.g. "100m" or "0.1". Overrides the value in "resources-config"
       --custom-config string                  The name of a configMap that holds custom PostgreSQL configuration files used to override defaults.
   -d, --database string                       If specified, sets the name of the initial database that is created for the user. Defaults to the value set in the PostgreSQL Operator configuration, or if that is not present, the name of the cluster
       --disable-autofail                      Disables autofail capabitilies in the cluster following cluster initialization.
   -h, --help                                  help for cluster
   -l, --labels string                         The labels to apply to this cluster.
+      --memory string                         Set the amount of RAM to request, e.g. 1GiB. Overrides the value in "resources-config"
       --metrics                               Adds the crunchy-collect container to the database pod.
       --node-label string                     The node label (key=value) to use in placing the primary database. If not set, any node is used.
       --password string                       The password to use for standard user account created during cluster initialization.
@@ -88,4 +90,4 @@ pgo create cluster [flags]
 
 * [pgo create](/pgo-client/reference/pgo_create/)	 - Create a Postgres Operator resource
 
-###### Auto generated by spf13/cobra on 19-Mar-2020
+###### Auto generated by spf13/cobra on 26-Mar-2020

operator/common.go

@@ -124,23 +124,19 @@ func Initialize(clientset *kubernetes.Clientset) {
 	log.Info("EventTCPAddress set to " + EventTCPAddress)
 }
 
-// GetContainerResources ...
+// GetContainerResources is a legacy method that  creates the JSON snippet that
+// is applied for setting the CPU and memory in a container.
 func GetContainerResourcesJSON(resources *crv1.PgContainerResources) string {
-
-	//test for the case where no container resources are specified
-	if resources.RequestsMemory == "" || resources.RequestsCPU == "" ||
-		resources.LimitsMemory == "" || resources.LimitsCPU == "" {
-		return ""
+	fields := containerResourcesTemplateFields{
+		LimitsCPU:      resources.LimitsCPU,
+		LimitsMemory:   resources.LimitsMemory,
+		RequestsCPU:    resources.RequestsCPU,
+		RequestsMemory: resources.RequestsMemory,
 	}
-	fields := containerResourcesTemplateFields{}
-	fields.RequestsMemory = resources.RequestsMemory
-	fields.RequestsCPU = resources.RequestsCPU
-	fields.LimitsMemory = resources.LimitsMemory
-	fields.LimitsCPU = resources.LimitsCPU
 
 	doc := bytes.Buffer{}
-	err := config.ContainerResourcesTemplate.Execute(&doc, fields)
-	if err != nil {
+
+	if err := config.ContainerResourcesTemplate.Execute(&doc, fields); err != nil {
 		log.Error(err.Error())
 		return ""
 	}

pgo/cmd/cluster.go

@@ -27,6 +27,8 @@ import (
 	"github.com/crunchydata/postgres-operator/pgo/api"
 	"github.com/crunchydata/postgres-operator/pgo/util"
 	log "github.com/sirupsen/logrus"
+
+	"k8s.io/apimachinery/pkg/api/resource"
 )
 
 // below are the tablespace parameters and the expected values of each
@@ -283,6 +285,26 @@ func createCluster(args []string, ns string, createClusterCmd *cobra.Command) {
 		r.SyncReplication = &SyncReplication
 	}
 
+	// if the user provided resources for CPU or Memory, validate them to ensure
+	// they are valid Kubernetes values
+	if CPURequest != "" {
+		if _, err := resource.ParseQuantity(CPURequest); err != nil {
+			fmt.Println("Error:", err.Error(), `"`+CPURequest+`"`)
+			os.Exit(1)
+		}
+
+		r.CPURequest = CPURequest
+	}
+
+	if MemoryRequest != "" {
+		if _, err := resource.ParseQuantity(MemoryRequest); err != nil {
+			fmt.Println("Error:", err.Error(), `"`+MemoryRequest+`"`)
+			os.Exit(1)
+		}
+
+		r.MemoryRequest = MemoryRequest
+	}
+
 	response, err := api.CreateCluster(httpclient, &SessionCredentials, r)
 	if err != nil {
 		fmt.Println("Error: ", err)

pgo/cmd/create.go

@@ -32,7 +32,9 @@ var ArchiveFlag, DisableAutofailFlag, EnableAutofailFlag, PgbouncerFlag, Metrics
 var BackrestRestoreFrom string
 var CCPImage string
 var CCPImageTag string
+var CPURequest string
 var Database string
+var MemoryRequest string
 var Password string
 var SecretFrom string
 var PoliciesFlag, PolicyFile, PolicyURL string
@@ -254,10 +256,14 @@ func init() {
 	// flags for "pgo create cluster"
 	createClusterCmd.Flags().StringVarP(&CCPImage, "ccp-image", "", "", "The CCPImage name to use for cluster creation. If specified, overrides the value crunchy-postgres.")
 	createClusterCmd.Flags().StringVarP(&CCPImageTag, "ccp-image-tag", "c", "", "The CCPImageTag to use for cluster creation. If specified, overrides the pgo.yaml setting.")
+	createClusterCmd.Flags().StringVar(&CPURequest, "cpu", "", "Set the number of millicores to request for the CPU, e.g. "+
+		"\"100m\" or \"0.1\". Overrides the value in \"resources-config\"")
 	createClusterCmd.Flags().StringVarP(&CustomConfig, "custom-config", "", "", "The name of a configMap that holds custom PostgreSQL configuration files used to override defaults.")
 	createClusterCmd.Flags().StringVarP(&Database, "database", "d", "", "If specified, sets the name of the initial database that is created for the user. Defaults to the value set in the PostgreSQL Operator configuration, or if that is not present, the name of the cluster")
 	createClusterCmd.Flags().BoolVarP(&DisableAutofailFlag, "disable-autofail", "", false, "Disables autofail capabitilies in the cluster following cluster initialization.")
 	createClusterCmd.Flags().StringVarP(&UserLabels, "labels", "l", "", "The labels to apply to this cluster.")
+	createClusterCmd.Flags().StringVar(&MemoryRequest, "memory", "", "Set the amount of RAM to request, e.g. "+
+		"1GiB. Overrides the value in \"resources-config\"")
 	createClusterCmd.Flags().BoolVarP(&MetricsFlag, "metrics", "", false, "Adds the crunchy-collect container to the database pod.")
 	createClusterCmd.Flags().StringVarP(&NodeLabel, "node-label", "", "", "The node label (key=value) to use in placing the primary database. If not set, any node is used.")
 	createClusterCmd.Flags().StringVarP(&Password, "password", "", "", "The password to use for standard user account created during cluster initialization.")