Have pgBackRest and associated jobs respect cluster tolerations

Jonathan S. Katz · jkatz · commit 3c5143084e7e · 2021-01-13T15:42:22.000-05:00
pgBackRest is a close corollary to the PostgreSQL cluster itself
and should respect similar tolerations to the PostgreSQL cluster.

Future work could break out tolerations specific for pgBackRest,
but given the two do need to work in concert, it is prudent to
start with this approach.

Issue: [ch10146]
diff --git a/cmd/pgo-scheduler/scheduler/policy.go b/cmd/pgo-scheduler/scheduler/policy.go
@@ -142,6 +142,7 @@ func (p PolicyJob) Run() {
 		PGDatabase:     p.database,
 		PGSQLConfigMap: name,
 		PGUserSecret:   p.secret,
+		Tolerations:    util.GetTolerations(cluster.Spec.Tolerations),
 	}
 
 	var doc bytes.Buffer
diff --git a/cmd/pgo-scheduler/scheduler/types.go b/cmd/pgo-scheduler/scheduler/types.go
@@ -70,4 +70,5 @@ type PolicyTemplate struct {
 	PGDatabase     string
 	PGUserSecret   string
 	PGSQLConfigMap string
+	Tolerations    string
 }
diff --git a/installers/ansible/roles/pgo-operator/files/pgo-configs/backrest-job.json b/installers/ansible/roles/pgo-operator/files/pgo-configs/backrest-job.json
@@ -29,6 +29,9 @@
                 ],
                 "securityContext": {{.SecurityContext}},
                 "serviceAccountName": "pgo-backrest",
+                {{ if .Tolerations }}
+                "tolerations": {{ .Tolerations }},
+                {{ end }}
                 "containers": [{
                     "name": "backrest",
                     "image": "{{.CCPImagePrefix}}/crunchy-pgbackrest:{{.CCPImageTag}}",
diff --git a/installers/ansible/roles/pgo-operator/files/pgo-configs/cluster-bootstrap-job.json b/installers/ansible/roles/pgo-operator/files/pgo-configs/cluster-bootstrap-job.json
@@ -23,6 +23,9 @@
             "spec": {
                 "securityContext": {{.SecurityContext}},
                 "serviceAccountName": "pgo-pg",
+                {{ if .Tolerations }}
+                "tolerations": {{ .Tolerations }},
+                {{ end }}
                 "containers": [{
                     "name": "database",
                     "image": "{{.CCPImagePrefix}}/{{.CCPImage}}:{{.CCPImageTag}}",
diff --git a/installers/ansible/roles/pgo-operator/files/pgo-configs/pgdump-job.json b/installers/ansible/roles/pgo-operator/files/pgo-configs/pgdump-job.json
@@ -31,6 +31,9 @@
                 ],
                 "securityContext": {{.SecurityContext}},
                 "serviceAccountName": "pgo-default",
+                {{ if .Tolerations }}
+                "tolerations": {{ .Tolerations }},
+                {{ end }}
                 "containers": [{
                         "name": "pgdump",
                         "image": "{{.CCPImagePrefix}}/crunchy-postgres-ha:{{.CCPImageTag}}",
diff --git a/installers/ansible/roles/pgo-operator/files/pgo-configs/pgo-backrest-repo-template.json b/installers/ansible/roles/pgo-operator/files/pgo-configs/pgo-backrest-repo-template.json
@@ -46,6 +46,9 @@
             "spec": {
                 "securityContext": {{.SecurityContext}},
                 "serviceAccountName": "pgo-default",
+                {{ if .Tolerations }}
+                "tolerations": {{ .Tolerations }},
+                {{ end }}
                 "containers": [{
                     "name": "database",
                     "image": "{{.CCPImagePrefix}}/crunchy-pgbackrest-repo:{{.CCPImageTag}}",
diff --git a/installers/ansible/roles/pgo-operator/files/pgo-configs/pgo.sqlrunner-template.json b/installers/ansible/roles/pgo-operator/files/pgo-configs/pgo.sqlrunner-template.json
@@ -21,6 +21,9 @@
             },
             "spec": {
                 "serviceAccountName": "pgo-default",
+                {{ if .Tolerations }}
+                "tolerations": {{ .Tolerations }},
+                {{ end }}
                 "containers": [
                     {
                         "name": "sqlrunner",
diff --git a/installers/ansible/roles/pgo-operator/files/pgo-configs/pgrestore-job.json b/installers/ansible/roles/pgo-operator/files/pgo-configs/pgrestore-job.json
@@ -31,6 +31,9 @@
                 ],
                 "securityContext": {{.SecurityContext}},
                 "serviceAccountName": "pgo-default",
+                {{ if .Tolerations }}
+                "tolerations": {{ .Tolerations }},
+                {{ end }}
                 "containers": [
                     {
                         "name": "pgrestore",
diff --git a/installers/ansible/roles/pgo-operator/files/pgo-configs/rmdata-job.json b/installers/ansible/roles/pgo-operator/files/pgo-configs/rmdata-job.json
@@ -21,6 +21,9 @@
             },
             "spec": {
                 "serviceAccountName": "pgo-target",
+                {{ if .Tolerations }}
+                "tolerations": {{ .Tolerations }},
+                {{ end }}
                 "containers": [{
                     "name": "rmdata",
                     "image": "{{.PGOImagePrefix}}/pgo-rmdata:{{.PGOImageTag}}",
diff --git a/internal/config/labels.go b/internal/config/labels.go
@@ -60,6 +60,7 @@ const (
 	LABEL_DELETE_BACKUPS      = "delete-backups"
 	LABEL_IS_REPLICA          = "is-replica"
 	LABEL_IS_BACKUP           = "is-backup"
+	LABEL_RM_TOLERATIONS      = "rmdata-tolerations"
 	LABEL_STARTUP             = "startup"
 	LABEL_SHUTDOWN            = "shutdown"
 )
diff --git a/internal/operator/backrest/backup.go b/internal/operator/backrest/backup.go
@@ -61,6 +61,7 @@ type backrestJobTemplateFields struct {
 	PgbackrestS3VerifyTLS         string
 	PgbackrestRestoreVolumes      string
 	PgbackrestRestoreVolumeMounts string
+	Tolerations                   string
 }
 
 var (
@@ -117,6 +118,7 @@ func Backrest(namespace string, clientset kubeapi.Interface, task *crv1.Pgtask)
 		PgbackrestRepo1Type:           repoType,
 		BackrestLocalAndS3Storage:     operator.IsLocalAndS3Storage(cluster),
 		PgbackrestS3VerifyTLS:         task.Spec.Parameters[config.LABEL_BACKREST_S3_VERIFY_TLS],
+		Tolerations:                   util.GetTolerations(cluster.Spec.Tolerations),
 	}
 
 	podCommandOpts, err := getCommandOptsFromPod(clientset, task, namespace)
diff --git a/internal/operator/backrest/repo.go b/internal/operator/backrest/repo.go
@@ -65,6 +65,7 @@ type RepoDeploymentTemplateFields struct {
 	PodAntiAffinityLabelValue string
 	Replicas                  int
 	BootstrapCluster          string
+	Tolerations               string
 }
 
 type RepoServiceTemplateFields struct {
@@ -250,6 +251,7 @@ func getRepoDeploymentFields(clientset kubernetes.Interface, cluster *crv1.Pgclu
 		PodAntiAffinityLabelName: config.LABEL_POD_ANTI_AFFINITY,
 		PodAntiAffinityLabelValue: string(operator.GetPodAntiAffinityType(cluster,
 			crv1.PodAntiAffinityDeploymentPgBackRest, cluster.Spec.PodAntiAffinity.PgBackRest)),
+		Tolerations: util.GetTolerations(cluster.Spec.Tolerations),
 	}
 
 	return &repoFields
diff --git a/internal/operator/cluster/clusterlogic.go b/internal/operator/cluster/clusterlogic.go
@@ -347,7 +347,7 @@ func getClusterDeploymentFields(clientset kubernetes.Interface,
 		ReplicationTLSSecret:      cl.Spec.TLS.ReplicationTLSSecret,
 		CASecret:                  cl.Spec.TLS.CASecret,
 		Standby:                   cl.Spec.Standby,
-		Tolerations:               operator.GetTolerations(cl.Spec.Tolerations),
+		Tolerations:               util.GetTolerations(cl.Spec.Tolerations),
 	}
 
 	return deploymentFields
@@ -491,8 +491,8 @@ func scaleReplicaCreateDeployment(clientset kubernetes.Interface,
 		// Give precedence to the tolerations defined on the replica spec, otherwise
 		// take any tolerations defined on the cluster spec
 		Tolerations: util.GetValueOrDefault(
-			operator.GetTolerations(replica.Spec.Tolerations),
-			operator.GetTolerations(cluster.Spec.Tolerations)),
+			util.GetTolerations(replica.Spec.Tolerations),
+			util.GetTolerations(cluster.Spec.Tolerations)),
 	}
 
 	switch replica.Spec.ReplicaStorage.StorageType {
diff --git a/internal/operator/clusterutilities.go b/internal/operator/clusterutilities.go
@@ -955,25 +955,6 @@ func GetSyncReplication(specSyncReplication *bool) bool {
 	return Pgo.Cluster.SyncReplication
 }
 
-// GetTolerations returns any tolerations that may be defined in a tolerations
-// in JSON format. Otherwise, it returns an empty string
-func GetTolerations(tolerations []v1.Toleration) string {
-	// if no tolerations, exit early
-	if len(tolerations) == 0 {
-		return ""
-	}
-
-	// turn into a JSON string
-	s, err := json.MarshalIndent(tolerations, "", "  ")
-
-	if err != nil {
-		log.Errorf("%s: returning empty string", err.Error())
-		return ""
-	}
-
-	return string(s)
-}
-
 // OverrideClusterContainerImages is a helper function that provides the
 // appropriate hooks to override any of the container images that might be
 // deployed with a PostgreSQL cluster
diff --git a/internal/operator/pgdump/dump.go b/internal/operator/pgdump/dump.go
@@ -53,6 +53,7 @@ type pgDumpJobTemplateFields struct {
 	PgDumpFilename   string
 	PgDumpAll        string
 	PgDumpPVC        string
+	Tolerations      string
 }
 
 // Dump ...
@@ -118,6 +119,7 @@ func Dump(namespace string, clientset kubeapi.Interface, task *crv1.Pgtask) {
 		PgDumpOpts:       task.Spec.Parameters[config.LABEL_PGDUMP_OPTS],
 		PgDumpAll:        task.Spec.Parameters[config.LABEL_PGDUMP_ALL],
 		PgDumpPVC:        pvcName,
+		Tolerations:      util.GetTolerations(cluster.Spec.Tolerations),
 	}
 
 	var doc2 bytes.Buffer
diff --git a/internal/operator/pgdump/restore.go b/internal/operator/pgdump/restore.go
@@ -51,6 +51,7 @@ type restorejobTemplateFields struct {
 	CCPImageTag         string
 	PgPort              string
 	NodeSelector        string
+	Tolerations         string
 }
 
 // Restore ...
@@ -106,6 +107,7 @@ func Restore(namespace string, clientset kubeapi.Interface, task *crv1.Pgtask) {
 		CCPImagePrefix:      util.GetValueOrDefault(cluster.Spec.CCPImagePrefix, operator.Pgo.Cluster.CCPImagePrefix),
 		CCPImageTag:         operator.Pgo.Cluster.CCPImageTag,
 		NodeSelector:        operator.GetNodeAffinity(nodeAffinity),
+		Tolerations:         util.GetTolerations(cluster.Spec.Tolerations),
 	}
 
 	var doc2 bytes.Buffer
diff --git a/internal/operator/task/rmdata.go b/internal/operator/task/rmdata.go
@@ -47,6 +47,7 @@ type rmdatajobTemplateFields struct {
 	RemoveBackup     string
 	IsBackup         string
 	IsReplica        string
+	Tolerations      string
 }
 
 // RemoveData ...
@@ -98,6 +99,7 @@ func RemoveData(namespace string, clientset kubeapi.Interface, task *crv1.Pgtask
 		PGOImagePrefix:   util.GetValueOrDefault(task.Spec.Parameters[config.LABEL_IMAGE_PREFIX], operator.Pgo.Pgo.PGOImagePrefix),
 		PGOImageTag:      operator.Pgo.Pgo.PGOImageTag,
 		SecurityContext:  operator.GetPodSecurityContext(task.Spec.StorageSpec.GetSupplementalGroups()),
+		Tolerations:      task.Spec.Parameters[config.LABEL_RM_TOLERATIONS],
 	}
 
 	log.Debugf("creating rmdata job %s for cluster %s ", jobName, task.Spec.Name)
diff --git a/internal/util/cluster.go b/internal/util/cluster.go
@@ -17,6 +17,7 @@ package util
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"strconv"
@@ -262,6 +263,7 @@ func CreateRMDataTask(clientset kubeapi.Interface, cluster *crv1.Pgcluster, repl
 				config.LABEL_PG_CLUSTER:     cluster.Name,
 				config.LABEL_REPLICA_NAME:   replicaName,
 				config.LABEL_PGHA_SCOPE:     cluster.ObjectMeta.GetLabels()[config.LABEL_PGHA_SCOPE],
+				config.LABEL_RM_TOLERATIONS: GetTolerations(cluster.Spec.Tolerations),
 			},
 			TaskType: crv1.PgtaskDeleteData,
 		},
@@ -379,6 +381,25 @@ func GetS3CredsFromBackrestRepoSecret(clientset kubernetes.Interface, namespace,
 	return s3Secret, nil
 }
 
+// GetTolerations returns any tolerations that may be defined in a tolerations
+// in JSON format. Otherwise, it returns an empty string
+func GetTolerations(tolerations []v1.Toleration) string {
+	// if no tolerations, exit early
+	if len(tolerations) == 0 {
+		return ""
+	}
+
+	// turn into a JSON string
+	s, err := json.MarshalIndent(tolerations, "", "  ")
+
+	if err != nil {
+		log.Errorf("%s: returning empty string", err.Error())
+		return ""
+	}
+
+	return string(s)
+}
+
 // SetPostgreSQLPassword updates the password for a PostgreSQL role in the
 // PostgreSQL cluster by executing into the primary Pod and changing it
 //

Original file line number	Diff line number	Diff line change
`@@ -142,6 +142,7 @@ func (p PolicyJob) Run() {`
`142`	`142`	`PGDatabase: p.database,`
`143`	`143`	`PGSQLConfigMap: name,`
`144`	`144`	`PGUserSecret: p.secret,`
	`145`	`+ Tolerations: util.GetTolerations(cluster.Spec.Tolerations),`
`145`	`146`	`}`
`146`	`147`
`147`	`148`	`var doc bytes.Buffer`
Original file line number	Diff line number	Diff line change
`@@ -70,4 +70,5 @@ type PolicyTemplate struct {`
`70`	`70`	`PGDatabase string`
`71`	`71`	`PGUserSecret string`
`72`	`72`	`PGSQLConfigMap string`
	`73`	`+ Tolerations string`
`73`	`74`	`}`
Original file line number	Diff line number	Diff line change
`@@ -60,6 +60,7 @@ const (`
`60`	`60`	`LABEL_DELETE_BACKUPS = "delete-backups"`
`61`	`61`	`LABEL_IS_REPLICA = "is-replica"`
`62`	`62`	`LABEL_IS_BACKUP = "is-backup"`
	`63`	`+ LABEL_RM_TOLERATIONS = "rmdata-tolerations"`
`63`	`64`	`LABEL_STARTUP = "startup"`
`64`	`65`	`LABEL_SHUTDOWN = "shutdown"`
`65`	`66`	`)`