Allow tablespaces to be added to existing PostgreSQL clusters

This commit introduces the "pgo update cluster --tablespace"
flag, which allows for users to add additional tablespaces to a
PostgreSQL cluster.

The Operator takes care of creating PVCs for the additional PVCs
for the primary and all replicas in the PostgreSQL cluster, as well
as updating the Deployment template for each instance. This does
have the side effect of causing the PostgreSQL Pods to restart,
as the new volumes need to be mounted.

Issue: [ch7232]

Author: Jonathan S. Katz

apiserver/clusterservice/clusterimpl.go

@@ -557,23 +557,10 @@ func CreateCluster(request *msgs.CreateClusterRequest, ns, pgouser string) msgs.
 	// validate the storage type for each specified tablespace actually exists.
 	// if a PVCSize is passed in, also validate that it follows the Kubernetes
 	// format
-	if len(request.Tablespaces) > 0 {
-		for _, tablespace := range request.Tablespaces {
-			if !apiserver.IsValidStorageName(tablespace.StorageConfig) {
-				resp.Status.Code = msgs.Error
-				resp.Status.Msg = fmt.Sprintf("%s storage config for tablespace %s was not found",
-					tablespace.StorageConfig, tablespace.Name)
-				return resp
-			}
-
-			if tablespace.PVCSize != "" {
-				if err := apiserver.ValidateQuantity(tablespace.PVCSize); err != nil {
-					resp.Status.Code = msgs.Error
-					resp.Status.Msg = fmt.Sprintf(apiserver.ErrMessagePVCSize, tablespace.PVCSize, err.Error())
-					return resp
-				}
-			}
-		}
+	if err := validateTablespaces(request.Tablespaces); err != nil {
+		resp.Status.Code = msgs.Error
+		resp.Status.Msg = err.Error()
+		return resp
 	}
 
 	// validate the TLS parameters for enabling TLS in a PostgreSQL cluster
@@ -987,24 +974,20 @@ func getClusterParams(request *msgs.CreateClusterRequest, name string, userLabel
 		spec.PrimaryStorage.Size = request.PVCSize
 	}
 
-	// extract the parameters for th TablespacEMounts and put them in the format
+	// extract the parameters for the TablespaceMounts and put them in the format
 	// that is required by the pgcluster CRD
-	if len(request.Tablespaces) > 0 {
-		tablespaceMountsMap := map[string]crv1.PgStorageSpec{}
+	spec.TablespaceMounts = map[string]crv1.PgStorageSpec{}
 
-		for _, tablespace := range request.Tablespaces {
-			storageSpec, _ := apiserver.Pgo.GetStorageSpec(tablespace.StorageConfig)
-
-			// if a PVCSize is specified, override the value of the Size parameter in
-			// storage spec
-			if tablespace.PVCSize != "" {
-				storageSpec.Size = tablespace.PVCSize
-			}
+	for _, tablespace := range request.Tablespaces {
+		storageSpec, _ := apiserver.Pgo.GetStorageSpec(tablespace.StorageConfig)
 
-			tablespaceMountsMap[tablespace.Name] = storageSpec
+		// if a PVCSize is specified, override the value of the Size parameter in
+		// storage spec
+		if tablespace.PVCSize != "" {
+			storageSpec.Size = tablespace.PVCSize
 		}
 
-		spec.TablespaceMounts = tablespaceMountsMap
+		spec.TablespaceMounts[tablespace.Name] = storageSpec
 	}
 
 	spec.ReplicaStorage, _ = apiserver.Pgo.GetStorageSpec(apiserver.Pgo.ReplicaStorage)
@@ -1423,6 +1406,15 @@ func UpdateCluster(request *msgs.UpdateClusterRequest) msgs.UpdateClusterRespons
 		return response
 	}
 
+	// validate the storage type for each specified tablespace actually exists.
+	// if a PVCSize is passed in, also validate that it follows the Kubernetes
+	// format
+	if err := validateTablespaces(request.Tablespaces); err != nil {
+		response.Status.Code = msgs.Error
+		response.Status.Msg = err.Error()
+		return response
+	}
+
 	clusterList := crv1.PgclusterList{}
 
 	//get the clusters list
@@ -1502,9 +1494,21 @@ func UpdateCluster(request *msgs.UpdateClusterRequest) msgs.UpdateClusterRespons
 			cluster.Spec.Shutdown = true
 		}
 
-		err = kubeapi.Updatepgcluster(apiserver.RESTClient,
-			&cluster, cluster.Spec.Name, request.Namespace)
-		if err != nil {
+		// extract the parameters for the TablespaceMounts and put them in the
+		// format that is required by the pgcluster CRD
+		for _, tablespace := range request.Tablespaces {
+			storageSpec, _ := apiserver.Pgo.GetStorageSpec(tablespace.StorageConfig)
+
+			// if a PVCSize is specified, override the value of the Size parameter in
+			// storage spec
+			if tablespace.PVCSize != "" {
+				storageSpec.Size = tablespace.PVCSize
+			}
+
+			cluster.Spec.TablespaceMounts[tablespace.Name] = storageSpec
+		}
+
+		if err := kubeapi.Updatepgcluster(apiserver.RESTClient, &cluster, cluster.Spec.Name, request.Namespace); err != nil {
 			response.Status.Code = msgs.Error
 			response.Status.Msg = err.Error()
 			return response
@@ -1622,6 +1626,27 @@ func validateClusterTLS(request *msgs.CreateClusterRequest) error {
 	return nil
 }
 
+// validateTablespaces validates the tablespace parameters. if there is an error
+// it aborts and returns an error
+func validateTablespaces(tablespaces []msgs.ClusterTablespaceDetail) error {
+	// iterate through the list of tablespaces and return any erors
+	for _, tablespace := range tablespaces {
+		if !apiserver.IsValidStorageName(tablespace.StorageConfig) {
+			return fmt.Errorf("%s storage config for tablespace %s was not found",
+				tablespace.StorageConfig, tablespace.Name)
+		}
+
+		if tablespace.PVCSize != "" {
+			if err := apiserver.ValidateQuantity(tablespace.PVCSize); err != nil {
+				return fmt.Errorf(apiserver.ErrMessagePVCSize,
+					tablespace.PVCSize, err.Error())
+			}
+		}
+	}
+
+	return nil
+}
+
 // determines if any of the required S3 configuration settings (bucket, endpoint
 // and region) are missing from both the incoming request or the pgo.yaml config file
 func isMissingS3Config(request *msgs.CreateClusterRequest) bool {

apiservermsgs/clustermsgs.go

@@ -273,6 +273,7 @@ type UpdateClusterRequest struct {
 	Standby       UpdateClusterStandbyStatus
 	Startup       bool
 	Shutdown      bool
+	Tablespaces   []ClusterTablespaceDetail
 }
 
 // UpdateClusterResponse ...

controller/pgcluster/pgclustercontroller.go

@@ -19,6 +19,7 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"reflect"
 	"strconv"
 	"strings"
 	"sync"
@@ -212,6 +213,15 @@ func (c *Controller) onUpdate(oldObj, newObj interface{}) {
 			return
 		}
 	}
+
+	// if we are not in a standby state, check to see if the tablespaces have
+	// differed, and if so, add the additional volumes to the primary and replicas
+	if !reflect.DeepEqual(oldcluster.Spec.TablespaceMounts, newcluster.Spec.TablespaceMounts) {
+		if err := updateTablespaces(c, oldcluster, newcluster); err != nil {
+			log.Error(err)
+			return
+		}
+	}
 }
 
 // onDelete is called when a pgcluster is deleted
@@ -314,3 +324,131 @@ func addIdentifier(clusterCopy *crv1.Pgcluster) {
 
 	clusterCopy.ObjectMeta.Labels[config.LABEL_PG_CLUSTER_IDENTIFIER] = string(u[:len(u)-1])
 }
+
+// updateTablespaces updates the PostgreSQL instance Deployments to reflect the
+// new PostgreSQL tablespaces that should be added
+func updateTablespaces(c *Controller, oldCluster *crv1.Pgcluster, newCluster *crv1.Pgcluster) error {
+	// first, get a list of all of the available deployments so we can properly
+	// mount the tablespace PVCs after we create them
+	// NOTE: this will also get the pgBackRest deployments, but we will filter
+	// these out later
+	selector := fmt.Sprintf("%s=%s,%s=%s", config.LABEL_VENDOR, config.LABEL_CRUNCHY,
+		config.LABEL_PG_CLUSTER, newCluster.Name)
+
+	deployments, err := kubeapi.GetDeployments(c.PgclusterClientset, selector, newCluster.Namespace)
+
+	if err != nil {
+		return err
+	}
+
+	// now get the instance names, which will make it easier to create all the
+	// PVCs
+	instanceNames := []string{}
+
+	for _, deployment := range deployments.Items {
+		labels := deployment.ObjectMeta.GetLabels()
+
+		// get the name of the PostgreSQL instance. If the "deployment-name"
+		// label is not present, then we know it's not a PostgreSQL cluster.
+		// Otherwise, the "deployment-name" label doubles as the name of the
+		// instance
+		if instanceName, ok := labels[config.LABEL_DEPLOYMENT_NAME]; ok {
+			log.Debugf("instance found [%s]", instanceName)
+
+			instanceNames = append(instanceNames, instanceName)
+		}
+	}
+
+	// iterate through the the tablespace mount map that is present in the new
+	// cluster. Any entry that is not in the old cluster, create PVCs
+	newTablespaces := map[string]crv1.PgStorageSpec{}
+
+	for tablespaceName, storageSpec := range newCluster.Spec.TablespaceMounts {
+		// if the tablespace does not exist in the old version of the cluster,
+		// then add it in!
+		if _, ok := oldCluster.Spec.TablespaceMounts[tablespaceName]; !ok {
+			log.Debugf("new tablespace found: [%s]", tablespaceName)
+
+			newTablespaces[tablespaceName] = storageSpec
+		}
+	}
+
+	// now we can start creating the new tablespaces! First, create the new
+	// PVCs. The PVCs are created for each **instance** in the cluster, as every
+	// instance needs to have a distinct PVC for each tablespace
+	for tablespaceName, storageSpec := range newTablespaces {
+		for _, instanceName := range instanceNames {
+			// get the name of the tablespace PVC for that instance
+			tablespacePVCName := operator.GetTablespacePVCName(instanceName, tablespaceName)
+
+			log.Debugf("creating tablespace PVC [%s] for [%s]", tablespacePVCName, instanceName)
+
+			// and now create it! If it errors, we just need to return, which
+			// potentially leaves things in an inconsistent state, but at this point
+			// only PVC objects have been created
+			if err := clusteroperator.CreateTablespacePVC(c.PgclusterClientset, newCluster.Namespace, newCluster.Name,
+				tablespacePVCName, &storageSpec); err != nil {
+				return err
+			}
+		}
+	}
+
+	// now the fun step: update each deployment with the new volumes
+	for _, deployment := range deployments.Items {
+		labels := deployment.ObjectMeta.GetLabels()
+
+		// same deal as before: if this is not a PostgreSQL instance, skip it
+		instanceName, ok := labels[config.LABEL_DEPLOYMENT_NAME]
+		if !ok {
+			continue
+		}
+
+		log.Debugf("attach tablespace volumes to [%s]", instanceName)
+
+		// iterate through each table space and prepare the Volume and
+		// VolumeMount clause for each instance
+		for tablespaceName, _ := range newTablespaces {
+			// this is the volume to be added for the tablespace
+			volume := v1.Volume{
+				Name: operator.GetTablespaceVolumeName(tablespaceName),
+				VolumeSource: v1.VolumeSource{
+					PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
+						ClaimName: operator.GetTablespacePVCName(instanceName, tablespaceName),
+					},
+				},
+			}
+
+			// add the volume to the list of volumes
+			deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, volume)
+
+			// now add the volume mount point to that of the database container
+			volumeMount := v1.VolumeMount{
+				MountPath: fmt.Sprintf("%s%s", config.VOLUME_TABLESPACE_PATH_PREFIX, tablespaceName),
+				Name:      operator.GetTablespaceVolumeName(tablespaceName),
+			}
+
+			// we can do this as we always know that the "database" contianer is the
+			// first container in the list
+			deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(
+				deployment.Spec.Template.Spec.Containers[0].VolumeMounts, volumeMount)
+		}
+
+		// find the "PGHA_TABLESPACES" value and update it with the new tablespace
+		// name list
+		for i, envVar := range deployment.Spec.Template.Spec.Containers[0].Env {
+			// yup, it's an old fashioned linear time lookup
+			if envVar.Name == "PGHA_TABLESPACES" {
+				deployment.Spec.Template.Spec.Containers[0].Env[i].Value = operator.GetTablespaceNames(
+					newCluster.Spec.TablespaceMounts)
+			}
+		}
+
+		// finally, update the Deployment. Potential to put things into an
+		// inconsistent state if any of these updates fail
+		if err := kubeapi.UpdateDeployment(c.PgclusterClientset, &deployment); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

docs/content/architecture/tablespaces.md

@@ -73,7 +73,7 @@ a recovery
 - Deprovisioining: Tablespaces are deleted when a PostgreSQL instance or cluster
 is deleted
 
-## Creating Tablespaces
+## Adding Tablespaces to a New Cluster
 
 Tablespaces can be used in a cluster with the [`pgo create cluster`](/pgo-client/reference/pgo_create_cluster/)
 command. The command follows this general format:
@@ -107,5 +107,27 @@ CREATE TABLE sensor_data (
 TABLESPACE faststorage1;
 ```
 
+## Adding Tablespaces to Existing Clusters
+
+You can also add a tablespace to an existing PostgreSQL cluster with the
+[`pgo update cluster`](/pgo-client/reference/pgo_update_cluster/) command.
+Adding a tablespace to a cluster uses a similar syntax to creating a cluster
+with tablespaces, for example:
+
+```shell
+pgo update cluster hacluster \
+    --tablespace=name=tablespace3:storageconfig=storageconfigname
+```
+
+**NOTE**: This operation can cause downtime. In order to add a tablespace to a
+PostgreSQL cluster, persistent volume claims (PVCs) need to be created and
+mounted to each PostgreSQL instance in the cluster. The act of mounting a new
+PVC to a Kubernetes Deployment causes the Pods in the deployment to restart.
+
+When the operation completes, the tablespace will be set up and accessible to
+use within the PostgreSQL cluster.
+
+## More Information
+
 For more information on how tablespaces work in PostgreSQL please refer to the
 [PostgreSQL manual](https://www.postgresql.org/docs/current/manage-ag-tablespaces.html).

docs/content/pgo-client/common-tasks.md

@@ -826,6 +826,38 @@ pgo scaledown hacluster --target=hacluster-abcd
 where `hacluster-abcd` is the name of the PostgreSQL replica that you want to
 destroy.
 
+## Cluster Maintenance & Resource Management
+
+There are several operations that you can perform to modify a PostgreSQL cluster
+over its lifetime.
+
+#### Adding a Tablespace to a Cluster
+
+Based on your workload or volume of data, you may wish to add a
+[tablespace](https://www.postgresql.org/docs/current/manage-ag-tablespaces.html) to
+your PostgreSQL cluster.
+
+You can add a tablespace to an existing PostgreSQL cluster with the
+[`pgo update cluster`](/pgo-client/reference/pgo_update_cluster/) command.
+Adding a tablespace to a cluster uses a similar syntax to
+[creating a cluster with a tablespace](#create-a-postgresql-cluster-with-a-tablespace), for example:
+
+```shell
+pgo update cluster hacluster \
+    --tablespace=name=tablespace3:storageconfig=storageconfigname
+```
+
+**NOTE**: This operation can cause downtime. In order to add a tablespace to a
+PostgreSQL cluster, persistent volume claims (PVCs) need to be created and
+mounted to each PostgreSQL instance in the cluster. The act of mounting a new
+PVC to a Kubernetes Deployment causes the Pods in the deployment to restart.
+
+When the operation completes, the tablespace will be set up and accessible to
+use within the PostgreSQL cluster.
+
+For more information on tablespaces, please visit the [tablespace](/architecture/tablespaces/)
+section of the documentation.
+
 ## Clone a PostgreSQL Cluster
 
 You can create a copy of an existing PostgreSQL cluster in a new PostgreSQL