Enable user-defined forced failovers

While this was the default behavior in Operator's past when using
`pgo failover`, a previous commit changed the Operator internals
to leverage a "controlled switchover" which is a bit nicer (i.e.
it only works if there is a healthy instance to fail over to).
However, there are situations where one must force a failover,
and as such, we need to allow for one to do so.

This adds the `--force` flag to `pgo failover` to allow for a
forced failover. Note that `--target` must explicitly be set
when forcing a failover.

Author: Jonathan S. Katz

cmd/pgo/cmd/failover.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/crunchydata/postgres-operator/cmd/pgo/api"
 	"github.com/crunchydata/postgres-operator/cmd/pgo/util"
@@ -60,29 +61,38 @@ var failoverCmd = &cobra.Command{
 func init() {
 	RootCmd.AddCommand(failoverCmd)
 
-	failoverCmd.Flags().BoolVarP(&Query, "query", "", false, "Prints the list of failover candidates.")
+	failoverCmd.Flags().BoolVar(&Force, "force", false, "Force the failover to occur, regardless "+
+		"of the health of the target instance. Must be used with \"--target\".")
 	failoverCmd.Flags().BoolVar(&NoPrompt, "no-prompt", false, "No command line confirmation.")
+	failoverCmd.Flags().BoolVar(&Query, "query", false, "Prints the list of failover candidates.")
 	failoverCmd.Flags().StringVarP(&Target, "target", "", "", "The replica target which the failover will occur on.")
 }
 
 // createFailover ....
 func createFailover(args []string, ns string) {
 	log.Debugf("createFailover called %v", args)
 
-	request := new(msgs.CreateFailoverRequest)
-	request.Namespace = ns
-	request.ClusterName = args[0]
-	request.Target = Target
-	request.ClientVersion = msgs.PGO_VERSION
+	request := &msgs.CreateFailoverRequest{
+		ClientVersion: msgs.PGO_VERSION,
+		ClusterName:   args[0],
+		Force:         Force,
+		Namespace:     ns,
+		Target:        Target,
+	}
 
 	response, err := api.CreateFailover(httpclient, &SessionCredentials, request)
 	if err != nil {
 		fmt.Println("Error: " + err.Error())
-		os.Exit(2)
+		os.Exit(1)
 	}
 
 	if response.Status.Code != msgs.Ok {
-		fmt.Println("Error: " + response.Status.Msg)
+		fmt.Println("Error:", strings.ReplaceAll(response.Status.Msg, "Error: ", ""))
+
+		if strings.Contains(response.Status.Msg, "no primary") {
+			fmt.Println(`Hint: Try using the "--force" flag`)
+		}
+
 		os.Exit(1)
 	}
 

cmd/pgo/cmd/flags.go

@@ -22,7 +22,16 @@ var DeleteData bool
 // even after a cluster is deleted. This is DEPRECATED
 var KeepData bool
 
-var Query bool
+var (
+	// Force indicates that the "force" action should be taken for that step. This
+	// is different than NoPrompt as "Force" is for indicating that the API server
+	// must try at all costs
+	Force bool
+
+	// Query indicates that the attempted request is "querying" information
+	// instead of taking some action
+	Query bool
+)
 
 var (
 	Target  string

docs/content/pgo-client/reference/pgo_failover.md

@@ -23,6 +23,7 @@ pgo failover [flags]
 ### Options
 
 ```
+      --force           Force the failover to occur, regardless of the health of the target instance. Must be used with "--target".
   -h, --help            help for failover
       --no-prompt       No command line confirmation.
       --query           Prints the list of failover candidates.
@@ -46,4 +47,4 @@ pgo failover [flags]
 
 * [pgo](/pgo-client/reference/pgo/)	 - The pgo command line interface.
 
-###### Auto generated by spf13/cobra on 1-Jan-2021
+###### Auto generated by spf13/cobra on 4-Jan-2021

internal/apiserver/failoverservice/failoverimpl.go

@@ -19,15 +19,19 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 
 	"github.com/crunchydata/postgres-operator/internal/apiserver"
 	"github.com/crunchydata/postgres-operator/internal/config"
 	"github.com/crunchydata/postgres-operator/internal/operator"
 	"github.com/crunchydata/postgres-operator/internal/util"
 	crv1 "github.com/crunchydata/postgres-operator/pkg/apis/crunchydata.com/v1"
 	msgs "github.com/crunchydata/postgres-operator/pkg/apiservermsgs"
+
 	log "github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
 )
 
 // CreateFailover is the API endpoint for triggering a manual failover of a
@@ -60,18 +64,24 @@ func CreateFailover(request *msgs.CreateFailoverRequest, ns, pgouser string) msg
 		return resp
 	}
 
-	if request.Target != "" {
-		if err := isValidFailoverTarget(request.Target, request.ClusterName, ns); err != nil {
-			resp.Status.Code = msgs.Error
-			resp.Status.Msg = err.Error()
-			return resp
-		}
+	if err := isValidFailoverTarget(request); err != nil {
+		resp.Status.Code = msgs.Error
+		resp.Status.Msg = err.Error()
+		return resp
 	}
 
-	// perform the switchover
-	if err := operator.Switchover(apiserver.Clientset, apiserver.RESTConfig, cluster, request.Target); err != nil {
+	// perform the switchover or failover, depending on which flag is selected
+	// if we are forcing the failover, we need to use "Failover", otherwise we
+	// perform a controlled switchover
+	if request.Force {
+		err = operator.Failover(apiserver.Clientset, apiserver.RESTConfig, cluster, request.Target)
+	} else {
+		err = operator.Switchover(apiserver.Clientset, apiserver.RESTConfig, cluster, request.Target)
+	}
+
+	if err != nil {
 		resp.Status.Code = msgs.Error
-		resp.Status.Msg = err.Error()
+		resp.Status.Msg = strings.ReplaceAll(err.Error(), "master", "primary")
 		return resp
 	}
 
@@ -161,31 +171,53 @@ func validateClusterName(clusterName, ns string) (*crv1.Pgcluster, error) {
 // specified, and then ensuring the PG pod created by the deployment is not the current primary.
 // If the deployment is not found, or if the pod is the current primary, an error will be returned.
 // Otherwise the deployment is returned.
-func isValidFailoverTarget(deployName, clusterName, ns string) error {
+func isValidFailoverTarget(request *msgs.CreateFailoverRequest) error {
 	ctx := context.TODO()
 
+	// if we're not forcing a failover and the target is blank, we can
+	// return here
+	// However, if we are forcing a failover and the target is blank, then we do
+	// have an error
+	if request.Target == "" {
+		if !request.Force {
+			return nil
+		}
+
+		return fmt.Errorf("target is required when forcing a failover.")
+	}
+
 	// Using the following label selector, ensure the deployment specified using deployName exists in the
 	// cluster specified using clusterName:
 	// pg-cluster=clusterName,deployment-name=deployName
-	selector := config.LABEL_PG_CLUSTER + "=" + clusterName + "," + config.LABEL_DEPLOYMENT_NAME + "=" + deployName
-	deployments, err := apiserver.Clientset.
-		AppsV1().Deployments(ns).
-		List(ctx, metav1.ListOptions{LabelSelector: selector})
+	options := metav1.ListOptions{
+		LabelSelector: fields.AndSelectors(
+			fields.OneTermEqualSelector(config.LABEL_PG_CLUSTER, request.ClusterName),
+			fields.OneTermEqualSelector(config.LABEL_DEPLOYMENT_NAME, request.Target),
+		).String(),
+	}
+	deployments, err := apiserver.Clientset.AppsV1().Deployments(request.Namespace).List(ctx, options)
+
 	if err != nil {
 		log.Error(err)
 		return err
 	} else if len(deployments.Items) == 0 {
-		return fmt.Errorf("no target found named %s", deployName)
+		return fmt.Errorf("no target found named %s", request.Target)
 	} else if len(deployments.Items) > 1 {
-		return fmt.Errorf("more than one target found named %s", deployName)
+		return fmt.Errorf("more than one target found named %s", request.Target)
 	}
 
 	// Using the following label selector, determine if the target specified is the current
 	// primary for the cluster and return an error if it is:
 	// pg-cluster=clusterName,deployment-name=deployName,role=primary
-	selector = config.LABEL_PG_CLUSTER + "=" + clusterName + "," + config.LABEL_DEPLOYMENT_NAME + "=" + deployName +
-		"," + config.LABEL_PGHA_ROLE + "=" + config.LABEL_PGHA_ROLE_PRIMARY
-	pods, _ := apiserver.Clientset.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: selector})
+	options.FieldSelector = fields.OneTermEqualSelector("status.phase", string(v1.PodRunning)).String()
+	options.LabelSelector = fields.AndSelectors(
+		fields.OneTermEqualSelector(config.LABEL_PG_CLUSTER, request.ClusterName),
+		fields.OneTermEqualSelector(config.LABEL_DEPLOYMENT_NAME, request.Target),
+		fields.OneTermEqualSelector(config.LABEL_PGHA_ROLE, config.LABEL_PGHA_ROLE_PRIMARY),
+	).String()
+
+	pods, _ := apiserver.Clientset.CoreV1().Pods(request.Namespace).List(ctx, options)
+
 	if len(pods.Items) > 0 {
 		return fmt.Errorf("The primary database cannot be selected as a failover target")
 	}

internal/controller/job/backresthandler.go

@@ -26,8 +26,8 @@ import (
 	"github.com/crunchydata/postgres-operator/internal/config"
 	"github.com/crunchydata/postgres-operator/internal/controller"
 	"github.com/crunchydata/postgres-operator/internal/kubeapi"
+	"github.com/crunchydata/postgres-operator/internal/operator"
 	"github.com/crunchydata/postgres-operator/internal/operator/backrest"
-	clusteroperator "github.com/crunchydata/postgres-operator/internal/operator/cluster"
 	crv1 "github.com/crunchydata/postgres-operator/pkg/apis/crunchydata.com/v1"
 )
 
@@ -92,9 +92,8 @@ func (c *Controller) handleBackrestBackupUpdate(job *apiv1.Job) error {
 			job.ObjectMeta.Namespace)
 
 	} else if labels[config.LABEL_PGHA_BACKUP_TYPE] == crv1.BackupTypeFailover {
-		err := clusteroperator.RemovePrimaryOnRoleChangeTag(c.Client, c.Client.Config,
-			labels[config.LABEL_PG_CLUSTER], job.ObjectMeta.Namespace)
-		if err != nil {
+		if err := operator.RemovePrimaryOnRoleChangeTag(c.Client, c.Client.Config,
+			labels[config.LABEL_PG_CLUSTER], job.ObjectMeta.Namespace); err != nil {
 			log.Error(err)
 			return err
 		}

internal/operator/common.go

@@ -32,6 +32,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/client-go/kubernetes"
 )
 
@@ -302,6 +303,49 @@ func SetContainerImageOverride(containerImageName string, container *v1.Containe
 	}
 }
 
+// getCandidatePod tries to get the candidate Pod for a switchover or failover.
+// If "candidateName" is provided, it will seek out the specific PostgreSQL
+// instance. Otherwise, it will just attempt to find a running Pod.
+//
+// If such a Pod cannot be found, we likely cannot use the instance for a
+// switchover for failover candidate as it is not running.
+func getCandidatePod(clientset kubernetes.Interface, cluster *crv1.Pgcluster, candidateName string) (*v1.Pod, error) {
+	ctx := context.TODO()
+
+	// build the label selector. we are looking for any PostgreSQL instance within
+	// this cluster, so that part is easy
+	labelSelector := fields.Set{
+		config.LABEL_PG_CLUSTER:  cluster.Name,
+		config.LABEL_PG_DATABASE: config.LABEL_TRUE,
+	}
+
+	// if a candidateName is supplied, use that as part of the label selector to
+	// find the candidate Pod
+	if candidateName != "" {
+		labelSelector[config.LABEL_DEPLOYMENT_NAME] = candidateName
+	}
+
+	// ensure the Pod is part of the cluster and is running
+	options := metav1.ListOptions{
+		FieldSelector: fields.OneTermEqualSelector("status.phase", string(v1.PodRunning)).String(),
+		LabelSelector: labelSelector.String(),
+	}
+
+	pods, err := clientset.CoreV1().Pods(cluster.Namespace).List(ctx, options)
+	if err != nil {
+		return nil, err
+	}
+
+	// if no Pods are found, then also return an error as we then cannot switch
+	// over to this instance
+	if len(pods.Items) == 0 {
+		return nil, fmt.Errorf("no pods found for instance %s", candidateName)
+	}
+
+	// the list returns multiple Pods, so just return the first one
+	return &pods.Items[0], nil
+}
+
 // initializeContainerImageOverrides initializes the container image overrides
 // that could be set if there are any `RELATED_IMAGE_*` environmental variables
 func initializeContainerImageOverrides() {

internal/operator/cluster/failover.go internal/operator/failover.gointernal/operator/cluster/failover.go renamed to internal/operator/failover.go

@@ -1,7 +1,4 @@
-// Package cluster holds the cluster CRD logic and definitions
-// A cluster is comprised of a primary service, replica service,
-// primary deployment, and replica deployment
-package cluster
+package operator
 
 /*
  Copyright 2018 - 2021 Crunchy Data Solutions, Inc.
@@ -24,6 +21,7 @@ import (
 
 	"github.com/crunchydata/postgres-operator/internal/config"
 	"github.com/crunchydata/postgres-operator/internal/kubeapi"
+	crv1 "github.com/crunchydata/postgres-operator/pkg/apis/crunchydata.com/v1"
 	log "github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,6 +33,56 @@ import (
 var roleChangeCmd = []string{"patronictl", "edit-config", "--force",
 	"--set", "tags.primary_on_role_change=null"}
 
+// Failover performs a failover to a PostgreSQL cluster, which is effectively
+// a "forced switchover." In other words, failover will force ensure that
+// there is a primary available.
+//
+// NOTE: This is reserve as the "last resort" case. If you want a controlled
+// failover, you want "Switchover".
+//
+// A target must be specified. The target should contain the name of the target
+// instances (Deployment), is not empty then we will attempt to locate that
+// target Pod.
+//
+// The target Pod name, called the candidate is passed into the failover
+// command generation function, and then is ultimately used in the failover.
+func Failover(clientset kubernetes.Interface, restConfig *rest.Config, cluster *crv1.Pgcluster, target string) error {
+	// ensure target is not empty
+	if target == "" {
+		return fmt.Errorf("failover requires a target instance to be specified.")
+	}
+
+	// When the target is specified, we will attempt to get the Pod that
+	// represents that target.
+	//
+	// If it is not specified, then we will attempt to get any Pod.
+	//
+	// If either errors, we will return an error
+	pod, err := getCandidatePod(clientset, cluster, target)
+
+	if err != nil {
+		return err
+	}
+
+	candidate := pod.Name
+
+	// generate the command
+	cmd := generatePostgresFailoverCommand(cluster.Name, candidate)
+
+	// good to generally log which instances are being used in the failover
+	log.Infof("failover started for cluster %q", cluster.Name)
+
+	if _, stderr, err := kubeapi.ExecToPodThroughAPI(restConfig, clientset,
+		cmd, "database", pod.Name, cluster.Namespace, nil); err != nil {
+		return fmt.Errorf(stderr)
+	}
+
+	log.Infof("failover completed for cluster %q", cluster.Name)
+
+	// and that's all
+	return nil
+}
+
 // RemovePrimaryOnRoleChangeTag sets the 'primary_on_role_change' tag to null in the
 // Patroni DCS, effectively removing the tag.  This is accomplished by exec'ing into
 // the primary PG pod, and sending a patch request to update the appropriate data (i.e.
@@ -77,3 +125,21 @@ func RemovePrimaryOnRoleChangeTag(clientset kubernetes.Interface, restconfig *re
 	}
 	return nil
 }
+
+// generatePostgresFailoverCommand creates the command that is used to issue
+// a failover command (ensure that there is a promoted primary).
+//
+// There are two ways to run this command:
+//
+// 1. Pass in only a clusterName. Patroni will select the best candidate
+// 2. Pass in a clusterName AND a target candidate name, which has to be the
+//    name of a Pod
+func generatePostgresFailoverCommand(clusterName, candidate string) []string {
+	cmd := []string{"patronictl", "failover", "--force", clusterName}
+
+	if candidate != "" {
+		cmd = append(cmd, "--candidate", candidate)
+	}
+
+	return cmd
+}