final failover logic added plus docs

Author: jmccormick2001

docs/operator-docs.asciidoc

@@ -492,44 +492,6 @@ Or if you have DNS configured on your client host:
 export CO_APISERVER_URL=https://postgres-operator.demo.svc.cluster.local:8443
 ....
 
-
-== Performing a Smoke Test
-
-A simple *smoke test* of the postgres operator includes testing
-the following:
-
- * get version information (*pgo version*)
- * create a cluster (*pgo create cluster testcluster*)
- * scale a cluster (*pgo scale testcluster *)
- * show a cluster (*pgo show cluster testcluster*)
- * show all clusters (*pgo show cluster all*)
- * backup a cluster (*pgo backup testcluster*)
- * show backup of cluster (*pgo show backup testcluster*)
- * show backup pvc of cluster (*pgo show pvc testcluster-backup-pvc*)
- * restore a cluster (*pgo create cluster restoredcluster --backup-pvc=testcluster-backup-pvc --backup-path=testcluster-backups/2017-01-01-01-01-01 --secret-from=testcluster*)
- * test a cluster (*pgo test restoredcluster*)
- * minor upgrade a cluster (*pgo upgrade testcluster*)
- * major upgrade a cluster (*pgo upgrade testcluster --upgrade-type=major*)
- * delete a cluster (*pgo delete cluster testcluster --delete-data --delete-backups*)
- * create a policy from local file (*pgo create policy policy1 --in-file=./examples/policy/policy1.sql*)
- * create a policy from git repo (*pgo create policy gitpolicy --url=https://github.com/CrunchyData/postgres-operator/blob/master/examples/policy/gitpolicy.sql*)
- * repeat testing using emptydir storage type
- * repeat testing using create storage type
- * repeat testing using existing storage type
- * create a series of clusters  (*pgo create cluster myseries --series=2*)
- * apply labels at cluster creation (*pgo create cluster xraydb --series=2 --labels=project=xray*)
- * apply a label to an existing set of clusters (*pgo label --label=env=research --selector=project=xray*)
- * create a user for a given cluster (*pgo create user user0 --valid-days=30 --managed --db=userdb --selector=name=xraydb0*)
- * load a csv file into a cluster (*pgo load --load-config=./sample-load-config.json --selector=project=xray*)
- * extend a user's password allowed age (*pgo user --change-password=user0 --valid-days=10 --selector=name=xraydb1*)
- * drop user access (*pgo user --delete-user=user2 --selector=project=xray*)
- * check password age (*pgo user --expired=10 --selector=project=xray*)
- * backup an entire project (*pgo backup --selector=project=xray*)
- * delete an entire project (*pgo delete cluster --selector=project=xray*)
- * create a cluster with a crunchy-collect sidecar(*pgo create cluster testcluster --metrics*)
-
-More detailed explanations of the commands can be found below <<pgo Commands>>.
-
 == Makefile Targets
 
 The following table describes the Makefile targets:
@@ -1374,6 +1336,47 @@ The load configuration file has the following YAML attributes:
 |SecurityContext| either fsGroup or SupplementalGroup values
 |======================
 
+=== pgo failover
+
+Starting with Release 2.6, there is a manual failover command which
+can be used to promote a replica to a primary role in a PostgreSQL
+cluster.
+
+This process includes the following actions:
+ * pick a target replica to become the new primary
+ * delete the current primary deployment to avoid user requests from
+   going to multiple primary databases (split brain)
+ * promote the targeted replica using *pg_ctl promote*, this will
+   cause PostgreSQL to go into read-write mode
+ * re-label the targeted replica to use the primary labels, this
+   will match the primary service selector and cause new requests
+   to the primary to be routed to the new primary (targeted replica)
+
+The command works like this:
+....
+pgo failover mycluster --query
+....
+
+That command will show you a list of replica targets you can choose
+to failover to.  You will select one of those for the following
+command:
+....
+pgo failover mycluster --target=mycluster-abxq
+....
+
+There is a CRD called *pgtask* that will hold the failover request
+and also the status of that request.  You can view the status
+by viewing it:
+....
+kubectl get pgtasks mycluster-failover -o yaml
+....
+
+Once completed, you will see a new replica has been started to replace 
+the promoted replica, this happens automatically due to the re-lable, the
+Deployment will recreate its pod because of this.   The failover typically
+takes only a few seconds, however, the creation of the replacement
+replica can take longer depending on how much data is being replicated.
+
 == bash Completion
 
 There is a bash completion file that is included for users to try, this

docs/operator-testing.asciidoc

@@ -0,0 +1,40 @@
+= PostgreSQL Operator Testing
+:toc:
+v2.6, {docdate}
+
+== Performing a Smoke Test
+
+A simple *smoke test* of the postgres operator includes testing
+the following:
+
+ * get version information (*pgo version*)
+ * create a cluster (*pgo create cluster testcluster*)
+ * scale a cluster (*pgo scale testcluster *)
+ * show a cluster (*pgo show cluster testcluster*)
+ * show all clusters (*pgo show cluster all*)
+ * backup a cluster (*pgo backup testcluster*)
+ * show backup of cluster (*pgo show backup testcluster*)
+ * show backup pvc of cluster (*pgo show pvc testcluster-backup-pvc*)
+ * restore a cluster (*pgo create cluster restoredcluster --backup-pvc=testcluster-backup-pvc --backup-path=testcluster-backups/2017-01-01-01-01-01 --secret-from=testcluster*)
+ * test a cluster (*pgo test restoredcluster*)
+ * minor upgrade a cluster (*pgo upgrade testcluster*)
+ * major upgrade a cluster (*pgo upgrade testcluster --upgrade-type=major*)
+ * delete a cluster (*pgo delete cluster testcluster --delete-data --delete-backups*)
+ * create a policy from local file (*pgo create policy policy1 --in-file=./examples/policy/policy1.sql*)
+ * create a policy from git repo (*pgo create policy gitpolicy --url=https://github.com/CrunchyData/postgres-operator/blob/master/examples/policy/gitpolicy.sql*)
+ * repeat testing using emptydir storage type
+ * repeat testing using create storage type
+ * repeat testing using existing storage type
+ * create a series of clusters  (*pgo create cluster myseries --series=2*)
+ * apply labels at cluster creation (*pgo create cluster xraydb --series=2 --labels=project=xray*)
+ * apply a label to an existing set of clusters (*pgo label --label=env=research --selector=project=xray*)
+ * create a user for a given cluster (*pgo create user user0 --valid-days=30 --managed --db=userdb --selector=name=xraydb0*)
+ * load a csv file into a cluster (*pgo load --load-config=./sample-load-config.json --selector=project=xray*)
+ * extend a user's password allowed age (*pgo user --change-password=user0 --valid-days=10 --selector=name=xraydb1*)
+ * drop user access (*pgo user --delete-user=user2 --selector=project=xray*)
+ * check password age (*pgo user --expired=10 --selector=project=xray*)
+ * backup an entire project (*pgo backup --selector=project=xray*)
+ * delete an entire project (*pgo delete cluster --selector=project=xray*)
+ * create a cluster with a crunchy-collect sidecar(*pgo create cluster testcluster --metrics*)
+ * perform a failover (*pgo failover mycluster*)
+

examples/pgo-bash-completion

@@ -304,8 +304,8 @@ _pgo_create_cluster()
     flags+=("--metrics")
     flags+=("-m")
     local_nonpersistent_flags+=("--metrics")
-    flags+=("--node-name=")
-    local_nonpersistent_flags+=("--node-name=")
+    flags+=("--node-label=")
+    local_nonpersistent_flags+=("--node-label=")
     flags+=("--password=")
     two_word_flags+=("-w")
     local_nonpersistent_flags+=("--password=")
@@ -595,6 +595,32 @@ _pgo_delete()
     noun_aliases=()
 }
 
+_pgo_failover()
+{
+    last_command="pgo_failover"
+    commands=()
+
+    flags=()
+    two_word_flags=()
+    local_nonpersistent_flags=()
+    flags_with_completion=()
+    flags_completion=()
+
+    flags+=("--no-prompt")
+    flags+=("-n")
+    local_nonpersistent_flags+=("--no-prompt")
+    flags+=("--query")
+    local_nonpersistent_flags+=("--query")
+    flags+=("--target=")
+    local_nonpersistent_flags+=("--target=")
+    flags+=("--apiserver-url=")
+    flags+=("--debug")
+
+    must_have_one_flag=()
+    must_have_one_noun=()
+    noun_aliases=()
+}
+
 _pgo_label()
 {
     last_command="pgo_label"
@@ -665,6 +691,8 @@ _pgo_scale()
     flags_with_completion=()
     flags_completion=()
 
+    flags+=("--node-label=")
+    local_nonpersistent_flags+=("--node-label=")
     flags+=("--replica-count=")
     two_word_flags+=("-r")
     local_nonpersistent_flags+=("--replica-count=")
@@ -954,6 +982,7 @@ _pgo()
     commands+=("backup")
     commands+=("create")
     commands+=("delete")
+    commands+=("failover")
     commands+=("label")
     commands+=("load")
     commands+=("scale")

operator/cluster/failover_strategy_1.go

@@ -50,12 +50,12 @@ func (r Strategy1) Failover(clientset *kubernetes.Clientset, client *rest.RESTCl
 
 	log.Info("strategy 1 Failover called on " + clusterName + " target is " + target)
 
-	if target == "" {
-		log.Debug("failover target not set, will use best estimate")
-		pod, err = util.GetBestTarget(clientset, clusterName, namespace)
-	} else {
-		pod, err = util.GetPod(clientset, target, namespace)
-	}
+	//if target == "" {
+	//	log.Debug("failover target not set, will use best estimate")
+	//	pod, target, err = util.GetBestTarget(clientset, clusterName, namespace)
+	//} else {
+	pod, err = util.GetPod(clientset, target, namespace)
+	//}
 	if err != nil {
 		log.Error(err)
 		return err
@@ -71,20 +71,20 @@ func (r Strategy1) Failover(clientset *kubernetes.Clientset, client *rest.RESTCl
 	updateFailoverStatus(client, task, namespace, clusterName, "deleting primary deployment "+clusterName)
 
 	//trigger the failover on the replica
-	err = promote(pod, clientset, client, namespace, target, restconfig)
+	err = promote(pod, clientset, client, namespace, restconfig)
 	//if err != nil {
 	//log.Error(err)
 	//return err
 	//}
-	updateFailoverStatus(client, task, namespace, clusterName, "promoting replica"+target)
+	updateFailoverStatus(client, task, namespace, clusterName, "promoting pod "+pod.Name+" target "+target)
 
 	//relabel the deployment with primary labels
 	err = relabel(pod, clientset, namespace, clusterName, target)
 	//if err != nil {
 	//log.Error(err)
 	////return err
 	//}
-	updateFailoverStatus(client, task, namespace, clusterName, "re-labeling replica")
+	updateFailoverStatus(client, task, namespace, clusterName, "re-labeling deployment...pod "+pod.Name+"was the failover target...failover completed")
 
 	return err
 
@@ -144,7 +144,10 @@ func deletePrimary(clientset *kubernetes.Clientset, namespace, clusterName strin
 	return err
 }
 
-func promote(pod *v1.Pod, clientset *kubernetes.Clientset, client *rest.RESTClient, namespace, target string, restconfig *rest.Config) error {
+func promote(
+	pod *v1.Pod,
+	clientset *kubernetes.Clientset,
+	client *rest.RESTClient, namespace string, restconfig *rest.Config) error {
 	var err error
 
 	//get the target pod that matches the replica-name=target
@@ -175,14 +178,14 @@ func relabel(pod *v1.Pod, clientset *kubernetes.Clientset, namespace, clusterNam
 	//set replica=false on the deployment
 	//set name=clustername on the deployment
 	newLabels := make(map[string]string)
-	newLabels["replica"] = "false"
 	newLabels["name"] = clusterName
 
 	err = updateLabels(namespace, clientset, targetDeployment, target, newLabels)
 	if err != nil {
 		log.Error(err)
 	}
 
+	newLabels["replica"] = "false"
 	err = updatePodLabels(namespace, clientset, pod, target, newLabels)
 	if err != nil {
 		log.Error(err)

pgo/cmd/failover.go

@@ -41,6 +41,10 @@ var failoverCmd = &cobra.Command{
 			if Query {
 				createFailover(args)
 			} else if util.AskForConfirmation(NoPrompt) {
+				if Target == "" {
+					fmt.Println(`--target is required for failover.`)
+					return
+				}
 				createFailover(args)
 			} else {
 				fmt.Println("Aborting...")

util/failover.go

@@ -21,6 +21,7 @@ import (
 	"k8s.io/api/core/v1"
 	//"k8s.io/apimachinery/pkg/api/errors"
 	"errors"
+	"k8s.io/api/extensions/v1beta1"
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"k8s.io/client-go/kubernetes"
@@ -31,11 +32,43 @@ import (
 )
 
 // GetBestTarget
-func GetBestTarget(clientset *kubernetes.Clientset, clusterName, namespace string) (*v1.Pod, error) {
+func GetBestTarget(clientset *kubernetes.Clientset, clusterName, namespace string) (*v1.Pod, *v1beta1.Deployment, error) {
 
 	var err error
-	var pod *v1.Pod
-	return pod, err
+
+	//get all the replica deployment pods for this cluster
+	var pod v1.Pod
+	var deployment v1beta1.Deployment
+
+	//get all the deployments that are replicas for this clustername
+
+	//selector=replica=true,pg-cluster=clusterName
+	var pods *v1.PodList
+	lo := meta_v1.ListOptions{LabelSelector: "pg-cluster=" + clusterName + ",replica=true"}
+	pods, err = clientset.CoreV1().Pods(namespace).List(lo)
+	if err != nil {
+		log.Error(err)
+		return &pod, &deployment, err
+	}
+	if len(pods.Items) == 0 {
+		return &pod, &deployment, errors.New("no replica pods found for cluster " + clusterName)
+	}
+
+	for _, p := range pods.Items {
+		pod = p
+		log.Debug("pod found for replica " + pod.Name)
+		if len(pods.Items) == 1 {
+			log.Debug("only 1 pod found for failover best match..using it by default")
+			return &pod, &deployment, err
+		}
+
+		for _, c := range pod.Spec.Containers {
+			log.Debug("container " + c.Name + " found in pod")
+		}
+
+	}
+
+	return &pod, &deployment, err
 }
 
 // GetPodName from a deployment name