feat(ha): print warning on control plane node reset (#677)

warn about removing the antepenultimate (third to last) controller node
if ha is enabled.

Author: ricardomaraschini

cmd/embedded-cluster/uninstall.go

@@ -11,9 +11,11 @@ import (
 	autopilot "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2"
 	"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	"github.com/k0sproject/k0s/pkg/etcd"
+	embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
@@ -55,6 +57,8 @@ var (
 	k0s     = "/usr/local/bin/k0s"
 )
 
+var haWarningMessage = "WARNING: High-availability clusters must maintain at least three controller nodes, but resetting this node will leave only two. This can lead to a loss of functionality and non-recoverable failures. You should re-add a third node as soon as possible."
+
 // deleteNode removes the node from the cluster
 func (h *hostInfo) deleteNode(ctx context.Context) error {
 	if h.KclientError != nil {
@@ -287,6 +291,43 @@ func checkErrPrompt(c *cli.Context, err error) bool {
 	return prompts.New().Confirm("Do you want to continue anyway?", false)
 }
 
+// maybePrintHAWarning prints a warning message when the user is running a reset a node
+// in a high availability cluster and there are only 3 control nodes.
+func maybePrintHAWarning(c *cli.Context) error {
+	kubeconfig := defaults.PathToKubeConfig()
+	if _, err := os.Stat(kubeconfig); err != nil {
+		return nil
+	}
+
+	os.Setenv("KUBECONFIG", kubeconfig)
+	kubecli, err := kubeutils.KubeClient()
+	if err != nil {
+		return fmt.Errorf("unable to create kube client: %w", err)
+	}
+	embeddedclusterv1beta1.AddToScheme(kubecli.Scheme())
+
+	if in, err := kubeutils.GetLatestInstallation(c.Context, kubecli); err != nil {
+		return fmt.Errorf("unable to get installation: %w", err)
+	} else if !in.Spec.HighAvailability {
+		return nil
+	}
+
+	opts := &client.ListOptions{
+		LabelSelector: labels.SelectorFromSet(
+			labels.Set{"node-role.kubernetes.io/control-plane": "true"},
+		),
+	}
+	var nodes corev1.NodeList
+	if err := kubecli.List(c.Context, &nodes, opts); err != nil {
+		return fmt.Errorf("unable to list nodes: %w", err)
+	}
+	if len(nodes.Items) == 3 {
+		logrus.Warn(haWarningMessage)
+		logrus.Info("")
+	}
+	return nil
+}
+
 var resetCommand = &cli.Command{
 	Name: "reset",
 	Before: func(c *cli.Context) error {
@@ -315,6 +356,10 @@ var resetCommand = &cli.Command{
 	},
 	Usage: fmt.Sprintf("Remove %s from the current node", binName),
 	Action: func(c *cli.Context) error {
+		if err := maybePrintHAWarning(c); err != nil && !c.Bool("force") {
+			return err
+		}
+
 		logrus.Info("This will remove this node from the cluster and completely reset it, removing all data stored on the node.")
 		logrus.Info("Do not reset another node until this is complete.")
 		if !c.Bool("force") && !c.Bool("no-prompt") && !prompts.New().Confirm("Do you want to continue?", false) {

e2e/install_test.go

@@ -1051,6 +1051,17 @@ func TestMultiNodeHAInstallation(t *testing.T) {
 		t.Fatalf("fail to check post ha state: %v", err)
 	}
 
+	bin := strings.Split(command, " ")[0]
+	t.Logf("%s: resetting controller node", time.Now().Format(time.RFC3339))
+	stdout, stderr, err = RunCommandOnNode(t, tc, 2, []string{bin, "reset", "--no-prompt"})
+	if err != nil {
+		t.Fatalf("fail to remove controller node %s:", err)
+	}
+	if !strings.Contains(stderr, "High-availability clusters must maintain at least three controller nodes") {
+		t.Errorf("reset output does not contain the ha warning")
+		t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
+	}
+
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }