Remove node role from sidecar side. Don't run node controller on old clusters. Set readyNodes to zero when doing upgrade.

Author: AMecea

docs/deploy-mysql-cluster.md

@@ -154,6 +154,7 @@ spec:
 
 For more configuration details regarding`hostPath` go [here](https://godoc.org/k8s.io/api/core/v1#HostPathVolumeSource), and more information about `emptyDir` can be found [here](https://godoc.org/k8s.io/api/core/v1#EmptyDirVolumeSource).
 
+> NOTE that EmptyDir is not fully supported from version `v0.3.x` of the operator. Please use `hostPath` instead. See information about the known [issue](https://github.com/presslabs/mysql-operator/issues/250).
 
 ## Cluster MySQL configuration
 

docs/operator-upgrades.md

@@ -43,3 +43,4 @@ field) to make it smaller that will prevent you from hitting this
 The operator should do all the work for you but you have to make sure that you have the latest
 `0.2.x` version of the operator.
 
+This release drop support for `emptyDir` volume source.

pkg/controller/mysqlcluster/internal/syncer/master_service.go

@@ -42,9 +42,6 @@ func NewMasterSVCSyncer(c client.Client, scheme *runtime.Scheme, cluster *mysqlc
 		out.Labels = cluster.GetLabels()
 		out.Labels["mysql.presslabs.org/service-type"] = "master"
 
-		// make the master endpoint headless, it should always point to only one node
-		out.Spec.ClusterIP = "None"
-
 		// set selectors for master node
 		out.Spec.Selector = cluster.GetSelectorLabels()
 		out.Spec.Selector["role"] = "master"

pkg/controller/mysqlcluster/internal/upgrades/upgrades.go

@@ -35,7 +35,7 @@ import (
 	orc "github.com/presslabs/mysql-operator/pkg/orchestrator"
 )
 
-var log = logf.Log.WithName("upgrader.cluster")
+var log = logf.Log.WithName("upgrades.cluster")
 
 const (
 	// VersionAnnotation represents the annotation used to annotate a cluster to it's version
@@ -74,8 +74,8 @@ func (u *upgrader) Run(ctx context.Context) error {
 		return err
 	}
 
-	// more than 1 replca so there is the case when node 0 is slave so mark all other nodes as
-	// in maintenance except node 0. When doint failover the node 0 will be promoted
+	// more than 1 replica so there is the case when node 0 is slave so mark all other nodes as
+	// in maintenance except node 0.
 	// TODO: or set promotion rules
 	if int(*sts.Spec.Replicas) > 1 {
 
@@ -88,6 +88,12 @@ func (u *upgrader) Run(ctx context.Context) error {
 		if err = u.client.Update(ctx, sts); err != nil {
 			return err
 		}
+
+		// set ready nodes on cluster to 0
+		u.cluster.Status.ReadyNodes = 0
+		if err = u.client.Status().Update(ctx, u.cluster.Unwrap()); err != nil {
+			return err
+		}
 	}
 
 	if sts.Status.ReadyReplicas > 1 {

pkg/controller/mysqlcluster/mysqlcluster_controller.go

@@ -155,10 +155,11 @@ func (r *ReconcileMysqlCluster) Reconcile(request reconcile.Request) (reconcile.
 	}
 	log.Info("syncing cluster", "cluster", request.NamespacedName.String())
 
-	// run upgrader
+	// run upgrades
+	// TODO: this should be removed in next version (v0.4)
 	up := upgrades.NewUpgrader(r.Client, r.recorder, cluster, r.opt)
 	if up.ShouldUpdate() {
-		log.V(1).Info("running upgrader")
+		log.V(1).Info("running upgrades")
 		if err = up.Run(context.TODO()); err != nil {
 			return reconcile.Result{}, err
 		}

pkg/controller/node/node_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"regexp"
+	"strconv"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
@@ -154,29 +155,37 @@ func (r *ReconcileMysqlNode) Reconcile(request reconcile.Request) (reconcile.Res
 
 	log.Info("syncing MySQL Node", "pod", request.NamespacedName.String())
 
+	// try to get the related MySQL Cluster for current node
 	var cluster *mysqlcluster.MysqlCluster
 	cluster, err = r.getNodeCluster(ctx, pod)
 	if err != nil {
 		log.Info("cluster is not found")
 		return reconcile.Result{}, err
 	}
 
+	// if cluster is deleted then don't do anything
 	if cluster.DeletionTimestamp != nil {
 		log.Info("cluster is deleted nothing to do", "pod", pod.Spec.Hostname)
 		return reconcile.Result{}, nil
 	}
 
+	// if it's a old version cluster then don't do anything
+	if shouldUpdateToVersion(cluster, 300) {
+		return reconcile.Result{}, nil
+	}
+
+	// get cluster credentials from k8s secret, like replication and operator credentials
 	var creds *credentials
-	creds, err = r.getCredsSecret(ctx, cluster)
-	if err != nil {
+	if creds, err = r.getCredsSecret(ctx, cluster); err != nil {
 		return reconcile.Result{}, err
 	}
 
 	// initialize SQL interface
 	sql := r.getMySQLConnection(cluster, pod, creds)
 
-	err = r.initializeMySQL(ctx, sql, cluster, creds)
-	if err != nil {
+	// run the initializer, this will connect to MySQL server and run init queries
+	if err = r.initializeMySQL(ctx, sql, cluster, creds); err != nil {
+		// initialization failed, mark node as not yet initialized (by updating pod init condition)
 		updatePodStatusCondition(pod, mysqlcluster.NodeInitializedConditionType,
 			corev1.ConditionFalse, "mysqlInitializationFailed", err.Error())
 
@@ -187,6 +196,7 @@ func (r *ReconcileMysqlNode) Reconcile(request reconcile.Request) (reconcile.Res
 		return reconcile.Result{}, err
 	}
 
+	// initialization complete
 	updatePodStatusCondition(pod, mysqlcluster.NodeInitializedConditionType,
 		corev1.ConditionTrue, "mysqlInitializationSucceeded", "success")
 
@@ -199,6 +209,7 @@ func (r *ReconcileMysqlNode) initializeMySQL(ctx context.Context, sql SQLInterfa
 		return err
 	}
 
+	// disable MySQL SUPER readonly to be able to modify settings in MySQL
 	enableSuperReadOnly, err := sql.DisableSuperReadOnly(ctx)
 	if err != nil {
 		return err
@@ -217,6 +228,7 @@ func (r *ReconcileMysqlNode) initializeMySQL(ctx context.Context, sql SQLInterfa
 		}
 	}
 
+	// write the configuration complete flag into MySQL, this will make the node ready
 	if err := sql.MarkConfigurationDone(ctx); err != nil {
 		return err
 	}
@@ -341,3 +353,21 @@ func podCondIndex(p *corev1.Pod, condType corev1.PodConditionType) (int, bool) {
 
 	return 0, false
 }
+
+func shouldUpdateToVersion(cluster *mysqlcluster.MysqlCluster, targetVersion int) bool {
+	var version string
+	var ok bool
+	if version, ok = cluster.ObjectMeta.Annotations["mysql.presslabs.org/version"]; !ok {
+		// no version annotation present, (it's a cluster older than 0.3.0) or it's a new cluster
+		log.Info("annotation not set on cluster")
+		return true
+	}
+
+	ver, err := strconv.ParseInt(version, 10, 32)
+	if err != nil {
+		log.Error(err, "annotation version can't be parsed", "value", version)
+		return true
+	}
+
+	return int(ver) < targetVersion
+}

pkg/controller/node/node_ctrl_test.go

@@ -87,6 +87,9 @@ var _ = Describe("MysqlNode controller", func() {
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      fmt.Sprintf("cluster-%d", rand.Int31()),
 					Namespace: "default",
+					Annotations: map[string]string{
+						"mysql.presslabs.org/version": "300",
+					},
 				},
 				Spec: api.MysqlClusterSpec{
 					Replicas:   &one,

pkg/sidecar/appclone.go

@@ -37,13 +37,7 @@ func RunCloneCommand(cfg *Config) error {
 		return fmt.Errorf("removing lost+found: %s", err)
 	}
 
-	role, err := cfg.NodeRole()
-	if err != nil {
-		return err
-	}
-
-	log.Info("configuring server as", "host", cfg.Hostname, "role", role)
-	if role == MasterNode {
+	if cfg.ServerID() == 100 {
 		if len(cfg.InitBucketURL) == 0 {
 			log.Info("skip cloning init bucket uri is not set.")
 			// let mysqld initialize data dir
@@ -55,23 +49,10 @@ func RunCloneCommand(cfg *Config) error {
 		}
 	} else {
 		// clonging from prior node
-		if cfg.ServerID() > 100 {
-			sourceHost := cfg.FQDNForServer(cfg.ServerID() - 1)
-			err := cloneFromSource(cfg, sourceHost)
-			if err != nil {
-				return fmt.Errorf("failed to clone from %s, err: %s", sourceHost, err)
-			}
-			// clone from master if no prior node and node is not master
-		} else if cfg.ServerID() == 100 {
-			sourceHost := cfg.MasterFQDN()
-			err := cloneFromSource(cfg, sourceHost)
-			if err != nil {
-				return fmt.Errorf("failed to clone from %s, err: %s", sourceHost, err)
-			}
-		} else {
-			return fmt.Errorf(
-				"failed to initialize because no prior node exists, check orchestrator maybe",
-			)
+		sourceHost := cfg.FQDNForServer(cfg.ServerID() - 1)
+		err := cloneFromSource(cfg, sourceHost)
+		if err != nil {
+			return fmt.Errorf("failed to clone from %s, err: %s", sourceHost, err)
 		}
 	}
 

pkg/sidecar/configs.go

@@ -32,18 +32,6 @@ import (
 	"github.com/presslabs/mysql-operator/pkg/internal/mysqlcluster"
 )
 
-// NodeRole represents the kind of the MySQL server
-type NodeRole string
-
-const (
-	// MasterNode represents the master role for MySQL server
-	MasterNode NodeRole = "master"
-	// SlaveNode represents the slave role for MySQL server
-	SlaveNode NodeRole = "slave"
-	// Unknown usually used for error, when the role can't be determined
-	Unknown NodeRole = "unknown"
-)
-
 // Config contains information related with the pod.
 type Config struct {
 	// Hostname represents the pod hostname
@@ -99,42 +87,6 @@ func (cfg *Config) MasterFQDN() string {
 	return mysqlcluster.GetNameForResource(mysqlcluster.MasterService, cfg.ClusterName)
 }
 
-// NodeRole returns the role of the current node
-func (cfg *Config) NodeRole() (NodeRole, error) {
-	pod := cfg.FQDNForServer(cfg.ServerID())
-	podIPs, err := retryLookupHost(pod)
-	if err != nil {
-		log.Info("can't lookup pod IP", "pod", pod, "err", err)
-		return Unknown, err
-	}
-
-	if len(podIPs) != 1 {
-		// no IPs for given pod
-		log.Info("no IPs for pod found", "pod", pod)
-		return Unknown, nil
-	}
-
-	masterSVC := mysqlcluster.GetNameForResource(mysqlcluster.MasterService, cfg.ClusterName)
-	masterIPs, err := retryLookupHost(masterSVC)
-	if err != nil {
-		log.Info("can't lookup master service IP", "svc", masterSVC, "err", err)
-		return MasterNode, nil
-	}
-
-	if len(masterIPs) != 1 {
-		log.Info("no IPs for master service", "svc", masterSVC)
-		return MasterNode, nil
-	}
-
-	if podIPs[0] == masterIPs[0] {
-		log.Info("this pod is MASTER")
-		return MasterNode, nil
-	}
-
-	log.Info("this pod is SLAVE")
-	return SlaveNode, nil
-}
-
 // ServerID returns the MySQL server id
 func (cfg *Config) ServerID() int {
 	ordinal := getOrdinalFromHostname(cfg.Hostname)

pkg/sidecar/configs_test.go

@@ -51,14 +51,4 @@ var _ = Describe("Test sidecar configs", func() {
 	It("should determine the host ip", func() {
 		Expect(retryLookupHost("localhost")).To(ContainElement("127.0.0.1"))
 	})
-
-	It("should determine the node role", func() {
-		_, err := cfg.NodeRole()
-		Expect(err).ToNot(Succeed())
-		// TODO: fix this test
-		// Expect().To(Equal(MasterNode))
-
-		// cfg.Hostname = "cluster-mysql-2"
-		// Expect(cfg.NodeRole()).To(Equal(SlaveNode))
-	})
 })