Increase orchestrator controller worker to 10

Author: AMecea

Changelog.md

@@ -37,6 +37,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
  * Add `InitContainers` and `Containers` in `.Spec.PodSpec` to allow the user specifying custom containers.
  * Add `MetricsExporterResources` and `MySQLOperatorSidecarResrouces` in `.Spec.PodSpec` to allow
    the user specifying resources for thos sidecars containers.
+ * Add command line flag to configure number of workers for orchestrator controller.
 ### Changed
  * [#422](https://github.com/presslabs/mysql-operator/pull/422) adds the `SidecarServerPort` to the
    `MasterService` and introduces one new service, HealthyReplicasService, so that we can try to
@@ -64,6 +65,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
    [`extra_max_connections`](https://www.percona.com/doc/percona-server/5.7/performance/threadpool.html#extra_max_connections)
    is larger than the default `1`. If MySQL server runs out of available connections, using `extra_port`
    allows the exporter to continue collecting MySQL metrics.
+ * Change the default number of workers for orchestrator controller from 1 to 10.
 ### Removed
 ### Fixed
  * Update and fix e2e tests

pkg/controller/orchestrator/orchestrator_controller.go

@@ -90,7 +90,10 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
 	clusters := &sync.Map{}
 
 	// Create a new controller
-	c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r})
+	c, err := controller.New(controllerName, mgr, controller.Options{
+		Reconciler:              r,
+		MaxConcurrentReconciles: int(options.GetOptions().OrchestratorConcurrentReconciles),
+	})
 	if err != nil {
 		return err
 	}

pkg/controller/orchestrator/orchestrator_reconcile.go

@@ -171,6 +171,7 @@ func (ou *orcUpdater) updateClusterReadyStatus() {
 		if master {
 			hasMaster = true
 		} else if !replicating {
+			// TODO: check for replicating to be not Unknown here
 			ou.cluster.UpdateStatusCondition(api.ClusterConditionReady, core.ConditionFalse, "NotReplicating",
 				fmt.Sprintf("Node %s is part of topology and not replicating", hostname))
 			return
@@ -513,7 +514,6 @@ func (ou *orcUpdater) markReadOnlyNodesInOrc(insts InstancesSet, master *orc.Ins
 	if master == nil {
 		// master is not found
 		// set cluster read only
-		ou.log.Info("setting cluster in read-only")
 		for _, inst := range insts {
 			if err = ou.setReadOnlyNode(inst); err != nil {
 				ou.log.Error(err, "failed to set read only", "instance", instToLog(&inst))

pkg/options/options.go

@@ -69,6 +69,9 @@ type Options struct {
 	// MySQLVersionImageOverride define a map between MySQL version and image.
 	// This overrides the default versions and has priority.
 	MySQLVersionImageOverride map[string]string
+
+	// OrchestratorConcurrentReconciles sets the orchestrator controller workers
+	OrchestratorConcurrentReconciles int32
 }
 
 type pullpolicy corev1.PullPolicy
@@ -141,6 +144,9 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
 
 	fs.StringToStringVar(&o.MySQLVersionImageOverride, "mysql-versions-to-image", map[string]string{},
 		"A map to override default image for different mysql versions. Example: 5.7.23=mysql:5.7,5.7.24=mysql:5.7")
+
+	fs.Int32Var(&o.OrchestratorConcurrentReconciles, "orchestrator-concurrent-reconciles", 10,
+		"Set the number of workers for orchestrator reconciler.")
 }
 
 var instance *Options