add multi-master option to galera

zzzeek · zzzeek · commit bddafb33ce94 · 2024-09-11T15:06:45.000-04:00
This change adds the new value enableMultiMaster to the
Galera CR defaulting to false; when set to true, the
"statefulset.kubernetes.io/pod-name" key is removed
from the mariadb service spec, and when set to false it is
restored.  This has the effect of publishing the service either
under one specific pod for active/passive use, or using
any pod for any request which is multi-master use.  the value
of the attribute is also propigated out to the status within
the MariaDBDatabase CR, which is then consumable by openstack
services in a similar manner as the TLS settings.   Openstack
services will then want to set wsrep_sync_wait accordingly
for AP or MM mode, which will be proposed in followup changes.

New teardown steps added to kuttl tests galera_cluster_restart
and galera_log_to_disk as there seem to be pvcs being not cleaned
up on CI leading to hangs

Fixes: OSPRH-7406
diff --git a/api/bases/mariadb.openstack.org_galeras.yaml b/api/bases/mariadb.openstack.org_galeras.yaml
@@ -64,6 +64,10 @@ spec:
                 description: When TLS is configured, only allow connections to the
                   DB over TLS
                 type: boolean
+              enableMultiMaster:
+                default: false
+                description: Use multi-master service routing
+                type: boolean
               logToDisk:
                 description: Log Galera pod's output to disk
                 type: boolean
@@ -104,6 +108,7 @@ spec:
                 type: object
             required:
             - containerImage
+            - enableMultiMaster
             - replicas
             - secret
             - storageClass
diff --git a/api/bases/mariadb.openstack.org_mariadbdatabases.yaml b/api/bases/mariadb.openstack.org_mariadbdatabases.yaml
@@ -98,6 +98,9 @@ spec:
                   - type
                   type: object
                 type: array
+              enableMultiMaster:
+                description: whether the DB instance is using multi-master routing
+                type: boolean
               hash:
                 additionalProperties:
                   type: string
diff --git a/api/v1beta1/galera_types.go b/api/v1beta1/galera_types.go
@@ -89,6 +89,9 @@ type GaleraSpecCore struct {
 	// +kubebuilder:validation:Optional
 	// Log Galera pod's output to disk
 	LogToDisk bool `json:"logToDisk"`
+	// +kubebuilder:default=false
+	// Use multi-master service routing
+	EnableMultiMaster bool `json:"enableMultiMaster"`
 }
 
 // GaleraAttributes holds startup information for a Galera host
diff --git a/api/v1beta1/mariadbdatabase_types.go b/api/v1beta1/mariadbdatabase_types.go
@@ -55,6 +55,9 @@ type MariaDBDatabaseStatus struct {
 
 	// Whether TLS is supported by the DB instance
 	TLSSupport bool `json:"tlsSupport,omitempty"`
+
+	// whether the DB instance is using multi-master routing
+	EnableMultiMaster bool `json:"enableMultiMaster,omitempty"`
 }
 
 //+kubebuilder:object:root=true
diff --git a/config/crd/bases/mariadb.openstack.org_galeras.yaml b/config/crd/bases/mariadb.openstack.org_galeras.yaml
@@ -64,6 +64,10 @@ spec:
                 description: When TLS is configured, only allow connections to the
                   DB over TLS
                 type: boolean
+              enableMultiMaster:
+                default: false
+                description: Use multi-master service routing
+                type: boolean
               logToDisk:
                 description: Log Galera pod's output to disk
                 type: boolean
@@ -104,6 +108,7 @@ spec:
                 type: object
             required:
             - containerImage
+            - enableMultiMaster
             - replicas
             - secret
             - storageClass
diff --git a/config/crd/bases/mariadb.openstack.org_mariadbdatabases.yaml b/config/crd/bases/mariadb.openstack.org_mariadbdatabases.yaml
@@ -98,6 +98,9 @@ spec:
                   - type
                   type: object
                 type: array
+              enableMultiMaster:
+                description: whether the DB instance is using multi-master routing
+                type: boolean
               hash:
                 additionalProperties:
                   type: string
diff --git a/controllers/galera_controller.go b/controllers/galera_controller.go
@@ -509,16 +509,23 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 		// service get stuck.
 		controllerutil.AddFinalizer(service, helper.GetFinalizer())
 
-		// NOTE(dciabrin) We deploy Galera as an A/P service (i.e. no multi-master writes)
-		// by setting labels in the service's label selectors.
-		// This label is dynamically set based on the status of the Galera cluster,
-		// so in this CreateOrPatch block we must reuse whatever is present in
-		// the existing service CR in case we're patching it.
-		activePod, present := service.Spec.Selector[mariadb.ActivePodSelectorKey]
-		service.Spec = pkgsvc.Spec
-		if present {
-			service.Spec.Selector[mariadb.ActivePodSelectorKey] = activePod
+		if !instance.Spec.EnableMultiMaster {
+			// NOTE(dciabrin) We deploy Galera as an A/P service (i.e. no multi-master writes)
+			// by setting labels in the service's label selectors.
+			// This label is dynamically set based on the status of the Galera cluster,
+			// so in this CreateOrPatch block we must reuse whatever is present in
+			// the existing service CR in case we're patching it.
+			activePod, present := service.Spec.Selector[mariadb.ActivePodSelectorKey]
+			service.Spec = pkgsvc.Spec
+
+			if present {
+				service.Spec.Selector[mariadb.ActivePodSelectorKey] = activePod
+			}
+		} else {
+			service.Spec = pkgsvc.Spec
+			delete(service.Spec.Selector, mariadb.ActivePodSelectorKey)
 		}
+
 		err := controllerutil.SetControllerReference(instance, service, r.Client.Scheme())
 		if err != nil {
 			return err
diff --git a/controllers/mariadbdatabase_controller.go b/controllers/mariadbdatabase_controller.go
@@ -27,6 +27,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
 	helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper"
@@ -145,7 +147,7 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 
 	// here we know that Galera exists so add a finalizer to ourselves and to the db CR. Before this point there is no reason to have a finalizer on ourselves as nothing to cleanup.
 	if instance.DeletionTimestamp.IsZero() || isNewInstance { // this condition can be removed if you wish as it is always true at this point otherwise we would returned earlier.
-		if controllerutil.AddFinalizer(dbGalera, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) {
+		if dbGalera.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(dbGalera, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) {
 			err := r.Update(ctx, dbGalera)
 			if err != nil {
 				return ctrl.Result{}, err
@@ -200,6 +202,9 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		return ctrl.Result{}, err
 	}
 
+	// DB instances setup for multi master
+	instance.Status.EnableMultiMaster = dbGalera.Spec.EnableMultiMaster
+
 	dbCreateHash := instance.Status.Hash[databasev1beta1.DbCreateHash]
 	dbCreateJob := job.NewJob(
 		jobDef,
@@ -245,8 +250,43 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 
 // SetupWithManager -
 func (r *MariaDBDatabaseReconciler) SetupWithManager(mgr ctrl.Manager) error {
+
+	updateStatusFn := func(ctx context.Context, o client.Object) []reconcile.Request {
+		log := GetLog(ctx, "MariaDBDatabase")
+
+		result := []reconcile.Request{}
+
+		mariaDBDatabases := &databasev1beta1.MariaDBDatabaseList{}
+
+		listOpts := []client.ListOption{
+			client.InNamespace(o.GetNamespace()),
+		}
+		if err := r.Client.List(ctx, mariaDBDatabases, listOpts...); err != nil {
+			log.Error(err, "Unable to retrieve MariaDBDatabase CRs %w")
+			return nil
+		}
+
+		for _, cr := range mariaDBDatabases.Items {
+
+			if o.GetName() == cr.GetLabels()["dbName"] {
+				name := client.ObjectKey{
+					Namespace: o.GetNamespace(),
+					Name:      cr.Name,
+				}
+				log.Info(fmt.Sprintf("Galera %s is used by MariaDBDatabase CR %s", o.GetName(), cr.Name))
+				result = append(result, reconcile.Request{NamespacedName: name})
+			}
+		}
+
+		if len(result) > 0 {
+			return result
+		}
+		return nil
+	}
+
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&databasev1beta1.MariaDBDatabase{}).
+		Watches(&databasev1beta1.Galera{}, handler.EnqueueRequestsFromMapFunc(updateStatusFn)).
 		Complete(r)
 }
 
diff --git a/tests/kuttl/tests/galera_cluster_restart/04-teardown.yaml b/tests/kuttl/tests/galera_cluster_restart/04-teardown.yaml
@@ -0,0 +1,10 @@
+apiVersion: kuttl.dev/v1beta
+kind: TestStep
+delete:
+- apiVersion: mariadb.openstack.org/v1beta1
+  kind: Galera
+  name: openstack
+commands:
+  - script: |
+      oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2
+      for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done
diff --git a/tests/kuttl/tests/galera_log_to_disk/03-teardown.yaml b/tests/kuttl/tests/galera_log_to_disk/03-teardown.yaml
@@ -0,0 +1,10 @@
+apiVersion: kuttl.dev/v1beta
+kind: TestStep
+delete:
+- apiVersion: mariadb.openstack.org/v1beta1
+  kind: Galera
+  name: openstack
+commands:
+  - script: |
+      oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2
+      for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done
diff --git a/tests/kuttl/tests/galera_multi_master/01-deploy_ap_galera.yaml b/tests/kuttl/tests/galera_multi_master/01-deploy_ap_galera.yaml
@@ -0,0 +1,19 @@
+apiVersion: mariadb.openstack.org/v1beta1
+kind: Galera
+metadata:
+  name: openstack
+spec:
+  secret: osp-secret
+  storageClass: local-storage
+  storageRequest: 500M
+  replicas: 3
+  enableMultiMaster: false
+---
+apiVersion: mariadb.openstack.org/v1beta1
+kind: MariaDBDatabase
+metadata:
+  name: mydatabase
+  labels:
+    dbName: openstack
+spec:
+  name: mydatabase
diff --git a/tests/kuttl/tests/galera_multi_master/02-assert.yaml b/tests/kuttl/tests/galera_multi_master/02-assert.yaml
@@ -0,0 +1,131 @@
+#
+# Check for:
+#
+# - 1 MariaDB CR
+# - 1 Pod for MariaDB CR
+#
+
+apiVersion: mariadb.openstack.org/v1beta1
+kind: Galera
+metadata:
+  name: openstack
+spec:
+  replicas: 3
+  secret: osp-secret
+  storageRequest: 500M
+status:
+  bootstrapped: true
+  conditions:
+  - message: Setup complete
+    reason: Ready
+    status: "True"
+    type: Ready
+  - message: Deployment completed
+    reason: Ready
+    status: "True"
+    type: DeploymentReady
+  - message: Exposing service completed
+    reason: Ready
+    status: "True"
+    type: ExposeServiceReady
+  - message: Input data complete
+    reason: Ready
+    status: "True"
+    type: InputReady
+  - message: RoleBinding created
+    reason: Ready
+    status: "True"
+    type: RoleBindingReady
+  - message: Role created
+    reason: Ready
+    status: "True"
+    type: RoleReady
+  - message: ServiceAccount created
+    reason: Ready
+    status: "True"
+    type: ServiceAccountReady
+  - message: Service config create completed
+    reason: Ready
+    status: "True"
+    type: ServiceConfigReady
+  - message: Input data complete
+    reason: Ready
+    status: "True"
+    type: TLSInputReady
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: openstack-galera
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: galera
+      cr: galera-openstack
+      galera/name: openstack
+  serviceName: openstack-galera
+  template:
+    metadata:
+      labels:
+        app: galera
+        cr: galera-openstack
+        galera/name: openstack
+    spec:
+      containers:
+      - command:
+        - /usr/bin/dumb-init
+        - --
+        - /usr/local/bin/kolla_start
+        name: galera
+        ports:
+        - containerPort: 3306
+          name: mysql
+          protocol: TCP
+        - containerPort: 4567
+          name: galera
+          protocol: TCP
+      serviceAccount: galera-openstack
+      serviceAccountName: galera-openstack
+status:
+  availableReplicas: 3
+  readyReplicas: 3
+  replicas: 3
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: openstack-galera-0
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: openstack-galera-1
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: openstack-galera-2
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: openstack-galera
+spec:
+  ports:
+  - name: mysql
+    port: 3306
+    protocol: TCP
+    targetPort: 3306
+  selector:
+    app: galera
+    cr: galera-openstack
+---
+apiVersion: mariadb.openstack.org/v1beta1
+kind: MariaDBDatabase
+metadata:
+  labels:
+    dbName: openstack
+  name: mydatabase
+spec:
+  name: mydatabase
diff --git a/tests/kuttl/tests/galera_multi_master/03-assert.yaml b/tests/kuttl/tests/galera_multi_master/03-assert.yaml
@@ -0,0 +1,7 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: |
+      # ensure kubernetes pod key present
+      oc get -n ${NAMESPACE} service openstack -o yaml | grep "statefulset.kubernetes.io/pod-name: openstack-galera-"
diff --git a/tests/kuttl/tests/galera_multi_master/04-deploy_mm_galera.yaml b/tests/kuttl/tests/galera_multi_master/04-deploy_mm_galera.yaml
@@ -0,0 +1,10 @@
+apiVersion: mariadb.openstack.org/v1beta1
+kind: Galera
+metadata:
+  name: openstack
+spec:
+  secret: osp-secret
+  storageClass: local-storage
+  storageRequest: 500M
+  replicas: 3
+  enableMultiMaster: true
diff --git a/tests/kuttl/tests/galera_multi_master/05-assert.yaml b/tests/kuttl/tests/galera_multi_master/05-assert.yaml
@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: |
+      # ensure kubernetes pod key *not* present
+      ! oc get -n ${NAMESPACE} service openstack -o yaml | grep "statefulset.kubernetes.io/pod-name: openstack-galera-"
diff --git a/tests/kuttl/tests/galera_multi_master/06-assert.yaml b/tests/kuttl/tests/galera_multi_master/06-assert.yaml
diff --git a/tests/kuttl/tests/galera_multi_master/07-teardown.yaml b/tests/kuttl/tests/galera_multi_master/07-teardown.yaml

Original file line number	Diff line number	Diff line change
`@@ -89,6 +89,9 @@ type GaleraSpecCore struct {`
`89`	`89`	`// +kubebuilder:validation:Optional`
`90`	`90`	`// Log Galera pod's output to disk`
`91`	`91`	LogToDisk bool `json:"logToDisk"`
	`92`	`+ // +kubebuilder:default=false`
	`93`	`+ // Use multi-master service routing`
	`94`	+ EnableMultiMaster bool `json:"enableMultiMaster"`
`92`	`95`	`}`
`93`	`96`
`94`	`97`	`// GaleraAttributes holds startup information for a Galera host`
Original file line number	Diff line number	Diff line change
`@@ -55,6 +55,9 @@ type MariaDBDatabaseStatus struct {`
`55`	`55`
`56`	`56`	`// Whether TLS is supported by the DB instance`
`57`	`57`	TLSSupport bool `json:"tlsSupport,omitempty"`
	`58`	`+`
	`59`	`+ // whether the DB instance is using multi-master routing`
	`60`	+ EnableMultiMaster bool `json:"enableMultiMaster,omitempty"`
`58`	`61`	`}`
`59`	`62`
`60`	`63`	`//+kubebuilder:object:root=true`