Get MYSQL_PWD using an on-demand cluster query

In order to facilitate an in-place change to the name of the
Secret that is referenced by a Galera instance for the
mysql root password, rework
the approach used by pods and shell scripts to no longer
require the root secret name and/or password be passed by
environment variable, instead using a pod-level cluster
query to retrieve the current root password.  The logic
to retrieve this password is encapsulated into a single
shell script that is present as a volume mount on running containers.

This allows Job objects to be created with hashes that
do not link to a specific Secret name, as well as to
create StatefulSet objects that don't refer to this name.
When the Secret name changes on a Galera instance for
an in-place root password change, the hashes / CRs for
these objects will remain unchanged.

A subsequent change to the mariadb operator will add the ability
to change the mysql root password of a Galera cluster using a
dual-reference architecture where
the "current" root secret will be part of <CR>/Status, while
the secret referenced in <CR>/Spec will be the "new" root
secret.  When these two names differ, that will indicate an
in-place password change should take place, as well
as allowing the pre-existing root password to be available
at the same time as the new one in order to do a root password
change.   The same
architecture will be applied to a new class of "system" MariaDBAccount
objects that are for use only by the Galera instance itself
and do not have a link to any MariaDBDatabase CR.  The
Galera CR itself will no longer use osp-secret
for the mysql root password nor will the secret be directly
referenced from the Galera CR, instead referenced by a
"system" MariaDBAccount CR which the Galera operator itself
will create.

Author: zzzeek

controllers/galera_controller.go

@@ -486,6 +486,21 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 			Resources: []string{"services"},
 			Verbs:     []string{"get", "list", "update", "patch"},
 		},
+		{
+			APIGroups: []string{"mariadb.openstack.org"},
+			Resources: []string{"galeras"},
+			Verbs:     []string{"get", "list"},
+		},
+		{
+			APIGroups: []string{"mariadb.openstack.org"},
+			Resources: []string{"mariadbaccounts"},
+			Verbs:     []string{"get"},
+		},
+		{
+			APIGroups: []string{""},
+			Resources: []string{"secrets"},
+			Verbs:     []string{"get"},
+		},
 	}
 	rbacResult, err := common_rbac.ReconcileRbac(ctx, helper, instance, rbacRules)
 	if err != nil {
@@ -935,19 +950,21 @@ func (r *GaleraReconciler) generateConfigMaps(
 ) error {
 	log := GetLog(ctx, "galera")
 	templateParameters := map[string]interface{}{
-		"logToDisk": instance.Spec.LogToDisk,
+		"logToDisk":          instance.Spec.LogToDisk,
+		"galeraInstanceName": instance.Name,
 	}
 	customData := make(map[string]string)
 	customData[mariadbv1.CustomServiceConfigFile] = instance.Spec.CustomServiceConfig
 
 	cms := []util.Template{
 		// ScriptsConfigMap
 		{
-			Name:         configMapNameForScripts(instance),
-			Namespace:    instance.Namespace,
-			Type:         util.TemplateTypeScripts,
-			InstanceType: instance.Kind,
-			Labels:       map[string]string{},
+			Name:          configMapNameForScripts(instance),
+			Namespace:     instance.Namespace,
+			Type:          util.TemplateTypeScripts,
+			InstanceType:  instance.Kind,
+			Labels:        map[string]string{},
+			ConfigOptions: templateParameters,
 		},
 		// ConfigMap
 		{

controllers/mariadbaccount_controller.go

@@ -242,14 +242,13 @@ func (r *MariaDBAccountReconciler) reconcileCreate(
 		return ctrl.Result{}, err
 	}
 
-	var dbAdminSecret, dbContainerImage, serviceAccountName string
+	var dbContainerImage, serviceAccountName string
 
 	if !dbGalera.Status.Bootstrapped {
 		log.Info("DB bootstrap not complete. Requeue...")
 		return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, nil
 	}
 
-	dbAdminSecret = dbGalera.Spec.Secret
 	dbContainerImage = dbGalera.Spec.ContainerImage
 	serviceAccountName = dbGalera.RbacResourceName()
 
@@ -287,7 +286,7 @@ func (r *MariaDBAccountReconciler) reconcileCreate(
 
 	log.Info(fmt.Sprintf("Running account create '%s' MariaDBDatabase '%s'", instance.Name, mariadbDatabaseName))
 
-	jobDef, err := mariadb.CreateDbAccountJob(instance, mariadbDatabase.Spec.Name, dbHostname, dbAdminSecret, dbContainerImage, serviceAccountName, dbGalera.Spec.NodeSelector)
+	jobDef, err := mariadb.CreateDbAccountJob(dbGalera, instance, mariadbDatabase.Spec.Name, dbHostname, dbContainerImage, serviceAccountName, dbGalera.Spec.NodeSelector)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
@@ -478,7 +477,7 @@ func (r *MariaDBAccountReconciler) reconcileDelete(
 		}
 	}
 
-	var dbAdminSecret, dbContainerImage, serviceAccountName string
+	var dbContainerImage, serviceAccountName string
 
 	if !dbGalera.Status.Bootstrapped {
 		log.Info("DB bootstrap not complete. Requeue...")
@@ -493,7 +492,6 @@ func (r *MariaDBAccountReconciler) reconcileDelete(
 		return ctrl.Result{RequeueAfter: time.Second * 10}, nil
 	}
 
-	dbAdminSecret = dbGalera.Spec.Secret
 	dbContainerImage = dbGalera.Spec.ContainerImage
 	serviceAccountName = dbGalera.RbacResourceName()
 
@@ -512,7 +510,7 @@ func (r *MariaDBAccountReconciler) reconcileDelete(
 
 	log.Info(fmt.Sprintf("Running account delete '%s' MariaDBDatabase '%s'", instance.Name, mariadbDatabaseName))
 
-	jobDef, err := mariadb.DeleteDbAccountJob(instance, mariadbDatabase.Spec.Name, dbHostname, dbAdminSecret, dbContainerImage, serviceAccountName, dbGalera.Spec.NodeSelector)
+	jobDef, err := mariadb.DeleteDbAccountJob(dbGalera, instance, mariadbDatabase.Spec.Name, dbHostname, dbContainerImage, serviceAccountName, dbGalera.Spec.NodeSelector)
 	if err != nil {
 		return ctrl.Result{}, err
 	}

controllers/mariadbdatabase_controller.go

@@ -165,7 +165,7 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	//
 	// Non-deletion (normal) flow follows
 	//
-	var dbSecret, dbContainerImage, serviceAccount string
+	var dbContainerImage, serviceAccount string
 	// NOTE(dciabrin) When configured to only allow TLS connections, all clients
 	// accessing this DB must support client connection via TLS.
 	useTLS := dbGalera.Spec.TLS.Enabled() && dbGalera.Spec.DisableNonTLSListeners
@@ -183,7 +183,6 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		return ctrl.Result{RequeueAfter: time.Second * 10}, nil
 	}
 
-	dbSecret = dbGalera.Spec.Secret
 	dbContainerImage = dbGalera.Spec.ContainerImage
 	serviceAccount = dbGalera.RbacResourceName()
 
@@ -199,7 +198,7 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	)
 
 	// Define a new Job object (hostname, password, containerImage)
-	jobDef, err := mariadb.DbDatabaseJob(instance, dbHostname, dbSecret, dbContainerImage, serviceAccount, useTLS, dbGalera.Spec.NodeSelector)
+	jobDef, err := mariadb.DbDatabaseJob(dbGalera, instance, dbHostname, dbContainerImage, serviceAccount, useTLS, dbGalera.Spec.NodeSelector)
 	if err != nil {
 		return ctrl.Result{}, err
 	}

pkg/mariadb/account.go

@@ -5,6 +5,7 @@ import (
 
 	util "github.com/openstack-k8s-operators/lib-common/modules/common/util"
 	databasev1beta1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
+	mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -18,7 +19,7 @@ type accountCreateOrDeleteOptions struct {
 	RequireTLS            string
 }
 
-func CreateDbAccountJob(account *databasev1beta1.MariaDBAccount, databaseName string, databaseHostName string, databaseSecret string, containerImage string, serviceAccountName string, nodeSelector *map[string]string) (*batchv1.Job, error) {
+func CreateDbAccountJob(galera *mariadbv1.Galera, account *databasev1beta1.MariaDBAccount, databaseName string, databaseHostName string, containerImage string, serviceAccountName string, nodeSelector *map[string]string) (*batchv1.Job, error) {
 	var tlsStatement string
 	if account.Spec.RequireTLS {
 		tlsStatement = " REQUIRE SSL"
@@ -60,17 +61,6 @@ func CreateDbAccountJob(account *databasev1beta1.MariaDBAccount, databaseName st
 							Image:   containerImage,
 							Command: []string{"/bin/sh", "-c", dbCmd},
 							Env: []corev1.EnvVar{
-								{
-									Name: "MYSQL_PWD",
-									ValueFrom: &corev1.EnvVarSource{
-										SecretKeyRef: &corev1.SecretKeySelector{
-											LocalObjectReference: corev1.LocalObjectReference{
-												Name: databaseSecret,
-											},
-											Key: databasev1beta1.DbRootPasswordSelector,
-										},
-									},
-								},
 								{
 									Name: "DatabasePassword",
 									ValueFrom: &corev1.EnvVarSource{
@@ -83,8 +73,10 @@ func CreateDbAccountJob(account *databasev1beta1.MariaDBAccount, databaseName st
 									},
 								},
 							},
+							VolumeMounts: getGaleraRootOnlyVolumeMounts(),
 						},
 					},
+					Volumes: getGaleraRootOnlyVolumes(galera),
 				},
 			},
 		},
@@ -97,7 +89,7 @@ func CreateDbAccountJob(account *databasev1beta1.MariaDBAccount, databaseName st
 	return job, nil
 }
 
-func DeleteDbAccountJob(account *databasev1beta1.MariaDBAccount, databaseName string, databaseHostName string, databaseSecret string, containerImage string, serviceAccountName string, nodeSelector *map[string]string) (*batchv1.Job, error) {
+func DeleteDbAccountJob(galera *mariadbv1.Galera, account *databasev1beta1.MariaDBAccount, databaseName string, databaseHostName string, containerImage string, serviceAccountName string, nodeSelector *map[string]string) (*batchv1.Job, error) {
 
 	opts := accountCreateOrDeleteOptions{account.Spec.UserName, databaseName, databaseHostName, "root", ""}
 
@@ -121,24 +113,13 @@ func DeleteDbAccountJob(account *databasev1beta1.MariaDBAccount, databaseName st
 					ServiceAccountName: serviceAccountName,
 					Containers: []corev1.Container{
 						{
-							Name:    "mariadb-account-delete",
-							Image:   containerImage,
-							Command: []string{"/bin/sh", "-c", delCmd},
-							Env: []corev1.EnvVar{
-								{
-									Name: "MYSQL_PWD",
-									ValueFrom: &corev1.EnvVarSource{
-										SecretKeyRef: &corev1.SecretKeySelector{
-											LocalObjectReference: corev1.LocalObjectReference{
-												Name: databaseSecret,
-											},
-											Key: databasev1beta1.DbRootPasswordSelector,
-										},
-									},
-								},
-							},
+							Name:         "mariadb-account-delete",
+							Image:        containerImage,
+							Command:      []string{"/bin/sh", "-c", delCmd},
+							VolumeMounts: getGaleraRootOnlyVolumeMounts(),
 						},
 					},
+					Volumes: getGaleraRootOnlyVolumes(galera),
 				},
 			},
 		},

pkg/mariadb/database.go

@@ -5,6 +5,7 @@ import (
 
 	util "github.com/openstack-k8s-operators/lib-common/modules/common/util"
 	databasev1beta1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
+	mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -20,7 +21,7 @@ type dbCreateOptions struct {
 }
 
 // DbDatabaseJob -
-func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName string, databaseSecret string, containerImage string, serviceAccountName string, useTLS bool, nodeSelector *map[string]string) (*batchv1.Job, error) {
+func DbDatabaseJob(galera *mariadbv1.Galera, database *databasev1beta1.MariaDBDatabase, databaseHostName string, containerImage string, serviceAccountName string, useTLS bool, nodeSelector *map[string]string) (*batchv1.Job, error) {
 	var tlsStatement string
 	if useTLS {
 		tlsStatement = " REQUIRE SSL"
@@ -48,17 +49,6 @@ func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName s
 
 	if database.Spec.Secret != nil {
 		scriptEnv = []corev1.EnvVar{
-			{
-				Name: "MYSQL_PWD",
-				ValueFrom: &corev1.EnvVarSource{
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: databaseSecret,
-						},
-						Key: "DbRootPassword",
-					},
-				},
-			},
 			// send deprecated Secret field but only if non-nil
 			{
 				Name: "DatabasePassword",
@@ -73,19 +63,7 @@ func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName s
 			},
 		}
 	} else {
-		scriptEnv = []corev1.EnvVar{
-			{
-				Name: "MYSQL_PWD",
-				ValueFrom: &corev1.EnvVarSource{
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: databaseSecret,
-						},
-						Key: "DbRootPassword",
-					},
-				},
-			},
-		}
+		scriptEnv = []corev1.EnvVar{}
 	}
 
 	job := &batchv1.Job{
@@ -104,12 +82,14 @@ func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName s
 					ServiceAccountName: serviceAccountName,
 					Containers: []corev1.Container{
 						{
-							Name:    "mariadb-database-create",
-							Image:   containerImage,
-							Command: []string{"/bin/sh", "-c", dbCmd},
-							Env:     scriptEnv,
+							Name:         "mariadb-database-create",
+							Image:        containerImage,
+							Command:      []string{"/bin/sh", "-c", dbCmd},
+							Env:          scriptEnv,
+							VolumeMounts: getGaleraRootOnlyVolumeMounts(),
 						},
 					},
+					Volumes: getGaleraRootOnlyVolumes(galera),
 				},
 			},
 		},
@@ -123,7 +103,7 @@ func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName s
 }
 
 // DeleteDbDatabaseJob -
-func DeleteDbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName string, databaseSecret string, containerImage string, serviceAccountName string, nodeSelector *map[string]string) (*batchv1.Job, error) {
+func DeleteDbDatabaseJob(galera *mariadbv1.Galera, database *databasev1beta1.MariaDBDatabase, databaseHostName string, containerImage string, serviceAccountName string, nodeSelector *map[string]string) (*batchv1.Job, error) {
 
 	opts := dbCreateOptions{
 		database.Spec.Name,
@@ -145,17 +125,6 @@ func DeleteDbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHost
 
 	if database.Spec.Secret != nil {
 		scriptEnv = []corev1.EnvVar{
-			{
-				Name: "MYSQL_PWD",
-				ValueFrom: &corev1.EnvVarSource{
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: databaseSecret,
-						},
-						Key: databasev1beta1.DbRootPasswordSelector,
-					},
-				},
-			},
 			// send deprecated Secret field but only if non-nil.  otherwise
 			// the script should not try to drop usernames from mysql.user
 			{
@@ -171,19 +140,7 @@ func DeleteDbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHost
 			},
 		}
 	} else {
-		scriptEnv = []corev1.EnvVar{
-			{
-				Name: "MYSQL_PWD",
-				ValueFrom: &corev1.EnvVarSource{
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{
-							Name: databaseSecret,
-						},
-						Key: databasev1beta1.DbRootPasswordSelector,
-					},
-				},
-			},
-		}
+		scriptEnv = []corev1.EnvVar{}
 	}
 
 	job := &batchv1.Job{
@@ -199,12 +156,14 @@ func DeleteDbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHost
 					ServiceAccountName: serviceAccountName,
 					Containers: []corev1.Container{
 						{
-							Name:    "mariadb-database-create",
-							Image:   containerImage,
-							Command: []string{"/bin/sh", "-c", delCmd},
-							Env:     scriptEnv,
+							Name:         "mariadb-database-create",
+							Image:        containerImage,
+							Command:      []string{"/bin/sh", "-c", delCmd},
+							Env:          scriptEnv,
+							VolumeMounts: getGaleraRootOnlyVolumeMounts(),
 						},
 					},
+					Volumes: getGaleraRootOnlyVolumes(galera),
 				},
 			},
 		},

pkg/mariadb/statefulset.go

@@ -104,16 +104,6 @@ func getGaleraInitContainers(g *mariadbv1.Galera) []corev1.Container {
 		}, {
 			Name:  "KOLLA_CONFIG_STRATEGY",
 			Value: "COPY_ALWAYS",
-		}, {
-			Name: "DB_ROOT_PASSWORD",
-			ValueFrom: &corev1.EnvVarSource{
-				SecretKeyRef: &corev1.SecretKeySelector{
-					LocalObjectReference: corev1.LocalObjectReference{
-						Name: g.Spec.Secret,
-					},
-					Key: "DbRootPassword",
-				},
-			},
 		}},
 		Resources:    g.Spec.Resources,
 		VolumeMounts: getGaleraInitVolumeMounts(g),
@@ -132,16 +122,6 @@ func getGaleraContainers(g *mariadbv1.Galera, configHash string) []corev1.Contai
 		}, {
 			Name:  "KOLLA_CONFIG_STRATEGY",
 			Value: "COPY_ALWAYS",
-		}, {
-			Name: "DB_ROOT_PASSWORD",
-			ValueFrom: &corev1.EnvVarSource{
-				SecretKeyRef: &corev1.SecretKeySelector{
-					LocalObjectReference: corev1.LocalObjectReference{
-						Name: g.Spec.Secret,
-					},
-					Key: "DbRootPassword",
-				},
-			},
 		}},
 		Ports: []corev1.ContainerPort{{
 			ContainerPort: 3306,