diff --git a/api/bases/mariadb.openstack.org_galeras.yaml b/api/bases/mariadb.openstack.org_galeras.yaml index f1c4e576..ca91f689 100644 --- a/api/bases/mariadb.openstack.org_galeras.yaml +++ b/api/bases/mariadb.openstack.org_galeras.yaml @@ -137,8 +137,18 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + rootDatabaseAccount: + description: |- + RootDatabaseAccount - name of MariaDBAccount which will be used to + generate root account / password. + this account is generated if not exists, and a name is chosen based + on a naming convention if not present + type: string secret: - description: Name of the secret to look for password keys + description: |- + Name of the legacy secret to locate the initial galera root + password + this field will be removed once scripts can adjust to using root_auth.sh type: string storageClass: description: Storage class to host the mariadb databases @@ -177,7 +187,6 @@ spec: required: - containerImage - replicas - - secret - storageClass - storageRequest type: object @@ -296,6 +305,9 @@ spec: the opentack-operator in the top-level CR (e.g. the ContainerImage) format: int64 type: integer + rootDatabaseSecret: + description: name of the Secret that is being used for the root password + type: string safeToBootstrap: description: Name of the node that can safely bootstrap a cluster type: string diff --git a/api/bases/mariadb.openstack.org_mariadbaccounts.yaml b/api/bases/mariadb.openstack.org_mariadbaccounts.yaml index 9a92f28f..51dd7196 100644 --- a/api/bases/mariadb.openstack.org_mariadbaccounts.yaml +++ b/api/bases/mariadb.openstack.org_mariadbaccounts.yaml @@ -48,6 +48,12 @@ spec: spec: description: MariaDBAccountSpec defines the desired state of MariaDBAccount properties: + accountType: + default: User + enum: + - User + - System + type: string requireTLS: default: false description: Account must use TLS to connect to the database @@ -108,6 +114,14 @@ spec: - type type: object type: array + currentSecret: + description: |- + the Secret that's currently in use for the account. + keeping a handle to this secret allows us to remove its finalizer + when it's replaced with a new one. It also is useful for storing + the current "root" secret separate from a newly proposed one which is + needed when changing the database root password. + type: string hash: additionalProperties: type: string diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index 7c816838..5c60a306 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -54,6 +54,9 @@ const ( // ReasonDBServiceNameError - error getting the DB service hostname ReasonDBServiceNameError condition.Reason = "DatabaseServiceNameError" + // ReasonDBResourceDeleted - the galera resource has been marked for deletion + ReasonDBResourceDeleted condition.Reason = "DatabaseResourceDeleted" + // ReasonDBSync - Database sync in progress ReasonDBSync condition.Reason = "DBSync" ) @@ -92,8 +95,12 @@ const ( MariaDBServerNotBootstrappedMessage = "MariaDB / Galera server not bootstrapped" + MariaDBServerDeletedMessage = "MariaDB / Galera server has been marked for deletion" + MariaDBAccountReadyInitMessage = "MariaDBAccount create / drop not started" + MariaDBSystemAccountReadyMessage = "MariaDBAccount System account '%s' creation complete" + MariaDBAccountReadyMessage = "MariaDBAccount creation complete" MariaDBAccountNotReadyMessage = "MariaDBAccount is not present: %s" diff --git a/api/v1beta1/galera_types.go b/api/v1beta1/galera_types.go index 7767d3f3..f813fbdd 100644 --- a/api/v1beta1/galera_types.go +++ b/api/v1beta1/galera_types.go @@ -51,9 +51,19 @@ type GaleraSpec struct { // GaleraSpec defines the desired state of Galera type GaleraSpecCore struct { - // Name of the secret to look for password keys - // +kubebuilder:validation:Required + // Name of the legacy secret to locate the initial galera root + // password + // this field will be removed once scripts can adjust to using root_auth.sh + // +kubebuilder:validation:Optional Secret string `json:"secret"` + + // RootDatabaseAccount - name of MariaDBAccount which will be used to + // generate root account / password. + // this account is generated if not exists, and a name is chosen based + // on a naming convention if not present + // +kubebuilder:validation:Optional + RootDatabaseAccount string `json:"rootDatabaseAccount"` + // Storage class to host the mariadb databases // +kubebuilder:validation:Required StorageClass string `json:"storageClass"` @@ -113,6 +123,9 @@ type GaleraAttributes struct { type GaleraStatus struct { // A map of database node attributes for each pod Attributes map[string]GaleraAttributes `json:"attributes,omitempty"` + // name of the Secret that is being used for the root password + // +kubebuilder:validation:Optional + RootDatabaseSecret string `json:"rootDatabaseSecret"` // Name of the node that can safely bootstrap a cluster SafeToBootstrap string `json:"safeToBootstrap,omitempty"` // Is the galera cluster currently running diff --git a/api/v1beta1/mariadbaccount_types.go b/api/v1beta1/mariadbaccount_types.go index 32819281..6b0b13f4 100644 --- a/api/v1beta1/mariadbaccount_types.go +++ b/api/v1beta1/mariadbaccount_types.go @@ -28,9 +28,6 @@ const ( // AccountDeleteHash hash AccountDeleteHash = "accountdelete" - // DbRootPassword selector for galera root account - DbRootPasswordSelector = "DbRootPassword" - // DatabasePassword selector for MariaDBAccount->Secret DatabasePasswordSelector = "DatabasePassword" ) @@ -48,10 +45,28 @@ type MariaDBAccountSpec struct { // Account must use TLS to connect to the database // +kubebuilder:default=false RequireTLS bool `json:"requireTLS"` + + // +kubebuilder:validation:Enum=User;System + // +kubebuilder:default=User + AccountType AccountType `json:"accountType,omitempty"` } +type AccountType string + +const ( + User AccountType = "User" + System AccountType = "System" +) + // MariaDBAccountStatus defines the observed state of MariaDBAccount type MariaDBAccountStatus struct { + // the Secret that's currently in use for the account. + // keeping a handle to this secret allows us to remove its finalizer + // when it's replaced with a new one. It also is useful for storing + // the current "root" secret separate from a newly proposed one which is + // needed when changing the database root password. + CurrentSecret string `json:"currentSecret,omitempty"` + // Deployment Conditions Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` @@ -85,3 +100,11 @@ type MariaDBAccountList struct { func init() { SchemeBuilder.Register(&MariaDBAccount{}, &MariaDBAccountList{}) } + +func (mariadbAccount MariaDBAccount) IsSystemAccount() bool { + return mariadbAccount.Spec.AccountType == System +} + +func (mariadbAccount MariaDBAccount) IsUserAccount() bool { + return mariadbAccount.Spec.AccountType == "" || mariadbAccount.Spec.AccountType == User +} diff --git a/api/v1beta1/mariadbdatabase_funcs.go b/api/v1beta1/mariadbdatabase_funcs.go index 33d84f9b..91211999 100644 --- a/api/v1beta1/mariadbdatabase_funcs.go +++ b/api/v1beta1/mariadbdatabase_funcs.go @@ -541,6 +541,50 @@ func DeleteDatabaseAndAccountFinalizers( namespace string, ) error { + err := DeleteAccountFinalizers( + ctx, + h, + accountName, + namespace, + ) + if err != nil { + return err + } + + // also do a delete for "unused" MariaDBAccounts, associated with + // this MariaDBDatabase. + err = DeleteUnusedMariaDBAccountFinalizers( + ctx, h, name, accountName, namespace, + ) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + + mariaDBDatabase, err := GetDatabase(ctx, h, name, namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } else if err == nil && controllerutil.RemoveFinalizer(mariaDBDatabase, h.GetFinalizer()) { + err := h.GetClient().Update(ctx, mariaDBDatabase) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + util.LogForObject(h, fmt.Sprintf("Removed finalizer %s from MariaDBDatabase %s", h.GetFinalizer(), mariaDBDatabase.Spec.Name), mariaDBDatabase) + } + + return nil +} + +// DeleteAccountFinalizers performs just the primary account + secret finalizer +// removal part of DeleteDatabaseAndAccountFinalizers +func DeleteAccountFinalizers( + ctx context.Context, + h *helper.Helper, + accountName string, + namespace string, +) error { + if accountName == "" { + return fmt.Errorf("Account name is blank") + } databaseAccount, err := GetAccount(ctx, h, accountName, namespace) if err != nil && !k8s_errors.IsNotFound(err) { return err @@ -572,26 +616,6 @@ func DeleteDatabaseAndAccountFinalizers( } } - // also do a delete for "unused" MariaDBAccounts, associated with - // this MariaDBDatabase. - err = DeleteUnusedMariaDBAccountFinalizers( - ctx, h, name, accountName, namespace, - ) - if err != nil && !k8s_errors.IsNotFound(err) { - return err - } - - mariaDBDatabase, err := GetDatabase(ctx, h, name, namespace) - if err != nil && !k8s_errors.IsNotFound(err) { - return err - } else if err == nil && controllerutil.RemoveFinalizer(mariaDBDatabase, h.GetFinalizer()) { - err := h.GetClient().Update(ctx, mariaDBDatabase) - if err != nil && !k8s_errors.IsNotFound(err) { - return err - } - util.LogForObject(h, fmt.Sprintf("Removed finalizer %s from MariaDBDatabase %s", h.GetFinalizer(), mariaDBDatabase.Spec.Name), mariaDBDatabase) - } - return nil } @@ -806,6 +830,32 @@ func EnsureMariaDBAccount(ctx context.Context, userNamePrefix string, ) (*MariaDBAccount, *corev1.Secret, error) { + return ensureMariaDBAccount( + ctx, helper, accountName, namespace, requireTLS, + userNamePrefix, "", "", map[string]string{}) + +} + +// EnsureMariaDBSystemAccount ensures a MariaDBAccount has been created for a given +// operator calling the function, and returns the MariaDBAccount and its +// Secret for use in consumption into a configuration. +// Unlike EnsureMariaDBAccount, the function accepts an exact username that +// expected to remain constant, supporting in-place password changes for the +// account. +func EnsureMariaDBSystemAccount(ctx context.Context, + helper *helper.Helper, + accountName string, galeraInstanceName string, namespace string, requireTLS bool, + exactUserName string, exactPassword string) (*MariaDBAccount, *corev1.Secret, error) { + return ensureMariaDBAccount( + ctx, helper, accountName, namespace, requireTLS, + "", exactUserName, exactPassword, map[string]string{"dbName": galeraInstanceName}) +} + +func ensureMariaDBAccount(ctx context.Context, + helper *helper.Helper, + accountName string, namespace string, requireTLS bool, + userNamePrefix string, exactUserName string, exactPassword string, labels map[string]string, +) (*MariaDBAccount, *corev1.Secret, error) { if accountName == "" { return nil, nil, fmt.Errorf("accountName is empty") } @@ -817,9 +867,20 @@ func EnsureMariaDBAccount(ctx context.Context, return nil, nil, err } - username, err := generateUniqueUsername(userNamePrefix) - if err != nil { - return nil, nil, err + var username string + var accountType AccountType + + if exactUserName == "" { + accountType = "User" + username, err = generateUniqueUsername(userNamePrefix) + if err != nil { + return nil, nil, err + } + } else if userNamePrefix != "" { + return nil, nil, fmt.Errorf("userNamePrefix and exactUserName are mutually exclusive") + } else { + accountType = "System" + username = exactUserName } account = &MariaDBAccount{ @@ -832,9 +893,10 @@ func EnsureMariaDBAccount(ctx context.Context, // MariaDBAccount once this is filled in }, Spec: MariaDBAccountSpec{ - UserName: username, - Secret: fmt.Sprintf("%s-db-secret", accountName), - RequireTLS: requireTLS, + UserName: username, + Secret: fmt.Sprintf("%s-db-secret", accountName), + RequireTLS: requireTLS, + AccountType: accountType, }, } @@ -844,6 +906,7 @@ func EnsureMariaDBAccount(ctx context.Context, if account.Spec.Secret == "" { account.Spec.Secret = fmt.Sprintf("%s-db-secret", accountName) } + } dbSecret, _, err := secret.GetSecret(ctx, helper, account.Spec.Secret, namespace) @@ -853,9 +916,14 @@ func EnsureMariaDBAccount(ctx context.Context, return nil, nil, err } - dbPassword, err := generateDBPassword() - if err != nil { - return nil, nil, err + var dbPassword string + if exactPassword == "" { + dbPassword, err = generateDBPassword() + if err != nil { + return nil, nil, err + } + } else { + dbPassword = exactPassword } dbSecret = &corev1.Secret{ @@ -869,7 +937,7 @@ func EnsureMariaDBAccount(ctx context.Context, } } - _, err = createOrPatchAccountAndSecret(ctx, helper, account, dbSecret, map[string]string{}) + _, err = createOrPatchAccountAndSecret(ctx, helper, account, dbSecret, labels) if err != nil { return nil, nil, err } @@ -885,6 +953,7 @@ func EnsureMariaDBAccount(ctx context.Context, ) return account, dbSecret, nil + } // generateUniqueUsername creates a MySQL-compliant database username based on diff --git a/config/crd/bases/mariadb.openstack.org_galeras.yaml b/config/crd/bases/mariadb.openstack.org_galeras.yaml index f1c4e576..ca91f689 100644 --- a/config/crd/bases/mariadb.openstack.org_galeras.yaml +++ b/config/crd/bases/mariadb.openstack.org_galeras.yaml @@ -137,8 +137,18 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + rootDatabaseAccount: + description: |- + RootDatabaseAccount - name of MariaDBAccount which will be used to + generate root account / password. + this account is generated if not exists, and a name is chosen based + on a naming convention if not present + type: string secret: - description: Name of the secret to look for password keys + description: |- + Name of the legacy secret to locate the initial galera root + password + this field will be removed once scripts can adjust to using root_auth.sh type: string storageClass: description: Storage class to host the mariadb databases @@ -177,7 +187,6 @@ spec: required: - containerImage - replicas - - secret - storageClass - storageRequest type: object @@ -296,6 +305,9 @@ spec: the opentack-operator in the top-level CR (e.g. the ContainerImage) format: int64 type: integer + rootDatabaseSecret: + description: name of the Secret that is being used for the root password + type: string safeToBootstrap: description: Name of the node that can safely bootstrap a cluster type: string diff --git a/config/crd/bases/mariadb.openstack.org_mariadbaccounts.yaml b/config/crd/bases/mariadb.openstack.org_mariadbaccounts.yaml index 9a92f28f..51dd7196 100644 --- a/config/crd/bases/mariadb.openstack.org_mariadbaccounts.yaml +++ b/config/crd/bases/mariadb.openstack.org_mariadbaccounts.yaml @@ -48,6 +48,12 @@ spec: spec: description: MariaDBAccountSpec defines the desired state of MariaDBAccount properties: + accountType: + default: User + enum: + - User + - System + type: string requireTLS: default: false description: Account must use TLS to connect to the database @@ -108,6 +114,14 @@ spec: - type type: object type: array + currentSecret: + description: |- + the Secret that's currently in use for the account. + keeping a handle to this secret allows us to remove its finalizer + when it's replaced with a new one. It also is useful for storing + the current "root" secret separate from a newly proposed one which is + needed when changing the database root password. + type: string hash: additionalProperties: type: string diff --git a/controllers/galera_controller.go b/controllers/galera_controller.go index 7b81606c..3ea62321 100644 --- a/controllers/galera_controller.go +++ b/controllers/galera_controller.go @@ -35,7 +35,7 @@ import ( helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/common/labels" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" - secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/service" commonstatefulset "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" @@ -500,6 +500,11 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res Resources: []string{"galeras"}, Verbs: []string{"get", "list"}, }, + { + APIGroups: []string{"mariadb.openstack.org"}, + Resources: []string{"mariadbaccounts"}, + Verbs: []string{"get", "list"}, + }, { APIGroups: []string{""}, Resources: []string{"secrets"}, @@ -567,6 +572,8 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res log.Info("", "Kind", instance.Kind, "Name", instance.Name, "database service", service.Name, "operation", string(op)) } + instance.IsReady() + instance.Status.Conditions.MarkTrue(condition.CreateServiceReadyCondition, condition.CreateServiceReadyMessage) // Map of all resources that may cause a rolling service restart @@ -576,8 +583,10 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res clusterPropertiesEnv := make(map[string]env.Setter) // Check and hash inputs - // NOTE do not hash the db root password, as its change requires - // more orchestration than a simple rolling restart + + // ******** TEMPORARY ************ + // Pull the DbRootPassword from osp-secret to allow CI / external + // scripts to work, until they can adapt to root_auth.sh _, res, err := secret.VerifySecret( ctx, types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret}, @@ -597,15 +606,67 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res condition.InputReadyWaitingMessage)) return res, fmt.Errorf("%w: %s", ErrOpenStackSecretNotFound, instance.Spec.Secret) } + return ctrl.Result{}, err + } + + legacySecret, _, err := secret.GetSecret(ctx, helper, instance.Spec.Secret, instance.Namespace) + if err != nil { + return ctrl.Result{}, err + } + legacyRootPassword := string(legacySecret.Data["DbRootPassword"]) + // ******** END TEMPORARY ************ + + databaseAccountName := r.getRootMariadbAccountName(instance) + + mariaDBAccount, rootSecret, err := mariadbv1.EnsureMariaDBSystemAccount( + ctx, helper, databaseAccountName, + instance.Name, instance.Namespace, false, "root", legacyRootPassword) + + if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, + mariadbv1.MariaDBAccountReadyCondition, condition.ErrorReason, condition.SeverityWarning, - condition.InputReadyErrorMessage, + mariadbv1.MariaDBAccountNotReadyMessage, err.Error())) + return ctrl.Result{}, err } - instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + + // the current root secret for the MariaDBAccount name is copied out to Status. + // this allows us to avoid having to change the Spec, which seems to be not + // best practice here and also seems to interfere with the openstack + // controller's ability to see the Galera status as Completed (though this + // might only be due to CRDs not being in sync between mariadb-operator and + // openstack-operator during development) + // Also by trying to always use CurrentSecret if available, we try to keep + // instance.Status.RootDatabaseSecret as the secret that should work for + // root right now, independent of changes in the mariadbaccount name + // or its secret that have not been reconciled w/ a new job hash. + if mariaDBAccount.Status.CurrentSecret != "" { + instance.Status.RootDatabaseSecret = mariaDBAccount.Status.CurrentSecret + } else if instance.Status.RootDatabaseSecret == "" { + // if we dont have any Status.RootDatabaseSecret at all, then use the + // new secret we just got. assume this is prior to bootstrap + instance.Status.RootDatabaseSecret = rootSecret.Name + } + // otherwise if mariaDBAccount.Status.CurrentSecret == "" and + // instance.Status.RootDatabaseSecret != "", this means we've already + // assigned rootSecret.Name to instance.Status.RootDatabaseSecret up front. + // if rootSecret.name is now different from what it was originally, that + // means the mariadbaccount was updated, and we assume is in the process + // of updating the root PW. if mariaDBAccount.Status.CurrentSecret is blank + // while this is happening, this pretty much means a test suite is changing + // the root pw on a brand new galera that hasn't had a chance to set up + // CurrentSecret in the first place, so just wait for that. + + instance.Status.Conditions.MarkTrue( + mariadbv1.MariaDBAccountReadyCondition, + mariadbv1.MariaDBSystemAccountReadyMessage, "root") + + instance.Status.Conditions.MarkTrue( + condition.InputReadyCondition, + condition.InputReadyMessage) // // TLS input validation @@ -1055,12 +1116,16 @@ func (r *GaleraReconciler) SetupWithManager(mgr ctrl.Manager) error { Watches(&topologyv1.Topology{}, handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Watches( + &mariadbv1.MariaDBAccount{}, + handler.EnqueueRequestsFromMapFunc(r.findGaleraForMariaDBAccount), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } -// GetDatabaseObject - returns either a Galera or MariaDB object (and an associated client.Object interface). +// GetDatabaseObject - returns a Galera object. // used by both MariaDBDatabaseReconciler and MariaDBAccountReconciler -// this will later return only Galera objects, so as a lookup it's part of the galera controller func GetDatabaseObject(ctx context.Context, clientObj client.Client, name string, namespace string) (*mariadbv1.Galera, error) { dbGalera := &mariadbv1.Galera{ ObjectMeta: metav1.ObjectMeta{ @@ -1114,6 +1179,53 @@ func (r *GaleraReconciler) findObjectsForSrc(ctx context.Context, src client.Obj return requests } +// findGaleraForMariaDBAccount - returns reconcile requests for Galera instances +// that should reconcile when a MariaDBAccount changes +func (r *GaleraReconciler) findGaleraForMariaDBAccount(ctx context.Context, account client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + mariadbAccount := account.(*mariadbv1.MariaDBAccount) + + // Only reconcile for system accounts, as these are used by Galera for root credentials + if !mariadbAccount.IsSystemAccount() { + return requests + } + + // List all Galera instances in the same namespace + galeraList := &mariadbv1.GaleraList{} + if err := r.List(ctx, galeraList, client.InNamespace(mariadbAccount.Namespace)); err != nil { + log.FromContext(ctx).Error(err, "Unable to list Galera instances for MariaDBAccount", "account", mariadbAccount.Name) + return requests + } + + // Find Galera instances that use this MariaDBAccount + for _, galera := range galeraList.Items { + // Check if this Galera uses this MariaDBAccount for root credentials + rootAccountName := r.getRootMariadbAccountName(&galera) + if rootAccountName == mariadbAccount.Name { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: galera.Name, + Namespace: galera.Namespace, + }, + }) + log.FromContext(ctx).Info("MariaDBAccount changed, triggering Galera reconcile", + "account", mariadbAccount.Name, + "galera", galera.Name) + } + } + + return requests +} + +func (r *GaleraReconciler) getRootMariadbAccountName(instance *mariadbv1.Galera) string { + databaseAccountName := instance.Spec.RootDatabaseAccount + if databaseAccountName == "" { + databaseAccountName = instance.Name + "-mariadb-root" + } + return databaseAccountName +} + func (r *GaleraReconciler) reconcileDelete(ctx context.Context, instance *mariadbv1.Galera, helper *helper.Helper) (ctrl.Result, error) { helper.GetLogger().Info("Reconciling Service delete") @@ -1141,6 +1253,13 @@ func (r *GaleraReconciler) reconcileDelete(ctx context.Context, instance *mariad return ctrlResult, err } + // remove finalizer from the system mariadbaccount and associated secret + err = mariadbv1.DeleteAccountFinalizers( + ctx, helper, r.getRootMariadbAccountName(instance), instance.Namespace) + if err != nil { + return ctrl.Result{}, err + } + // Service is deleted so remove the finalizer. controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) helper.GetLogger().Info("Reconciled Service delete successfully") diff --git a/controllers/mariadbaccount_controller.go b/controllers/mariadbaccount_controller.go index 498cffe3..ae5d4129 100644 --- a/controllers/mariadbaccount_controller.go +++ b/controllers/mariadbaccount_controller.go @@ -30,6 +30,7 @@ import ( util "github.com/openstack-k8s-operators/lib-common/modules/common/util" databasev1beta1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" mariadb "github.com/openstack-k8s-operators/mariadb-operator/pkg/mariadb" + batchv1 "k8s.io/api/batch/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -113,32 +114,45 @@ func (r *MariaDBAccountReconciler) Reconcile(ctx context.Context, req ctrl.Reque }() // initialize conditions used later as Status=Unknown - cl := condition.CreateList( - condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), - condition.UnknownCondition(databasev1beta1.MariaDBServerReadyCondition, condition.InitReason, databasev1beta1.MariaDBServerReadyInitMessage), - condition.UnknownCondition(databasev1beta1.MariaDBDatabaseReadyCondition, condition.InitReason, databasev1beta1.MariaDBDatabaseReadyInitMessage), - condition.UnknownCondition(databasev1beta1.MariaDBAccountReadyCondition, condition.InitReason, databasev1beta1.MariaDBAccountReadyInitMessage), - ) + cl := condition.Conditions{ + *condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + *condition.UnknownCondition(databasev1beta1.MariaDBServerReadyCondition, condition.InitReason, databasev1beta1.MariaDBServerReadyInitMessage), + } + if instance.IsUserAccount() { + // user accounts also need the database ready condition + // the element is being inserted into the list in a specific location + // to preseve expectations of tests suites that are hardcoded to + // expect a specific ordering of Conditions in YAML displays + cl = append(cl, *condition.UnknownCondition(databasev1beta1.MariaDBDatabaseReadyCondition, condition.InitReason, databasev1beta1.MariaDBDatabaseReadyInitMessage)) + } + cl = append(cl, *condition.UnknownCondition(databasev1beta1.MariaDBAccountReadyCondition, condition.InitReason, databasev1beta1.MariaDBAccountReadyInitMessage)) instance.Status.Conditions.Init(&cl) if instance.DeletionTimestamp.IsZero() || isNewInstance { //revive:disable:indent-error-flow - return r.reconcileCreate(ctx, log, helper, instance) + return r.reconcileCreateOrUpdate(ctx, log, helper, instance) } else { return r.reconcileDelete(ctx, log, helper, instance) } } -// reconcileDelete - run reconcile for case where delete timestamp is zero -func (r *MariaDBAccountReconciler) reconcileCreate( +// reconcileCreateOrUpdate - run reconcile for case where delete timestamp is zero +func (r *MariaDBAccountReconciler) reconcileCreateOrUpdate( ctx context.Context, log logr.Logger, helper *helper.Helper, instance *databasev1beta1.MariaDBAccount) (result ctrl.Result, _err error) { - // get a handle to the current, active MariaDBDatabase. - // if not ready yet, requeue. - mariadbDatabase, result, err := r.getMariaDBDatabaseForCreate(ctx, log, instance) - if mariadbDatabase == nil { - return result, err + var mariadbDatabase *databasev1beta1.MariaDBDatabase + var err error + + log.Info("Reconcile MariaDBAccount create or update") + + if instance.IsUserAccount() { + // for User account, get a handle to the current, active MariaDBDatabase. + // if not ready yet, requeue. + mariadbDatabase, result, err = r.getMariaDBDatabaseForCreate(ctx, log, instance) + if mariadbDatabase == nil { + return result, err + } } if controllerutil.AddFinalizer(instance, helper.GetFinalizer()) { @@ -146,11 +160,13 @@ func (r *MariaDBAccountReconciler) reconcileCreate( return ctrl.Result{}, nil } - // MariaDBdatabase exists and we are a create case. ensure finalizers set up - if controllerutil.AddFinalizer(mariadbDatabase, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) { - err = r.Update(ctx, mariadbDatabase) - if err != nil { - return ctrl.Result{}, err + if instance.IsUserAccount() { + // MariaDBdatabase exists and we are a create case. ensure finalizers set up + if controllerutil.AddFinalizer(mariadbDatabase, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) { + err = r.Update(ctx, mariadbDatabase) + if err != nil { + return ctrl.Result{}, err + } } } @@ -174,13 +190,34 @@ func (r *MariaDBAccountReconciler) reconcileCreate( return result, err } - log.Info(fmt.Sprintf("Running account create '%s' MariaDBDatabase '%s'", - instance.Name, mariadbDatabase.Spec.Name)) - jobDef, err := mariadb.CreateDbAccountJob( - dbGalera, instance, mariadbDatabase.Spec.Name, dbHostname, - dbContainerImage, serviceAccountName, dbGalera.Spec.NodeSelector) - if err != nil { - return ctrl.Result{}, err + var jobDef *batchv1.Job + + var createOrUpdate string + if instance.Status.CurrentSecret != "" { + createOrUpdate = "update" + } else { + createOrUpdate = "create" + } + + if instance.IsUserAccount() { + log.Info(fmt.Sprintf("Checking %s account job '%s' MariaDBDatabase '%s'", + createOrUpdate, + instance.Name, mariadbDatabase.Spec.Name)) + jobDef, err = mariadb.CreateOrUpdateDbAccountJob( + dbGalera, instance, mariadbDatabase.Spec.Name, dbHostname, + dbContainerImage, serviceAccountName, dbGalera.Spec.NodeSelector) + if err != nil { + return ctrl.Result{}, err + } + } else { + log.Info(fmt.Sprintf("Checking %s system account job '%s'", createOrUpdate, + instance.Name)) + jobDef, err = mariadb.CreateOrUpdateDbAccountJob( + dbGalera, instance, "", dbHostname, + dbContainerImage, serviceAccountName, dbGalera.Spec.NodeSelector) + if err != nil { + return ctrl.Result{}, err + } } accountCreateHash := instance.Status.Hash[databasev1beta1.AccountCreateHash] @@ -200,11 +237,24 @@ func (r *MariaDBAccountReconciler) reconcileCreate( } if accountCreateJob.HasChanged() { + if instance.Status.Hash == nil { instance.Status.Hash = make(map[string]string) } instance.Status.Hash[databasev1beta1.AccountCreateHash] = accountCreateJob.GetHash() - log.Info(fmt.Sprintf("Job %s hash added - %s", jobDef.Name, instance.Status.Hash[databasev1beta1.AccountCreateHash])) + log.Info(fmt.Sprintf("Job %s hash created or updated - %s", jobDef.Name, instance.Status.Hash[databasev1beta1.AccountCreateHash])) + + // set up new Secret and remove finalizer from old secret + if instance.Status.CurrentSecret != instance.Spec.Secret { + currentSecret := instance.Status.CurrentSecret + err = r.removeSecretFinalizer(ctx, log, helper, currentSecret, instance.Namespace) + if err == nil { + instance.Status.CurrentSecret = instance.Spec.Secret + } else { + return ctrl.Result{}, err + } + } + } // database creation finished @@ -228,9 +278,16 @@ func (r *MariaDBAccountReconciler) reconcileDelete( ctx context.Context, log logr.Logger, helper *helper.Helper, instance *databasev1beta1.MariaDBAccount) (result ctrl.Result, _err error) { - mariadbDatabase, result, err := r.getMariaDBDatabaseForDelete(ctx, log, helper, instance) - if mariadbDatabase == nil { - return result, err + var mariadbDatabase *databasev1beta1.MariaDBDatabase + var err error + + log.Info("Reconcile MariaDBAccount delete") + + if instance.IsUserAccount() { + mariadbDatabase, result, err = r.getMariaDBDatabaseForDelete(ctx, log, helper, instance) + if mariadbDatabase == nil { + return result, err + } } // dont do actual DROP USER until finalizers from downstream controllers @@ -243,6 +300,7 @@ func (r *MariaDBAccountReconciler) reconcileDelete( finalizersWeCareAbout = append(finalizersWeCareAbout, f) } } + if len(finalizersWeCareAbout) > 0 { instance.Status.Conditions.MarkFalse( databasev1beta1.MariaDBAccountReadyCondition, @@ -259,54 +317,83 @@ func (r *MariaDBAccountReconciler) reconcileDelete( databasev1beta1.MariaDBAccountReadyForDeleteMessage, ) - instance.Status.Conditions.MarkTrue( - databasev1beta1.MariaDBDatabaseReadyCondition, - databasev1beta1.MariaDBDatabaseReadyMessage, - ) + if instance.IsUserAccount() { + instance.Status.Conditions.MarkTrue( + databasev1beta1.MariaDBDatabaseReadyCondition, + databasev1beta1.MariaDBDatabaseReadyMessage, + ) + } // now proceed to do actual work. acquire the Galera instance // which will lead us to the hostname and container image to target dbGalera, dbHostname, result, err := r.getGaleraForCreateOrDelete( ctx, log, helper, instance, mariadbDatabase, ) - // if Galera CR was not found at all, this indicates MariaDBAccount is not - // implemented in the database either, so remove all finalizers and - // exit + + // if Galera CR was not found or in a deletion process, this indicates + // we won't ever have a DB server with which to run a DROP for the + // account, so remove all finalizers and exit + + var galeraGone bool + if k8s_errors.IsNotFound(err) { + log.Info("Galera instance not found, so we will remove finalizers and skip account delete") + galeraGone = true + } else if err != nil { + // unexpected error code + return result, err + } else if dbGalera == nil || !dbGalera.DeletionTimestamp.IsZero() { + log.Info("Galera deleted or deletion timestamp is non-zero, so we will remove finalizers and skip account delete") + galeraGone = true + } else { + galeraGone = false + } + + if galeraGone { // remove MariaDBAccount finalizer from MariaDBDatabase - this takes the // form such as openstack.org/mariadbaccount- (this naming // scheme allows multiple MariaDBAccounts to claim the same MariaDBDatabase // as a dependency) and allows a delete of the MariaDBDatabase to proceed // assuming no other finalizers - if controllerutil.RemoveFinalizer(mariadbDatabase, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) { - err = r.Update(ctx, mariadbDatabase) + if instance.IsUserAccount() { + if controllerutil.RemoveFinalizer(mariadbDatabase, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) { + err = r.Update(ctx, mariadbDatabase) - if err != nil && !k8s_errors.IsNotFound(err) { - return ctrl.Result{}, err + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } } - } // remove finalizer "openstack.org/mariadbaccount" from both the // MariaDBAccount as well as the Secret which is referenced from the // MariaDBAccount, allowing both to be deleted assuming no other // finalizers - err = r.removeAccountAndSecretFinalizer(ctx, helper, instance) + err := r.removeAccountAndSecretFinalizer(ctx, log, helper, instance) return ctrl.Result{}, err - } else if dbGalera == nil { - return result, err } dbContainerImage := dbGalera.Spec.ContainerImage serviceAccountName := dbGalera.RbacResourceName() - log.Info(fmt.Sprintf("Running account delete '%s' MariaDBDatabase '%s'", instance.Name, mariadbDatabase.Spec.Name)) + var jobDef *batchv1.Job - jobDef, err := mariadb.DeleteDbAccountJob(dbGalera, instance, mariadbDatabase.Spec.Name, dbHostname, dbContainerImage, serviceAccountName, dbGalera.Spec.NodeSelector) - if err != nil { - return ctrl.Result{}, err - } + if instance.IsUserAccount() { + + log.Info(fmt.Sprintf("Running account delete '%s' MariaDBDatabase '%s'", instance.Name, mariadbDatabase.Spec.Name)) + jobDef, err = mariadb.DeleteDbAccountJob(dbGalera, instance, mariadbDatabase.Spec.Name, dbHostname, dbContainerImage, serviceAccountName, dbGalera.Spec.NodeSelector) + if err != nil { + return ctrl.Result{}, err + } + } else { + log.Info(fmt.Sprintf("Running system account delete '%s'", instance.Name)) + + jobDef, err = mariadb.DeleteDbAccountJob(dbGalera, instance, "", dbHostname, dbContainerImage, serviceAccountName, dbGalera.Spec.NodeSelector) + if err != nil { + return ctrl.Result{}, err + } + } accountDeleteHash := instance.Status.Hash[databasev1beta1.AccountDeleteHash] accountDeleteJob := job.NewJob( jobDef, @@ -335,18 +422,19 @@ func (r *MariaDBAccountReconciler) reconcileDelete( // scheme allows multiple MariaDBAccounts to claim the same MariaDBDatabase // as a dependency) and allows a delete of the MariaDBDatabase to proceed // assuming no other finalizers - if controllerutil.RemoveFinalizer(mariadbDatabase, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) { - err = r.Update(ctx, mariadbDatabase) - if err != nil { - return ctrl.Result{}, err + if instance.IsUserAccount() { + if controllerutil.RemoveFinalizer(mariadbDatabase, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) { + err = r.Update(ctx, mariadbDatabase) + if err != nil { + return ctrl.Result{}, err + } } } // remove finalizer "openstack.org/mariadbaccount" from // both the MariaDBAccount as well as the Secret which is referenced // from the MariaDBAccount, allowing both to be deleted - err = r.removeAccountAndSecretFinalizer(ctx, helper, instance) - + err = r.removeAccountAndSecretFinalizer(ctx, log, helper, instance) return ctrl.Result{}, err } @@ -447,7 +535,7 @@ func (r *MariaDBAccountReconciler) getMariaDBDatabaseForDelete(ctx context.Conte // So remove finalizer "openstack.org/mariadbaccount" from // both the MariaDBAccount as well as the Secret which is referenced // from the MariaDBAccount, allowing both to be deleted - err := r.removeAccountAndSecretFinalizer(ctx, helper, instance) + err := r.removeAccountAndSecretFinalizer(ctx, log, helper, instance) return nil, ctrl.Result{}, err } @@ -467,7 +555,7 @@ func (r *MariaDBAccountReconciler) getMariaDBDatabaseForDelete(ctx context.Conte // So remove finalizer "openstack.org/mariadbaccount" from // both the MariaDBAccount as well as the Secret which is referenced // from the MariaDBAccount, allowing both to be deleted - err = r.removeAccountAndSecretFinalizer(ctx, helper, instance) + err = r.removeAccountAndSecretFinalizer(ctx, log, helper, instance) return nil, ctrl.Result{}, err } else { // unhandled error; exit without change @@ -510,7 +598,7 @@ func (r *MariaDBAccountReconciler) getMariaDBDatabaseForDelete(ctx context.Conte // remove finalizer "openstack.org/mariadbaccount" from // both the MariaDBAccount as well as the Secret which is referenced // from the MariaDBAccount - err = r.removeAccountAndSecretFinalizer(ctx, helper, instance) + err = r.removeAccountAndSecretFinalizer(ctx, log, helper, instance) return nil, ctrl.Result{}, err } @@ -526,12 +614,24 @@ func (r *MariaDBAccountReconciler) getGaleraForCreateOrDelete( helper *helper.Helper, instance *databasev1beta1.MariaDBAccount, mariadbDatabase *databasev1beta1.MariaDBDatabase) (*databasev1beta1.Galera, string, ctrl.Result, error) { - dbName := mariadbDatabase.Labels["dbName"] + var dbGalera *databasev1beta1.Galera + var err error + var dbName string - dbGalera, err := GetDatabaseObject(ctx, r.Client, dbName, instance.Namespace) + if instance.IsUserAccount() { + dbName = mariadbDatabase.Labels["dbName"] + } else { + // note mariadbDatabase is passed as nil in this case + dbName = instance.Labels["dbName"] + } + dbGalera, err = GetDatabaseObject(ctx, r.Client, dbName, instance.Namespace) if err != nil { - log.Error(err, "Error retrieving Galera instance") + if k8s_errors.IsNotFound(err) { + log.Info(fmt.Sprintf("Galera instance '%s' does not exist", dbName)) + } else { + log.Error(err, "Error retrieving Galera instance") + } instance.Status.Conditions.Set(condition.FalseCondition( databasev1beta1.MariaDBServerReadyCondition, @@ -556,6 +656,19 @@ func (r *MariaDBAccountReconciler) getGaleraForCreateOrDelete( return nil, "", ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, nil } + if !dbGalera.DeletionTimestamp.IsZero() { + log.Info("DB server marked for deletion, preventing account operations from proceeding. Will seek to remove finalizers and exit") + + instance.Status.Conditions.MarkFalse( + databasev1beta1.MariaDBServerReadyCondition, + databasev1beta1.ReasonDBResourceDeleted, + condition.SeverityInfo, + databasev1beta1.MariaDBServerDeletedMessage, + ) + + return dbGalera, "", ctrl.Result{}, nil + } + dbHostname, dbHostResult, err := databasev1beta1.GetServiceHostname(ctx, helper, dbGalera.Name, dbGalera.Namespace) if (err != nil || dbHostResult != ctrl.Result{}) { @@ -655,24 +768,41 @@ func (r *MariaDBAccountReconciler) ensureAccountSecret( // removeAccountAndSecretFinalizer - removes finalizer from mariadbaccount as well // as current primary secret func (r *MariaDBAccountReconciler) removeAccountAndSecretFinalizer(ctx context.Context, - helper *helper.Helper, instance *databasev1beta1.MariaDBAccount) error { + log logr.Logger, helper *helper.Helper, instance *databasev1beta1.MariaDBAccount) error { + + err := r.removeSecretFinalizer( + ctx, log, helper, instance.Spec.Secret, instance.Namespace, + ) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + + // remove mariadbaccount finalizer which will update at end of reconcile + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + + return nil +} - accountSecret, _, err := secret.GetSecret(ctx, helper, instance.Spec.Secret, instance.Namespace) +func (r *MariaDBAccountReconciler) removeSecretFinalizer(ctx context.Context, + log logr.Logger, helper *helper.Helper, secretName string, namespace string) error { + accountSecret, _, err := secret.GetSecret(ctx, helper, secretName, namespace) if err == nil { if controllerutil.RemoveFinalizer(accountSecret, helper.GetFinalizer()) { err = r.Update(ctx, accountSecret) if err != nil { + log.Error( + err, + fmt.Sprintf("Error removing mariadbaccount finalizer from secret '%s', will try again", secretName)) return err + } else { + log.Info(fmt.Sprintf("Successfully removed mariadbaccount finalizer from secret '%s'", secretName)) } } } else if !k8s_errors.IsNotFound(err) { return err } - // will take effect when reconcile ends - controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) - return nil } diff --git a/pkg/mariadb/account.go b/pkg/mariadb/account.go index 0dbf897f..15e9e87a 100644 --- a/pkg/mariadb/account.go +++ b/pkg/mariadb/account.go @@ -19,8 +19,8 @@ type accountCreateOrDeleteOptions struct { RequireTLS string } -// CreateDbAccountJob creates a Kubernetes job for creating a MariaDB database account -func CreateDbAccountJob(galera *mariadbv1.Galera, account *mariadbv1.MariaDBAccount, databaseName string, databaseHostName string, containerImage string, serviceAccountName string, nodeSelector *map[string]string) (*batchv1.Job, error) { +// CreateOrUpdateDbAccountJob creates or updates a Kubernetes job for creating a MariaDB database account +func CreateOrUpdateDbAccountJob(galera *mariadbv1.Galera, account *mariadbv1.MariaDBAccount, databaseName string, databaseHostName string, containerImage string, serviceAccountName string, nodeSelector *map[string]string) (*batchv1.Job, error) { var tlsStatement string if account.Spec.RequireTLS { tlsStatement = " REQUIRE SSL" @@ -47,7 +47,7 @@ func CreateDbAccountJob(galera *mariadbv1.Galera, account *mariadbv1.MariaDBAcco // provided db name is used as metadata name where underscore is a not allowed // character. Lets replace all underscores with hypen. Underscores in the db name are // possible. - Name: strings.ReplaceAll(account.Spec.UserName, "_", "-") + "-account-create", + Name: strings.ReplaceAll(account.Spec.UserName, "_", "-") + "-account-create-update", Namespace: account.Namespace, Labels: labels, }, @@ -58,7 +58,7 @@ func CreateDbAccountJob(galera *mariadbv1.Galera, account *mariadbv1.MariaDBAcco ServiceAccountName: serviceAccountName, Containers: []corev1.Container{ { - Name: "mariadb-account-create", + Name: "mariadb-account-create-update", Image: containerImage, Command: []string{"/bin/sh", "-c", dbCmd}, Env: []corev1.EnvVar{ diff --git a/templates/account.sh b/templates/account.sh index ba076cb3..0692337c 100755 --- a/templates/account.sh +++ b/templates/account.sh @@ -1,15 +1,42 @@ #!/bin/bash -MYSQL_REMOTE_HOST={{.DatabaseHostname}} source /var/lib/operator-scripts/mysql_root_auth.sh +MYSQL_REMOTE_HOST="{{.DatabaseHostname}}" source /var/lib/operator-scripts/mysql_root_auth.sh export DatabasePassword=${DatabasePassword:?"Please specify a DatabasePassword variable."} -mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -P 3306 -e "GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.UserName}}'@'localhost' IDENTIFIED BY '$DatabasePassword'{{.RequireTLS}};GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.UserName}}'@'%' IDENTIFIED BY '$DatabasePassword'{{.RequireTLS}};" +MYSQL_CMD="mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -P 3306" +if [ -n "{{.DatabaseName}}" ]; then + GRANT_DATABASE="{{.DatabaseName}}" +else + GRANT_DATABASE="*" +fi + +# going for maximum compatibility here: +# 1. MySQL 8 no longer allows implicit create user when GRANT is used +# 2. MariaDB has "CREATE OR REPLACE", but MySQL does not +# 3. create user with CREATE but then do all password and TLS with ALTER to +# support updates + +$MYSQL_CMD <Status->rootDatabaseSecret) and if not working, see if there is a +# different (newer) password in root galera->Spec->rootDatabaseAccount->Secret, +# and try that. This suits the case where a new password was placed in +# galera->Spec->rootDatabaseAccount->Secret, account.sh ran to update the root +# password, but failed to complete, even though the actual password got +# updated. account.sh will run again on a new pod but the password that's in +# galera->Status->rootDatabaseSecret is no longer valid, and would prevent +# account.sh from proceeding a second time. Try the "pending" password just to +# get through, so that account.sh can succeed and +# galera->Status->rootDatabaseSecret can then be updated. + +PASSWORD_VALID=true + +# test password with mysql command if socket exists, or we are remote +if [ "${MYSQL_ROOT_AUTH_BYPASS_CHECKS}" != "true" ] && { [ "${USE_SOCKET}" = "false" ] || [ -S "${MYSQL_SOCKET}" ]; }; then + if ! mysql ${MYSQL_CONN_PARAMS} -uroot -p"${PASSWORD}" -e "SELECT 1;" >/dev/null 2>&1; then + echo "WARNING: primary password retrieved from cluster failed authentication; will try fallback password" >&2 + PASSWORD_VALID=false + fi +fi + +# if password failed, look for alternate password from the mariadbdatabaseaccount +# spec directly. assume we are in root pw flight +if [ "${PASSWORD_VALID}" = "false" ]; then + + MARIADB_ACCOUNT=$(echo "${GALERA_CR}" | python3 -c "import json, sys; print(json.load(sys.stdin)['spec']['rootDatabaseAccount'] or '${GALERA_INSTANCE}-mariadb-root')") -# test again; warn if it doesn't work, however write to my.cnf in any -# case to allow the calling script to continue -if [ "${USE_SOCKET}" = "false" ] || [ -S "${MYSQL_SOCKET}" ]; then + MARIADB_ACCOUNT_CR=$(curl -s \ + --cacert ${CACERT} \ + --header "Content-Type:application/json" \ + --header "Authorization: Bearer ${TOKEN}" \ + "${APISERVER}/${MARIADB_API}/namespaces/${NAMESPACE}/mariadbaccounts/${MARIADB_ACCOUNT}") + + # look in spec.secret + FALLBACK_SECRET_NAME=$(echo "${MARIADB_ACCOUNT_CR}" | python3 -c "import json, sys; print(json.load(sys.stdin)['spec']['secret'])") + + # Get the new password from the fallback secret + PASSWORD=$(curl -s \ + --cacert ${CACERT} \ + --header "Content-Type:application/json" \ + --header "Authorization: Bearer ${TOKEN}" \ + "${APISERVER}/${K8S_API}/namespaces/${NAMESPACE}/secrets/${FALLBACK_SECRET_NAME}" \ + | python3 -c "import json, sys; print(json.load(sys.stdin)['data']['DatabasePassword'])" \ + | base64 -d) + + # test again; warn if it doesn't work, however write to my.cnf in any + # case to allow the calling script to continue if ! mysql ${MYSQL_CONN_PARAMS} -uroot -p"${PASSWORD}" -e "SELECT 1;" >/dev/null 2>&1; then - echo "WARNING: password retrieved from cluster failed authentication" >&2 + echo "WARNING: Both primary and fallback passwords failed authentication, will maintain fallback password" >&2 fi + fi diff --git a/tests/chainsaw/common/galera-assert.yaml b/tests/chainsaw/common/galera-assert.yaml index 3ffdd0ab..6834a81c 100644 --- a/tests/chainsaw/common/galera-assert.yaml +++ b/tests/chainsaw/common/galera-assert.yaml @@ -25,6 +25,10 @@ status: reason: Ready status: "True" type: InputReady + - message: MariaDBAccount System account 'root' creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady - message: RoleBinding created reason: Ready status: "True" diff --git a/tests/chainsaw/common/system-account-assert.yaml b/tests/chainsaw/common/system-account-assert.yaml new file mode 100644 index 00000000..1398dec6 --- /dev/null +++ b/tests/chainsaw/common/system-account-assert.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBAccount +metadata: + labels: + dbName: openstack + name: chainsawdb-some-system-db-account +status: + currentSecret: some-system-db-secret + conditions: + - message: Setup complete + reason: Ready + status: "True" + type: Ready + - message: MariaDBAccount creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady + - message: MariaDB / Galera server ready + reason: Ready + status: "True" + type: MariaDBServerReady +--- +apiVersion: v1 +data: + DatabasePassword: ZGJzZWNyZXQx +kind: Secret +metadata: + name: some-system-db-secret + # ensure finalizer was added + finalizers: + - openstack.org/mariadbaccount +type: Opaque +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: systemuser-account-create-update +spec: + template: + spec: {} +status: + succeeded: 1 diff --git a/tests/chainsaw/common/system-account-secret.yaml b/tests/chainsaw/common/system-account-secret.yaml new file mode 100644 index 00000000..c97d3756 --- /dev/null +++ b/tests/chainsaw/common/system-account-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +data: + # dbsecret1 + DatabasePassword: ZGJzZWNyZXQx +kind: Secret +metadata: + name: some-system-db-secret +type: Opaque diff --git a/tests/chainsaw/common/system-account.yaml b/tests/chainsaw/common/system-account.yaml new file mode 100644 index 00000000..2ad267cc --- /dev/null +++ b/tests/chainsaw/common/system-account.yaml @@ -0,0 +1,10 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBAccount +metadata: + labels: + dbName: openstack + name: chainsawdb-some-system-db-account +spec: + userName: systemuser + secret: some-system-db-secret + accountType: System diff --git a/tests/chainsaw/scripts/check_db_account.sh b/tests/chainsaw/scripts/check_db_account.sh new file mode 100755 index 00000000..d0186971 --- /dev/null +++ b/tests/chainsaw/scripts/check_db_account.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +set -x + +galera="$1" +dbname="$2" +username="$3" +password="$4" + + +found=0 +not_found=1 + +if [ "$5" = "--reverse" ];then + # sometimes we want to check that a user does not exist + found=1 + not_found=0 +fi + +found_username=$(oc exec -n ${NAMESPACE} -c galera ${galera} -- /bin/sh -c 'source /var/lib/operator-scripts/mysql_root_auth.sh; mysql -uroot -p${DB_ROOT_PASSWORD} -Nse "select user from mysql.user"' | grep -o -w ${username}) + +# username was not found, exit +if [ -z "$found_username" ]; then + exit $not_found +fi + +# username was found. if we wanted it to be found, then check the login also. +if [ "$found" = "0" ]; then + if [ -n "$dbname" ]; then + oc exec -n ${NAMESPACE} -c galera ${galera} -- /bin/sh -c "mysql -u${username} -p${password} -Nse 'select database();' ${dbname}" || exit -1 + else + oc exec -n ${NAMESPACE} -c galera ${galera} -- /bin/sh -c "mysql -u${username} -p${password} -Nse 'select 1'" || exit -1 + fi +fi + +exit $found diff --git a/tests/chainsaw/tests/create-system-account/chainsaw-test.yaml b/tests/chainsaw/tests/create-system-account/chainsaw-test.yaml new file mode 100644 index 00000000..fa46f1e8 --- /dev/null +++ b/tests/chainsaw/tests/create-system-account/chainsaw-test.yaml @@ -0,0 +1,83 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: create-system-account +spec: + steps: + - name: Deploy 1-node cluster + description: Deploy a 1-node cluster for tests + bindings: + - name: replicas + value: 1 + try: + - apply: + file: ../../common/galera.yaml + - assert: + file: ../../common/galera-assert.yaml + + - name: create system account without secret + description: system account CR has to wait for a secret to create account in the database + # we will delete the account manually + skipDelete: true + try: + - apply: + file: ../../common/system-account.yaml + - assert: + file: system-account-missing-secret-assert.yaml + + - name: add secret and finish system account creation + description: make sure the system account is created in the database + # we will delete the secret manually + skipDelete: true + try: + - apply: + file: ../../common/system-account-secret.yaml + - assert: + file: ../../common/system-account-assert.yaml + + - name: verify system account in database + description: check that system account exists in database with correct permissions + try: + - script: + content: | + ../../scripts/check_db_account.sh openstack-galera-0 '' systemuser dbsecret1 + check: + ($error): ~ + - script: + content: | + oc exec -n ${NAMESPACE} -c galera openstack-galera-0 -- /bin/sh -c 'source /var/lib/operator-scripts/mysql_root_auth.sh; mysql -uroot -p${DB_ROOT_PASSWORD} -e "show grants for \`systemuser\`@\`%\`;"' | grep 'GRANT ALL' | grep -v 'REQUIRE SSL' + check: + ($error): ~ + + - name: update secret in place + description: change the secret to a new one and verify password is updated in database + skipDelete: true + try: + - apply: + file: system-account-update-secret.yaml + - assert: + file: system-account-update-assert.yaml + - script: + content: | + ../../scripts/check_db_account.sh openstack-galera-0 '' systemuser dbsecret2 + check: + ($error): ~ + + - name: drop system account + description: delete the MariaDBAccount CR and verify account is removed from database + try: + - delete: + ref: + apiVersion: mariadb.openstack.org/v1beta1 + kind: MariaDBAccount + name: chainsawdb-some-system-db-account + - script: + content: | + ../../scripts/check_db_account.sh openstack-galera-0 "" systemuser dbsecret2 --reverse + check: + ($error): ~ + finally: + - delete: + file: ../../common/system-account-secret.yaml + - delete: + file: system-account-update-secret.yaml diff --git a/tests/chainsaw/tests/create-system-account/system-account-missing-secret-assert.yaml b/tests/chainsaw/tests/create-system-account/system-account-missing-secret-assert.yaml new file mode 100644 index 00000000..9f8506d4 --- /dev/null +++ b/tests/chainsaw/tests/create-system-account/system-account-missing-secret-assert.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBAccount +metadata: + labels: + dbName: openstack + name: chainsawdb-some-system-db-account +status: + conditions: + - reason: SecretMissing + severity: Warning + status: "False" + type: Ready + - reason: SecretMissing + severity: Warning + status: "False" + type: MariaDBAccountReady + - message: MariaDB / Galera server ready + reason: Ready + status: "True" + type: MariaDBServerReady diff --git a/tests/chainsaw/tests/create-system-account/system-account-update-assert.yaml b/tests/chainsaw/tests/create-system-account/system-account-update-assert.yaml new file mode 100644 index 00000000..9dc4652b --- /dev/null +++ b/tests/chainsaw/tests/create-system-account/system-account-update-assert.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBAccount +metadata: + labels: + dbName: openstack + name: chainsawdb-some-system-db-account +status: + currentSecret: some-new-system-db-secret + conditions: + - message: Setup complete + reason: Ready + status: "True" + type: Ready + - message: MariaDBAccount creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady + - message: MariaDB / Galera server ready + reason: Ready + status: "True" + type: MariaDBServerReady diff --git a/tests/chainsaw/tests/create-system-account/system-account-update-secret.yaml b/tests/chainsaw/tests/create-system-account/system-account-update-secret.yaml new file mode 100644 index 00000000..44308edb --- /dev/null +++ b/tests/chainsaw/tests/create-system-account/system-account-update-secret.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +data: + # dbsecret2 + DatabasePassword: ZGJzZWNyZXQy +kind: Secret +metadata: + name: some-new-system-db-secret +type: Opaque +--- +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBAccount +metadata: + labels: + dbName: openstack + name: chainsawdb-some-system-db-account +spec: + userName: systemuser + secret: some-new-system-db-secret + accountType: System diff --git a/tests/chainsaw/tests/system-account-preexisting/chainsaw-test.yaml b/tests/chainsaw/tests/system-account-preexisting/chainsaw-test.yaml new file mode 100644 index 00000000..e07474b3 --- /dev/null +++ b/tests/chainsaw/tests/system-account-preexisting/chainsaw-test.yaml @@ -0,0 +1,63 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: system-account-preexisting +spec: + steps: + - name: Deploy 1-node cluster + description: Deploy a 1-node cluster for tests + bindings: + - name: replicas + value: 1 + try: + - apply: + file: ../../common/galera.yaml + - assert: + file: ../../common/galera-assert.yaml + + - name: create user manually with old password + description: manually create the systemuser account with a different password than the MariaDBAccount will use + try: + - script: + content: | + oc exec -n ${NAMESPACE} -c galera openstack-galera-0 -- /bin/sh -c 'source /var/lib/operator-scripts/mysql_root_auth.sh; mysql -uroot -p${DB_ROOT_PASSWORD} -e "GRANT ALL PRIVILEGES ON *.* TO \`systemuser\`@\`localhost\` IDENTIFIED BY \"oldpassword123\"; GRANT ALL PRIVILEGES ON *.* TO \`systemuser\`@\`%\` IDENTIFIED BY \"oldpassword123\";"' + check: + ($error): ~ + + - name: verify old password works + description: verify that the manually created account works with the old password + try: + - script: + content: | + ../../scripts/check_db_account.sh openstack-galera-0 '' systemuser oldpassword123 + check: + ($error): ~ + + - name: apply secret with new password + description: apply the secret with the new password that will be used by MariaDBAccount + try: + - apply: + file: ../../common/system-account-secret.yaml + + - name: apply MariaDBAccount + description: apply the MariaDBAccount CR to update the existing account password + try: + - apply: + file: ../../common/system-account.yaml + - assert: + file: ../../common/system-account-assert.yaml + + - name: verify new password works and old password fails + description: verify that the account password was updated to the new password from the secret + try: + - script: + content: | + ../../scripts/check_db_account.sh openstack-galera-0 '' systemuser dbsecret1 + check: + ($error): ~ + - script: + content: | + # verify old password no longer works + ! oc exec -n ${NAMESPACE} -c galera openstack-galera-0 -- /bin/sh -c "mysql -usystemuser -poldpassword123 -Nse 'select 1'" + check: + ($error): ~ diff --git a/tests/chainsaw/tests/update-root-pw/chainsaw-test.yaml b/tests/chainsaw/tests/update-root-pw/chainsaw-test.yaml new file mode 100644 index 00000000..ec7bb03d --- /dev/null +++ b/tests/chainsaw/tests/update-root-pw/chainsaw-test.yaml @@ -0,0 +1,93 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: update-root-pw +spec: + steps: + - name: Deploy 3-node cluster + description: Deploy a 3-node galera cluster for root password update test + bindings: + - name: replicas + value: 3 + try: + - apply: + file: ../../common/galera.yaml + - assert: + file: ../../common/galera-assert.yaml + + - name: Verify root MariaDBAccount is created + description: Verify the root MariaDBAccount exists and has Status.CurrentSecret set + try: + - assert: + file: root-account-initial-assert.yaml + + - name: Verify initial setup + description: Verify the cluster is running with initial root password + try: + - script: + content: | + oc exec -n ${NAMESPACE} -c galera openstack-galera-0 -- /bin/sh -c ' + source /var/lib/operator-scripts/mysql_root_auth.sh + if [ "$MYSQL_PWD" = "newrootpassword123" ]; then + echo "ERROR: password == 'newrootpassword123', should not have changed yet" + exit 1 + fi + echo "PASS: password != 'newrootpassword123' (current: $MYSQL_PWD)" + mysql -e "SELECT 1" > /dev/null + echo "Initial root password works" + ' + + - name: Create new root password secret + description: Create a new secret with updated root password + skipDelete: true + try: + - apply: + file: new-root-secret.yaml + + - name: Update root MariaDBAccount with new secret + description: Apply the new secret to the root MariaDBAccount spec + skipDelete: true + try: + - apply: + file: root-account-updated.yaml + - assert: + file: root-account-assert.yaml + + - name: Verify Galera status updated + description: Wait for Galera status.rootDatabaseSecret to be updated + try: + - assert: + file: galera-status-assert.yaml + + - name: Verify login with new password on all nodes + description: Log into all galera nodes with new root password + try: + - script: + content: | + for pod in openstack-galera-0 openstack-galera-1 openstack-galera-2; do + echo "Testing login on $pod..." + oc exec -n ${NAMESPACE} -c galera $pod -- /bin/sh -c ' + # Clear the cached credentials to force using new password + rm -f $HOME/.my.cnf + source /var/lib/operator-scripts/mysql_root_auth.sh + if [ "$MYSQL_PWD" != "newrootpassword123" ]; then + echo "ERROR: password != 'newrootpassword123' (actual: $MYSQL_PWD)" + exit 1 + fi + echo "PASS: password == 'newrootpassword123'" + mysql -e "SELECT 1" > /dev/null + echo "New root password works on $(hostname)" + ' + done + echo "All nodes verified successfully" + + - name: Verify new password persists + description: Verify the new password still works after cache refresh + try: + - script: + content: | + oc exec -n ${NAMESPACE} -c galera openstack-galera-0 -- /bin/sh -c ' + source /var/lib/operator-scripts/mysql_root_auth.sh + mysql -e "SHOW DATABASES" > /dev/null + echo "New password persists and works correctly" + ' diff --git a/tests/chainsaw/tests/update-root-pw/galera-status-assert.yaml b/tests/chainsaw/tests/update-root-pw/galera-status-assert.yaml new file mode 100644 index 00000000..dc2036c0 --- /dev/null +++ b/tests/chainsaw/tests/update-root-pw/galera-status-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +status: + # Verify that rootDatabaseSecret has been updated to the new secret + rootDatabaseSecret: new-root-secret + bootstrapped: true diff --git a/tests/chainsaw/tests/update-root-pw/new-root-secret.yaml b/tests/chainsaw/tests/update-root-pw/new-root-secret.yaml new file mode 100644 index 00000000..be43bf05 --- /dev/null +++ b/tests/chainsaw/tests/update-root-pw/new-root-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: new-root-secret +type: Opaque +data: + # "newrootpassword123" base64 encoded + DatabasePassword: bmV3cm9vdHBhc3N3b3JkMTIz diff --git a/tests/chainsaw/tests/update-root-pw/root-account-assert.yaml b/tests/chainsaw/tests/update-root-pw/root-account-assert.yaml new file mode 100644 index 00000000..b9ed3d81 --- /dev/null +++ b/tests/chainsaw/tests/update-root-pw/root-account-assert.yaml @@ -0,0 +1,11 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBAccount +metadata: + name: openstack-mariadb-root +spec: + userName: root + secret: new-root-secret + accountType: System +status: + # Verify that the CurrentSecret field has been updated to the new secret + currentSecret: new-root-secret diff --git a/tests/chainsaw/tests/update-root-pw/root-account-initial-assert.yaml b/tests/chainsaw/tests/update-root-pw/root-account-initial-assert.yaml new file mode 100644 index 00000000..866f0419 --- /dev/null +++ b/tests/chainsaw/tests/update-root-pw/root-account-initial-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBAccount +metadata: + name: openstack-mariadb-root +spec: + userName: root + secret: openstack-mariadb-root-db-secret + accountType: System +status: + currentSecret: openstack-mariadb-root-db-secret diff --git a/tests/chainsaw/tests/update-root-pw/root-account-updated.yaml b/tests/chainsaw/tests/update-root-pw/root-account-updated.yaml new file mode 100644 index 00000000..dc3a89f7 --- /dev/null +++ b/tests/chainsaw/tests/update-root-pw/root-account-updated.yaml @@ -0,0 +1,8 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBAccount +metadata: + name: openstack-mariadb-root +spec: + userName: root + secret: new-root-secret + accountType: System diff --git a/tests/kuttl/common/assert_sample_deployment.yaml b/tests/kuttl/common/assert_sample_deployment.yaml index 251435c6..9d87349e 100644 --- a/tests/kuttl/common/assert_sample_deployment.yaml +++ b/tests/kuttl/common/assert_sample_deployment.yaml @@ -32,6 +32,10 @@ status: reason: Ready status: "True" type: InputReady + - message: MariaDBAccount System account 'root' creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady - message: RoleBinding created reason: Ready status: "True" diff --git a/tests/kuttl/tests/account_create/01-assert.yaml b/tests/kuttl/tests/account_create/01-assert.yaml index 98df972c..547b04cf 100644 --- a/tests/kuttl/tests/account_create/01-assert.yaml +++ b/tests/kuttl/tests/account_create/01-assert.yaml @@ -34,6 +34,10 @@ status: reason: Ready status: "True" type: InputReady + - message: MariaDBAccount System account 'root' creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady - message: RoleBinding created reason: Ready status: "True" diff --git a/tests/kuttl/tests/account_create/05-assert.yaml b/tests/kuttl/tests/account_create/05-assert.yaml index a3abac8c..ddccd6eb 100644 --- a/tests/kuttl/tests/account_create/05-assert.yaml +++ b/tests/kuttl/tests/account_create/05-assert.yaml @@ -11,7 +11,7 @@ commands: apiVersion: batch/v1 kind: Job metadata: - name: someuser-account-create + name: someuser-account-create-update spec: template: spec: diff --git a/tests/kuttl/tests/database_create/01-assert.yaml b/tests/kuttl/tests/database_create/01-assert.yaml index 74a26acb..c79ade1a 100644 --- a/tests/kuttl/tests/database_create/01-assert.yaml +++ b/tests/kuttl/tests/database_create/01-assert.yaml @@ -32,6 +32,10 @@ status: reason: Ready status: "True" type: InputReady + - message: MariaDBAccount System account 'root' creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady - message: RoleBinding created reason: Ready status: "True" diff --git a/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml b/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml index 08b5b237..ae2af301 100644 --- a/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml +++ b/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml @@ -27,6 +27,10 @@ status: reason: Ready status: "True" type: InputReady + - message: MariaDBAccount System account 'root' creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady - message: RoleBinding created reason: Ready status: "True" diff --git a/tests/kuttl/tests/galera_log_to_disk/01-assert.yaml b/tests/kuttl/tests/galera_log_to_disk/01-assert.yaml index 74a26acb..c79ade1a 100644 --- a/tests/kuttl/tests/galera_log_to_disk/01-assert.yaml +++ b/tests/kuttl/tests/galera_log_to_disk/01-assert.yaml @@ -32,6 +32,10 @@ status: reason: Ready status: "True" type: InputReady + - message: MariaDBAccount System account 'root' creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady - message: RoleBinding created reason: Ready status: "True"