From 866d336b6ff153f80a3c076b9ac1684e55b0543f Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 29 Apr 2025 14:55:37 -0400 Subject: [PATCH 1/2] support in-place secret changes The existing mariadb operator in fact already "supports" in-place change of secret, since if you change Spec.Secret to a new secret name, that would imply a new job hash, and the account.sh script running using only GRANT statements would update the password per mariadb. This already works for flipping the TLS flag on and off too. So in this patch, we clean this up and add a test to include: * a new field Status.CurrentSecret, which is used to indicate the previous secret from which the finalizer should be removed. This will also be used when we migrate the root password to use MariaDBAccount by providing the "current" root password when changing to a new root password * improved messaging in log messages, name of job. This changes the job hash for mariadbaccount which will incur a run on existing environments, however the job hashes are already changing on existing environments due to the change in how the mysql root password is sent, i.e. via volume mounted script rather than env var secret * update account.sh to use modern idiomatic patterns for user create /alter, while mariadb is fine with the legacy style of using only GRANT statements, MySQL 8 no longer allows this statement to proceed without a CREATE USER, so formalize the commands used here to use distinct CREATE USER, ALTER USER, GRANT statements and clarify the script is good for all create/update user operations. --- ...mariadb.openstack.org_mariadbaccounts.yaml | 8 +++ api/v1beta1/mariadbaccount_types.go | 7 +++ ...mariadb.openstack.org_mariadbaccounts.yaml | 8 +++ controllers/mariadbaccount_controller.go | 58 +++++++++++++++---- pkg/mariadb/account.go | 8 +-- templates/account.sh | 26 ++++++++- .../common/system-account-assert.yaml | 3 +- .../common/system-account-secret.yaml | 1 + .../create-system-account/chainsaw-test.yaml | 18 +++++- .../system-account-update-assert.yaml | 22 +++++++ .../system-account-update-secret.yaml | 19 ++++++ .../kuttl/tests/account_create/05-assert.yaml | 2 +- 12 files changed, 158 insertions(+), 22 deletions(-) create mode 100644 tests/chainsaw/tests/create-system-account/system-account-update-assert.yaml create mode 100644 tests/chainsaw/tests/create-system-account/system-account-update-secret.yaml diff --git a/api/bases/mariadb.openstack.org_mariadbaccounts.yaml b/api/bases/mariadb.openstack.org_mariadbaccounts.yaml index 36599ddb..51dd7196 100644 --- a/api/bases/mariadb.openstack.org_mariadbaccounts.yaml +++ b/api/bases/mariadb.openstack.org_mariadbaccounts.yaml @@ -114,6 +114,14 @@ spec: - type type: object type: array + currentSecret: + description: |- + the Secret that's currently in use for the account. + keeping a handle to this secret allows us to remove its finalizer + when it's replaced with a new one. It also is useful for storing + the current "root" secret separate from a newly proposed one which is + needed when changing the database root password. + type: string hash: additionalProperties: type: string diff --git a/api/v1beta1/mariadbaccount_types.go b/api/v1beta1/mariadbaccount_types.go index ee3a07f4..dd5885e2 100644 --- a/api/v1beta1/mariadbaccount_types.go +++ b/api/v1beta1/mariadbaccount_types.go @@ -63,6 +63,13 @@ const ( // MariaDBAccountStatus defines the observed state of MariaDBAccount type MariaDBAccountStatus struct { + // the Secret that's currently in use for the account. + // keeping a handle to this secret allows us to remove its finalizer + // when it's replaced with a new one. It also is useful for storing + // the current "root" secret separate from a newly proposed one which is + // needed when changing the database root password. + CurrentSecret string `json:"currentSecret,omitempty"` + // Deployment Conditions Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` diff --git a/config/crd/bases/mariadb.openstack.org_mariadbaccounts.yaml b/config/crd/bases/mariadb.openstack.org_mariadbaccounts.yaml index 36599ddb..51dd7196 100644 --- a/config/crd/bases/mariadb.openstack.org_mariadbaccounts.yaml +++ b/config/crd/bases/mariadb.openstack.org_mariadbaccounts.yaml @@ -114,6 +114,14 @@ spec: - type type: object type: array + currentSecret: + description: |- + the Secret that's currently in use for the account. + keeping a handle to this secret allows us to remove its finalizer + when it's replaced with a new one. It also is useful for storing + the current "root" secret separate from a newly proposed one which is + needed when changing the database root password. + type: string hash: additionalProperties: type: string diff --git a/controllers/mariadbaccount_controller.go b/controllers/mariadbaccount_controller.go index c5d70fd7..ab2d330f 100644 --- a/controllers/mariadbaccount_controller.go +++ b/controllers/mariadbaccount_controller.go @@ -129,15 +129,15 @@ func (r *MariaDBAccountReconciler) Reconcile(ctx context.Context, req ctrl.Reque instance.Status.Conditions.Init(&cl) if instance.DeletionTimestamp.IsZero() || isNewInstance { //revive:disable:indent-error-flow - return r.reconcileCreate(ctx, log, helper, instance) + return r.reconcileCreateOrUpdate(ctx, log, helper, instance) } else { return r.reconcileDelete(ctx, log, helper, instance) } } -// reconcileDelete - run reconcile for case where delete timestamp is zero -func (r *MariaDBAccountReconciler) reconcileCreate( +// reconcileCreateOrUpdate - run reconcile for case where delete timestamp is zero +func (r *MariaDBAccountReconciler) reconcileCreateOrUpdate( ctx context.Context, log logr.Logger, helper *helper.Helper, instance *databasev1beta1.MariaDBAccount) (result ctrl.Result, _err error) { @@ -190,18 +190,27 @@ func (r *MariaDBAccountReconciler) reconcileCreate( var jobDef *batchv1.Job + var createOrUpdate string + if instance.Status.CurrentSecret != "" { + createOrUpdate = "update" + } else { + createOrUpdate = "create" + } + if instance.IsUserAccount() { - log.Info(fmt.Sprintf("Running account create '%s' MariaDBDatabase '%s'", + log.Info(fmt.Sprintf("Checking %s account job '%s' MariaDBDatabase '%s'", + createOrUpdate, instance.Name, mariadbDatabase.Spec.Name)) - jobDef, err = mariadb.CreateDbAccountJob( + jobDef, err = mariadb.CreateOrUpdateDbAccountJob( dbGalera, instance, mariadbDatabase.Spec.Name, dbHostname, dbContainerImage, serviceAccountName, dbGalera.Spec.NodeSelector) if err != nil { return ctrl.Result{}, err } } else { - log.Info(fmt.Sprintf("Running system account create '%s'", instance.Name)) - jobDef, err = mariadb.CreateDbAccountJob( + log.Info(fmt.Sprintf("Checking %s system account job '%s'", createOrUpdate, + instance.Name)) + jobDef, err = mariadb.CreateOrUpdateDbAccountJob( dbGalera, instance, "", dbHostname, dbContainerImage, serviceAccountName, dbGalera.Spec.NodeSelector) if err != nil { @@ -226,11 +235,24 @@ func (r *MariaDBAccountReconciler) reconcileCreate( } if accountCreateJob.HasChanged() { + if instance.Status.Hash == nil { instance.Status.Hash = make(map[string]string) } instance.Status.Hash[databasev1beta1.AccountCreateHash] = accountCreateJob.GetHash() - log.Info(fmt.Sprintf("Job %s hash added - %s", jobDef.Name, instance.Status.Hash[databasev1beta1.AccountCreateHash])) + log.Info(fmt.Sprintf("Job %s hash created or updated - %s", jobDef.Name, instance.Status.Hash[databasev1beta1.AccountCreateHash])) + + // set up new Secret and remove finalizer from old secret + if instance.Status.CurrentSecret != instance.Spec.Secret { + currentSecret := instance.Status.CurrentSecret + err = r.removeSecretFinalizer(ctx, helper, currentSecret, instance.Namespace) + if err == nil { + instance.Status.CurrentSecret = instance.Spec.Secret + } else { + return ctrl.Result{}, err + } + } + } // database creation finished @@ -711,7 +733,22 @@ func (r *MariaDBAccountReconciler) ensureAccountSecret( func (r *MariaDBAccountReconciler) removeAccountAndSecretFinalizer(ctx context.Context, helper *helper.Helper, instance *databasev1beta1.MariaDBAccount) error { - accountSecret, _, err := secret.GetSecret(ctx, helper, instance.Spec.Secret, instance.Namespace) + err := r.removeSecretFinalizer( + ctx, helper, instance.Spec.Secret, instance.Namespace, + ) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + + // remove mariadbaccount finalizer which will update at end of reconcile + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + + return nil +} + +func (r *MariaDBAccountReconciler) removeSecretFinalizer(ctx context.Context, + helper *helper.Helper, secretName string, namespace string) error { + accountSecret, _, err := secret.GetSecret(ctx, helper, secretName, namespace) if err == nil { if controllerutil.RemoveFinalizer(accountSecret, helper.GetFinalizer()) { @@ -724,9 +761,6 @@ func (r *MariaDBAccountReconciler) removeAccountAndSecretFinalizer(ctx context.C return err } - // will take effect when reconcile ends - controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) - return nil } diff --git a/pkg/mariadb/account.go b/pkg/mariadb/account.go index 0dbf897f..15e9e87a 100644 --- a/pkg/mariadb/account.go +++ b/pkg/mariadb/account.go @@ -19,8 +19,8 @@ type accountCreateOrDeleteOptions struct { RequireTLS string } -// CreateDbAccountJob creates a Kubernetes job for creating a MariaDB database account -func CreateDbAccountJob(galera *mariadbv1.Galera, account *mariadbv1.MariaDBAccount, databaseName string, databaseHostName string, containerImage string, serviceAccountName string, nodeSelector *map[string]string) (*batchv1.Job, error) { +// CreateOrUpdateDbAccountJob creates or updates a Kubernetes job for creating a MariaDB database account +func CreateOrUpdateDbAccountJob(galera *mariadbv1.Galera, account *mariadbv1.MariaDBAccount, databaseName string, databaseHostName string, containerImage string, serviceAccountName string, nodeSelector *map[string]string) (*batchv1.Job, error) { var tlsStatement string if account.Spec.RequireTLS { tlsStatement = " REQUIRE SSL" @@ -47,7 +47,7 @@ func CreateDbAccountJob(galera *mariadbv1.Galera, account *mariadbv1.MariaDBAcco // provided db name is used as metadata name where underscore is a not allowed // character. Lets replace all underscores with hypen. Underscores in the db name are // possible. - Name: strings.ReplaceAll(account.Spec.UserName, "_", "-") + "-account-create", + Name: strings.ReplaceAll(account.Spec.UserName, "_", "-") + "-account-create-update", Namespace: account.Namespace, Labels: labels, }, @@ -58,7 +58,7 @@ func CreateDbAccountJob(galera *mariadbv1.Galera, account *mariadbv1.MariaDBAcco ServiceAccountName: serviceAccountName, Containers: []corev1.Container{ { - Name: "mariadb-account-create", + Name: "mariadb-account-create-update", Image: containerImage, Command: []string{"/bin/sh", "-c", dbCmd}, Env: []corev1.EnvVar{ diff --git a/templates/account.sh b/templates/account.sh index dccd5aca..891ede76 100755 --- a/templates/account.sh +++ b/templates/account.sh @@ -4,15 +4,35 @@ MYSQL_REMOTE_HOST={{.DatabaseHostname}} source /var/lib/operator-scripts/mysql_r export DatabasePassword=${DatabasePassword:?"Please specify a DatabasePassword variable."} +MYSQL_CMD="mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -P 3306" + if [ -n "{{.DatabaseName}}" ]; then - mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -P 3306 -e "GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.UserName}}'@'localhost' IDENTIFIED BY '$DatabasePassword'{{.RequireTLS}};GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.UserName}}'@'%' IDENTIFIED BY '$DatabasePassword'{{.RequireTLS}};" + GRANT_DATABASE="{{.DatabaseName}}" else - mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -P 3306 -e "GRANT ALL PRIVILEGES ON *.* TO '{{.UserName}}'@'localhost' IDENTIFIED BY '$DatabasePassword'{{.RequireTLS}};GRANT ALL PRIVILEGES ON *.* TO '{{.UserName}}'@'%' IDENTIFIED BY '$DatabasePassword'{{.RequireTLS}};" + GRANT_DATABASE="*" fi +# going for maximum compatibility here: +# 1. MySQL 8 no longer allows implicit create user when GRANT is used +# 2. MariaDB has "CREATE OR REPLACE", but MySQL does not +# 3. create user with CREATE but then do all password and TLS with ALTER to +# support updates + +$MYSQL_CMD < Date: Tue, 27 May 2025 13:38:35 -0400 Subject: [PATCH 2/2] use system MariaDBAccount for the galera server's root pw This commit ties together the previous ones to create a new MariaDBAccount when a Galera instance is created, and then to use the password from that account/secret in the mariadb bootstrap/maintenance scripts. Galera gets bootstrapped with this secret, then the mariadbaccount controller, who is waiting for galera to be available to set up this new "root" account, wakes up when galera is running, and changes the root password to itself, establishing the initial job hash for the mariadbaccount. As we now have a mariadbaccount linked to the outermost lifecycle of a galera instance, some hardening of the deletion process has been added to clarify that mariadbaccount will run deletion jobs only if Galera is not marked for deletion. If galera is marked for deletion, then we have to assume the service/pods are gone and no more drops can take place, even if the Galera CR is still present (chainsaw conveniently adds its own finalizer to Galera when running, preventing it from being fully deleted, which exposed this issue). Additional changes to the mysql_root_auth.sh and account.sh scripts allow for in-place changes of root password. --- api/bases/mariadb.openstack.org_galeras.yaml | 16 ++- api/v1beta1/conditions.go | 7 + api/v1beta1/galera_types.go | 17 ++- api/v1beta1/mariadbaccount_types.go | 3 - api/v1beta1/mariadbdatabase_funcs.go | 129 +++++++++++++---- .../bases/mariadb.openstack.org_galeras.yaml | 16 ++- controllers/galera_controller.go | 132 +++++++++++++++++- controllers/mariadbaccount_controller.go | 72 ++++++++-- templates/account.sh | 6 +- templates/database.sh | 2 +- templates/delete_account.sh | 2 +- templates/delete_database.sh | 2 +- templates/galera/bin/mysql_bootstrap.sh | 47 ++++++- templates/galera/bin/mysql_root_auth.sh | 58 +++++++- tests/chainsaw/common/galera-assert.yaml | 4 + .../tests/update-root-pw/chainsaw-test.yaml | 93 ++++++++++++ .../update-root-pw/galera-status-assert.yaml | 8 ++ .../tests/update-root-pw/new-root-secret.yaml | 8 ++ .../update-root-pw/root-account-assert.yaml | 11 ++ .../root-account-initial-assert.yaml | 10 ++ .../update-root-pw/root-account-updated.yaml | 8 ++ .../common/assert_sample_deployment.yaml | 4 + .../kuttl/tests/account_create/01-assert.yaml | 4 + .../tests/database_create/01-assert.yaml | 4 + .../galera_deploy_external_tls/01-assert.yaml | 4 + .../tests/galera_log_to_disk/01-assert.yaml | 4 + 26 files changed, 600 insertions(+), 71 deletions(-) create mode 100644 tests/chainsaw/tests/update-root-pw/chainsaw-test.yaml create mode 100644 tests/chainsaw/tests/update-root-pw/galera-status-assert.yaml create mode 100644 tests/chainsaw/tests/update-root-pw/new-root-secret.yaml create mode 100644 tests/chainsaw/tests/update-root-pw/root-account-assert.yaml create mode 100644 tests/chainsaw/tests/update-root-pw/root-account-initial-assert.yaml create mode 100644 tests/chainsaw/tests/update-root-pw/root-account-updated.yaml diff --git a/api/bases/mariadb.openstack.org_galeras.yaml b/api/bases/mariadb.openstack.org_galeras.yaml index f1c4e576..ca91f689 100644 --- a/api/bases/mariadb.openstack.org_galeras.yaml +++ b/api/bases/mariadb.openstack.org_galeras.yaml @@ -137,8 +137,18 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + rootDatabaseAccount: + description: |- + RootDatabaseAccount - name of MariaDBAccount which will be used to + generate root account / password. + this account is generated if not exists, and a name is chosen based + on a naming convention if not present + type: string secret: - description: Name of the secret to look for password keys + description: |- + Name of the legacy secret to locate the initial galera root + password + this field will be removed once scripts can adjust to using root_auth.sh type: string storageClass: description: Storage class to host the mariadb databases @@ -177,7 +187,6 @@ spec: required: - containerImage - replicas - - secret - storageClass - storageRequest type: object @@ -296,6 +305,9 @@ spec: the opentack-operator in the top-level CR (e.g. the ContainerImage) format: int64 type: integer + rootDatabaseSecret: + description: name of the Secret that is being used for the root password + type: string safeToBootstrap: description: Name of the node that can safely bootstrap a cluster type: string diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index 7c816838..5c60a306 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -54,6 +54,9 @@ const ( // ReasonDBServiceNameError - error getting the DB service hostname ReasonDBServiceNameError condition.Reason = "DatabaseServiceNameError" + // ReasonDBResourceDeleted - the galera resource has been marked for deletion + ReasonDBResourceDeleted condition.Reason = "DatabaseResourceDeleted" + // ReasonDBSync - Database sync in progress ReasonDBSync condition.Reason = "DBSync" ) @@ -92,8 +95,12 @@ const ( MariaDBServerNotBootstrappedMessage = "MariaDB / Galera server not bootstrapped" + MariaDBServerDeletedMessage = "MariaDB / Galera server has been marked for deletion" + MariaDBAccountReadyInitMessage = "MariaDBAccount create / drop not started" + MariaDBSystemAccountReadyMessage = "MariaDBAccount System account '%s' creation complete" + MariaDBAccountReadyMessage = "MariaDBAccount creation complete" MariaDBAccountNotReadyMessage = "MariaDBAccount is not present: %s" diff --git a/api/v1beta1/galera_types.go b/api/v1beta1/galera_types.go index 7767d3f3..f813fbdd 100644 --- a/api/v1beta1/galera_types.go +++ b/api/v1beta1/galera_types.go @@ -51,9 +51,19 @@ type GaleraSpec struct { // GaleraSpec defines the desired state of Galera type GaleraSpecCore struct { - // Name of the secret to look for password keys - // +kubebuilder:validation:Required + // Name of the legacy secret to locate the initial galera root + // password + // this field will be removed once scripts can adjust to using root_auth.sh + // +kubebuilder:validation:Optional Secret string `json:"secret"` + + // RootDatabaseAccount - name of MariaDBAccount which will be used to + // generate root account / password. + // this account is generated if not exists, and a name is chosen based + // on a naming convention if not present + // +kubebuilder:validation:Optional + RootDatabaseAccount string `json:"rootDatabaseAccount"` + // Storage class to host the mariadb databases // +kubebuilder:validation:Required StorageClass string `json:"storageClass"` @@ -113,6 +123,9 @@ type GaleraAttributes struct { type GaleraStatus struct { // A map of database node attributes for each pod Attributes map[string]GaleraAttributes `json:"attributes,omitempty"` + // name of the Secret that is being used for the root password + // +kubebuilder:validation:Optional + RootDatabaseSecret string `json:"rootDatabaseSecret"` // Name of the node that can safely bootstrap a cluster SafeToBootstrap string `json:"safeToBootstrap,omitempty"` // Is the galera cluster currently running diff --git a/api/v1beta1/mariadbaccount_types.go b/api/v1beta1/mariadbaccount_types.go index dd5885e2..6b0b13f4 100644 --- a/api/v1beta1/mariadbaccount_types.go +++ b/api/v1beta1/mariadbaccount_types.go @@ -28,9 +28,6 @@ const ( // AccountDeleteHash hash AccountDeleteHash = "accountdelete" - // DbRootPassword selector for galera root account - DbRootPasswordSelector = "DbRootPassword" - // DatabasePassword selector for MariaDBAccount->Secret DatabasePasswordSelector = "DatabasePassword" ) diff --git a/api/v1beta1/mariadbdatabase_funcs.go b/api/v1beta1/mariadbdatabase_funcs.go index 33d84f9b..91211999 100644 --- a/api/v1beta1/mariadbdatabase_funcs.go +++ b/api/v1beta1/mariadbdatabase_funcs.go @@ -541,6 +541,50 @@ func DeleteDatabaseAndAccountFinalizers( namespace string, ) error { + err := DeleteAccountFinalizers( + ctx, + h, + accountName, + namespace, + ) + if err != nil { + return err + } + + // also do a delete for "unused" MariaDBAccounts, associated with + // this MariaDBDatabase. + err = DeleteUnusedMariaDBAccountFinalizers( + ctx, h, name, accountName, namespace, + ) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + + mariaDBDatabase, err := GetDatabase(ctx, h, name, namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } else if err == nil && controllerutil.RemoveFinalizer(mariaDBDatabase, h.GetFinalizer()) { + err := h.GetClient().Update(ctx, mariaDBDatabase) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + util.LogForObject(h, fmt.Sprintf("Removed finalizer %s from MariaDBDatabase %s", h.GetFinalizer(), mariaDBDatabase.Spec.Name), mariaDBDatabase) + } + + return nil +} + +// DeleteAccountFinalizers performs just the primary account + secret finalizer +// removal part of DeleteDatabaseAndAccountFinalizers +func DeleteAccountFinalizers( + ctx context.Context, + h *helper.Helper, + accountName string, + namespace string, +) error { + if accountName == "" { + return fmt.Errorf("Account name is blank") + } databaseAccount, err := GetAccount(ctx, h, accountName, namespace) if err != nil && !k8s_errors.IsNotFound(err) { return err @@ -572,26 +616,6 @@ func DeleteDatabaseAndAccountFinalizers( } } - // also do a delete for "unused" MariaDBAccounts, associated with - // this MariaDBDatabase. - err = DeleteUnusedMariaDBAccountFinalizers( - ctx, h, name, accountName, namespace, - ) - if err != nil && !k8s_errors.IsNotFound(err) { - return err - } - - mariaDBDatabase, err := GetDatabase(ctx, h, name, namespace) - if err != nil && !k8s_errors.IsNotFound(err) { - return err - } else if err == nil && controllerutil.RemoveFinalizer(mariaDBDatabase, h.GetFinalizer()) { - err := h.GetClient().Update(ctx, mariaDBDatabase) - if err != nil && !k8s_errors.IsNotFound(err) { - return err - } - util.LogForObject(h, fmt.Sprintf("Removed finalizer %s from MariaDBDatabase %s", h.GetFinalizer(), mariaDBDatabase.Spec.Name), mariaDBDatabase) - } - return nil } @@ -806,6 +830,32 @@ func EnsureMariaDBAccount(ctx context.Context, userNamePrefix string, ) (*MariaDBAccount, *corev1.Secret, error) { + return ensureMariaDBAccount( + ctx, helper, accountName, namespace, requireTLS, + userNamePrefix, "", "", map[string]string{}) + +} + +// EnsureMariaDBSystemAccount ensures a MariaDBAccount has been created for a given +// operator calling the function, and returns the MariaDBAccount and its +// Secret for use in consumption into a configuration. +// Unlike EnsureMariaDBAccount, the function accepts an exact username that +// expected to remain constant, supporting in-place password changes for the +// account. +func EnsureMariaDBSystemAccount(ctx context.Context, + helper *helper.Helper, + accountName string, galeraInstanceName string, namespace string, requireTLS bool, + exactUserName string, exactPassword string) (*MariaDBAccount, *corev1.Secret, error) { + return ensureMariaDBAccount( + ctx, helper, accountName, namespace, requireTLS, + "", exactUserName, exactPassword, map[string]string{"dbName": galeraInstanceName}) +} + +func ensureMariaDBAccount(ctx context.Context, + helper *helper.Helper, + accountName string, namespace string, requireTLS bool, + userNamePrefix string, exactUserName string, exactPassword string, labels map[string]string, +) (*MariaDBAccount, *corev1.Secret, error) { if accountName == "" { return nil, nil, fmt.Errorf("accountName is empty") } @@ -817,9 +867,20 @@ func EnsureMariaDBAccount(ctx context.Context, return nil, nil, err } - username, err := generateUniqueUsername(userNamePrefix) - if err != nil { - return nil, nil, err + var username string + var accountType AccountType + + if exactUserName == "" { + accountType = "User" + username, err = generateUniqueUsername(userNamePrefix) + if err != nil { + return nil, nil, err + } + } else if userNamePrefix != "" { + return nil, nil, fmt.Errorf("userNamePrefix and exactUserName are mutually exclusive") + } else { + accountType = "System" + username = exactUserName } account = &MariaDBAccount{ @@ -832,9 +893,10 @@ func EnsureMariaDBAccount(ctx context.Context, // MariaDBAccount once this is filled in }, Spec: MariaDBAccountSpec{ - UserName: username, - Secret: fmt.Sprintf("%s-db-secret", accountName), - RequireTLS: requireTLS, + UserName: username, + Secret: fmt.Sprintf("%s-db-secret", accountName), + RequireTLS: requireTLS, + AccountType: accountType, }, } @@ -844,6 +906,7 @@ func EnsureMariaDBAccount(ctx context.Context, if account.Spec.Secret == "" { account.Spec.Secret = fmt.Sprintf("%s-db-secret", accountName) } + } dbSecret, _, err := secret.GetSecret(ctx, helper, account.Spec.Secret, namespace) @@ -853,9 +916,14 @@ func EnsureMariaDBAccount(ctx context.Context, return nil, nil, err } - dbPassword, err := generateDBPassword() - if err != nil { - return nil, nil, err + var dbPassword string + if exactPassword == "" { + dbPassword, err = generateDBPassword() + if err != nil { + return nil, nil, err + } + } else { + dbPassword = exactPassword } dbSecret = &corev1.Secret{ @@ -869,7 +937,7 @@ func EnsureMariaDBAccount(ctx context.Context, } } - _, err = createOrPatchAccountAndSecret(ctx, helper, account, dbSecret, map[string]string{}) + _, err = createOrPatchAccountAndSecret(ctx, helper, account, dbSecret, labels) if err != nil { return nil, nil, err } @@ -885,6 +953,7 @@ func EnsureMariaDBAccount(ctx context.Context, ) return account, dbSecret, nil + } // generateUniqueUsername creates a MySQL-compliant database username based on diff --git a/config/crd/bases/mariadb.openstack.org_galeras.yaml b/config/crd/bases/mariadb.openstack.org_galeras.yaml index f1c4e576..ca91f689 100644 --- a/config/crd/bases/mariadb.openstack.org_galeras.yaml +++ b/config/crd/bases/mariadb.openstack.org_galeras.yaml @@ -137,8 +137,18 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + rootDatabaseAccount: + description: |- + RootDatabaseAccount - name of MariaDBAccount which will be used to + generate root account / password. + this account is generated if not exists, and a name is chosen based + on a naming convention if not present + type: string secret: - description: Name of the secret to look for password keys + description: |- + Name of the legacy secret to locate the initial galera root + password + this field will be removed once scripts can adjust to using root_auth.sh type: string storageClass: description: Storage class to host the mariadb databases @@ -177,7 +187,6 @@ spec: required: - containerImage - replicas - - secret - storageClass - storageRequest type: object @@ -296,6 +305,9 @@ spec: the opentack-operator in the top-level CR (e.g. the ContainerImage) format: int64 type: integer + rootDatabaseSecret: + description: name of the Secret that is being used for the root password + type: string safeToBootstrap: description: Name of the node that can safely bootstrap a cluster type: string diff --git a/controllers/galera_controller.go b/controllers/galera_controller.go index f1b478cf..3ea62321 100644 --- a/controllers/galera_controller.go +++ b/controllers/galera_controller.go @@ -35,7 +35,7 @@ import ( helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/common/labels" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" - secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/service" commonstatefulset "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" @@ -500,6 +500,11 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res Resources: []string{"galeras"}, Verbs: []string{"get", "list"}, }, + { + APIGroups: []string{"mariadb.openstack.org"}, + Resources: []string{"mariadbaccounts"}, + Verbs: []string{"get", "list"}, + }, { APIGroups: []string{""}, Resources: []string{"secrets"}, @@ -567,6 +572,8 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res log.Info("", "Kind", instance.Kind, "Name", instance.Name, "database service", service.Name, "operation", string(op)) } + instance.IsReady() + instance.Status.Conditions.MarkTrue(condition.CreateServiceReadyCondition, condition.CreateServiceReadyMessage) // Map of all resources that may cause a rolling service restart @@ -576,8 +583,10 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res clusterPropertiesEnv := make(map[string]env.Setter) // Check and hash inputs - // NOTE do not hash the db root password, as its change requires - // more orchestration than a simple rolling restart + + // ******** TEMPORARY ************ + // Pull the DbRootPassword from osp-secret to allow CI / external + // scripts to work, until they can adapt to root_auth.sh _, res, err := secret.VerifySecret( ctx, types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret}, @@ -597,15 +606,67 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res condition.InputReadyWaitingMessage)) return res, fmt.Errorf("%w: %s", ErrOpenStackSecretNotFound, instance.Spec.Secret) } + return ctrl.Result{}, err + } + + legacySecret, _, err := secret.GetSecret(ctx, helper, instance.Spec.Secret, instance.Namespace) + if err != nil { + return ctrl.Result{}, err + } + legacyRootPassword := string(legacySecret.Data["DbRootPassword"]) + // ******** END TEMPORARY ************ + + databaseAccountName := r.getRootMariadbAccountName(instance) + + mariaDBAccount, rootSecret, err := mariadbv1.EnsureMariaDBSystemAccount( + ctx, helper, databaseAccountName, + instance.Name, instance.Namespace, false, "root", legacyRootPassword) + + if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, + mariadbv1.MariaDBAccountReadyCondition, condition.ErrorReason, condition.SeverityWarning, - condition.InputReadyErrorMessage, + mariadbv1.MariaDBAccountNotReadyMessage, err.Error())) + return ctrl.Result{}, err } - instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + + // the current root secret for the MariaDBAccount name is copied out to Status. + // this allows us to avoid having to change the Spec, which seems to be not + // best practice here and also seems to interfere with the openstack + // controller's ability to see the Galera status as Completed (though this + // might only be due to CRDs not being in sync between mariadb-operator and + // openstack-operator during development) + // Also by trying to always use CurrentSecret if available, we try to keep + // instance.Status.RootDatabaseSecret as the secret that should work for + // root right now, independent of changes in the mariadbaccount name + // or its secret that have not been reconciled w/ a new job hash. + if mariaDBAccount.Status.CurrentSecret != "" { + instance.Status.RootDatabaseSecret = mariaDBAccount.Status.CurrentSecret + } else if instance.Status.RootDatabaseSecret == "" { + // if we dont have any Status.RootDatabaseSecret at all, then use the + // new secret we just got. assume this is prior to bootstrap + instance.Status.RootDatabaseSecret = rootSecret.Name + } + // otherwise if mariaDBAccount.Status.CurrentSecret == "" and + // instance.Status.RootDatabaseSecret != "", this means we've already + // assigned rootSecret.Name to instance.Status.RootDatabaseSecret up front. + // if rootSecret.name is now different from what it was originally, that + // means the mariadbaccount was updated, and we assume is in the process + // of updating the root PW. if mariaDBAccount.Status.CurrentSecret is blank + // while this is happening, this pretty much means a test suite is changing + // the root pw on a brand new galera that hasn't had a chance to set up + // CurrentSecret in the first place, so just wait for that. + + instance.Status.Conditions.MarkTrue( + mariadbv1.MariaDBAccountReadyCondition, + mariadbv1.MariaDBSystemAccountReadyMessage, "root") + + instance.Status.Conditions.MarkTrue( + condition.InputReadyCondition, + condition.InputReadyMessage) // // TLS input validation @@ -1055,6 +1116,11 @@ func (r *GaleraReconciler) SetupWithManager(mgr ctrl.Manager) error { Watches(&topologyv1.Topology{}, handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Watches( + &mariadbv1.MariaDBAccount{}, + handler.EnqueueRequestsFromMapFunc(r.findGaleraForMariaDBAccount), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } @@ -1113,6 +1179,53 @@ func (r *GaleraReconciler) findObjectsForSrc(ctx context.Context, src client.Obj return requests } +// findGaleraForMariaDBAccount - returns reconcile requests for Galera instances +// that should reconcile when a MariaDBAccount changes +func (r *GaleraReconciler) findGaleraForMariaDBAccount(ctx context.Context, account client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + mariadbAccount := account.(*mariadbv1.MariaDBAccount) + + // Only reconcile for system accounts, as these are used by Galera for root credentials + if !mariadbAccount.IsSystemAccount() { + return requests + } + + // List all Galera instances in the same namespace + galeraList := &mariadbv1.GaleraList{} + if err := r.List(ctx, galeraList, client.InNamespace(mariadbAccount.Namespace)); err != nil { + log.FromContext(ctx).Error(err, "Unable to list Galera instances for MariaDBAccount", "account", mariadbAccount.Name) + return requests + } + + // Find Galera instances that use this MariaDBAccount + for _, galera := range galeraList.Items { + // Check if this Galera uses this MariaDBAccount for root credentials + rootAccountName := r.getRootMariadbAccountName(&galera) + if rootAccountName == mariadbAccount.Name { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: galera.Name, + Namespace: galera.Namespace, + }, + }) + log.FromContext(ctx).Info("MariaDBAccount changed, triggering Galera reconcile", + "account", mariadbAccount.Name, + "galera", galera.Name) + } + } + + return requests +} + +func (r *GaleraReconciler) getRootMariadbAccountName(instance *mariadbv1.Galera) string { + databaseAccountName := instance.Spec.RootDatabaseAccount + if databaseAccountName == "" { + databaseAccountName = instance.Name + "-mariadb-root" + } + return databaseAccountName +} + func (r *GaleraReconciler) reconcileDelete(ctx context.Context, instance *mariadbv1.Galera, helper *helper.Helper) (ctrl.Result, error) { helper.GetLogger().Info("Reconciling Service delete") @@ -1140,6 +1253,13 @@ func (r *GaleraReconciler) reconcileDelete(ctx context.Context, instance *mariad return ctrlResult, err } + // remove finalizer from the system mariadbaccount and associated secret + err = mariadbv1.DeleteAccountFinalizers( + ctx, helper, r.getRootMariadbAccountName(instance), instance.Namespace) + if err != nil { + return ctrl.Result{}, err + } + // Service is deleted so remove the finalizer. controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) helper.GetLogger().Info("Reconciled Service delete successfully") diff --git a/controllers/mariadbaccount_controller.go b/controllers/mariadbaccount_controller.go index ab2d330f..ae5d4129 100644 --- a/controllers/mariadbaccount_controller.go +++ b/controllers/mariadbaccount_controller.go @@ -144,6 +144,8 @@ func (r *MariaDBAccountReconciler) reconcileCreateOrUpdate( var mariadbDatabase *databasev1beta1.MariaDBDatabase var err error + log.Info("Reconcile MariaDBAccount create or update") + if instance.IsUserAccount() { // for User account, get a handle to the current, active MariaDBDatabase. // if not ready yet, requeue. @@ -245,7 +247,7 @@ func (r *MariaDBAccountReconciler) reconcileCreateOrUpdate( // set up new Secret and remove finalizer from old secret if instance.Status.CurrentSecret != instance.Spec.Secret { currentSecret := instance.Status.CurrentSecret - err = r.removeSecretFinalizer(ctx, helper, currentSecret, instance.Namespace) + err = r.removeSecretFinalizer(ctx, log, helper, currentSecret, instance.Namespace) if err == nil { instance.Status.CurrentSecret = instance.Spec.Secret } else { @@ -279,6 +281,8 @@ func (r *MariaDBAccountReconciler) reconcileDelete( var mariadbDatabase *databasev1beta1.MariaDBDatabase var err error + log.Info("Reconcile MariaDBAccount delete") + if instance.IsUserAccount() { mariadbDatabase, result, err = r.getMariaDBDatabaseForDelete(ctx, log, helper, instance) if mariadbDatabase == nil { @@ -296,6 +300,7 @@ func (r *MariaDBAccountReconciler) reconcileDelete( finalizersWeCareAbout = append(finalizersWeCareAbout, f) } } + if len(finalizersWeCareAbout) > 0 { instance.Status.Conditions.MarkFalse( databasev1beta1.MariaDBAccountReadyCondition, @@ -324,10 +329,27 @@ func (r *MariaDBAccountReconciler) reconcileDelete( dbGalera, dbHostname, result, err := r.getGaleraForCreateOrDelete( ctx, log, helper, instance, mariadbDatabase, ) - // if Galera CR was not found at all, this indicates MariaDBAccount is not - // implemented in the database either, so remove all finalizers and - // exit + + // if Galera CR was not found or in a deletion process, this indicates + // we won't ever have a DB server with which to run a DROP for the + // account, so remove all finalizers and exit + + var galeraGone bool + if k8s_errors.IsNotFound(err) { + log.Info("Galera instance not found, so we will remove finalizers and skip account delete") + galeraGone = true + } else if err != nil { + // unexpected error code + return result, err + } else if dbGalera == nil || !dbGalera.DeletionTimestamp.IsZero() { + log.Info("Galera deleted or deletion timestamp is non-zero, so we will remove finalizers and skip account delete") + galeraGone = true + } else { + galeraGone = false + } + + if galeraGone { // remove MariaDBAccount finalizer from MariaDBDatabase - this takes the // form such as openstack.org/mariadbaccount- (this naming // scheme allows multiple MariaDBAccounts to claim the same MariaDBDatabase @@ -347,10 +369,8 @@ func (r *MariaDBAccountReconciler) reconcileDelete( // MariaDBAccount as well as the Secret which is referenced from the // MariaDBAccount, allowing both to be deleted assuming no other // finalizers - err := r.removeAccountAndSecretFinalizer(ctx, helper, instance) + err := r.removeAccountAndSecretFinalizer(ctx, log, helper, instance) return ctrl.Result{}, err - } else if dbGalera == nil { - return result, err } dbContainerImage := dbGalera.Spec.ContainerImage @@ -414,7 +434,7 @@ func (r *MariaDBAccountReconciler) reconcileDelete( // remove finalizer "openstack.org/mariadbaccount" from // both the MariaDBAccount as well as the Secret which is referenced // from the MariaDBAccount, allowing both to be deleted - err = r.removeAccountAndSecretFinalizer(ctx, helper, instance) + err = r.removeAccountAndSecretFinalizer(ctx, log, helper, instance) return ctrl.Result{}, err } @@ -515,7 +535,7 @@ func (r *MariaDBAccountReconciler) getMariaDBDatabaseForDelete(ctx context.Conte // So remove finalizer "openstack.org/mariadbaccount" from // both the MariaDBAccount as well as the Secret which is referenced // from the MariaDBAccount, allowing both to be deleted - err := r.removeAccountAndSecretFinalizer(ctx, helper, instance) + err := r.removeAccountAndSecretFinalizer(ctx, log, helper, instance) return nil, ctrl.Result{}, err } @@ -535,7 +555,7 @@ func (r *MariaDBAccountReconciler) getMariaDBDatabaseForDelete(ctx context.Conte // So remove finalizer "openstack.org/mariadbaccount" from // both the MariaDBAccount as well as the Secret which is referenced // from the MariaDBAccount, allowing both to be deleted - err = r.removeAccountAndSecretFinalizer(ctx, helper, instance) + err = r.removeAccountAndSecretFinalizer(ctx, log, helper, instance) return nil, ctrl.Result{}, err } else { // unhandled error; exit without change @@ -578,7 +598,7 @@ func (r *MariaDBAccountReconciler) getMariaDBDatabaseForDelete(ctx context.Conte // remove finalizer "openstack.org/mariadbaccount" from // both the MariaDBAccount as well as the Secret which is referenced // from the MariaDBAccount - err = r.removeAccountAndSecretFinalizer(ctx, helper, instance) + err = r.removeAccountAndSecretFinalizer(ctx, log, helper, instance) return nil, ctrl.Result{}, err } @@ -607,7 +627,11 @@ func (r *MariaDBAccountReconciler) getGaleraForCreateOrDelete( dbGalera, err = GetDatabaseObject(ctx, r.Client, dbName, instance.Namespace) if err != nil { - log.Error(err, "Error retrieving Galera instance") + if k8s_errors.IsNotFound(err) { + log.Info(fmt.Sprintf("Galera instance '%s' does not exist", dbName)) + } else { + log.Error(err, "Error retrieving Galera instance") + } instance.Status.Conditions.Set(condition.FalseCondition( databasev1beta1.MariaDBServerReadyCondition, @@ -632,6 +656,19 @@ func (r *MariaDBAccountReconciler) getGaleraForCreateOrDelete( return nil, "", ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, nil } + if !dbGalera.DeletionTimestamp.IsZero() { + log.Info("DB server marked for deletion, preventing account operations from proceeding. Will seek to remove finalizers and exit") + + instance.Status.Conditions.MarkFalse( + databasev1beta1.MariaDBServerReadyCondition, + databasev1beta1.ReasonDBResourceDeleted, + condition.SeverityInfo, + databasev1beta1.MariaDBServerDeletedMessage, + ) + + return dbGalera, "", ctrl.Result{}, nil + } + dbHostname, dbHostResult, err := databasev1beta1.GetServiceHostname(ctx, helper, dbGalera.Name, dbGalera.Namespace) if (err != nil || dbHostResult != ctrl.Result{}) { @@ -731,10 +768,10 @@ func (r *MariaDBAccountReconciler) ensureAccountSecret( // removeAccountAndSecretFinalizer - removes finalizer from mariadbaccount as well // as current primary secret func (r *MariaDBAccountReconciler) removeAccountAndSecretFinalizer(ctx context.Context, - helper *helper.Helper, instance *databasev1beta1.MariaDBAccount) error { + log logr.Logger, helper *helper.Helper, instance *databasev1beta1.MariaDBAccount) error { err := r.removeSecretFinalizer( - ctx, helper, instance.Spec.Secret, instance.Namespace, + ctx, log, helper, instance.Spec.Secret, instance.Namespace, ) if err != nil && !k8s_errors.IsNotFound(err) { return err @@ -747,14 +784,19 @@ func (r *MariaDBAccountReconciler) removeAccountAndSecretFinalizer(ctx context.C } func (r *MariaDBAccountReconciler) removeSecretFinalizer(ctx context.Context, - helper *helper.Helper, secretName string, namespace string) error { + log logr.Logger, helper *helper.Helper, secretName string, namespace string) error { accountSecret, _, err := secret.GetSecret(ctx, helper, secretName, namespace) if err == nil { if controllerutil.RemoveFinalizer(accountSecret, helper.GetFinalizer()) { err = r.Update(ctx, accountSecret) if err != nil { + log.Error( + err, + fmt.Sprintf("Error removing mariadbaccount finalizer from secret '%s', will try again", secretName)) return err + } else { + log.Info(fmt.Sprintf("Successfully removed mariadbaccount finalizer from secret '%s'", secretName)) } } } else if !k8s_errors.IsNotFound(err) { diff --git a/templates/account.sh b/templates/account.sh index 891ede76..0692337c 100755 --- a/templates/account.sh +++ b/templates/account.sh @@ -1,6 +1,6 @@ #!/bin/bash -MYSQL_REMOTE_HOST={{.DatabaseHostname}} source /var/lib/operator-scripts/mysql_root_auth.sh +MYSQL_REMOTE_HOST="{{.DatabaseHostname}}" source /var/lib/operator-scripts/mysql_root_auth.sh export DatabasePassword=${DatabasePassword:?"Please specify a DatabasePassword variable."} @@ -29,6 +29,10 @@ GRANT ALL PRIVILEGES ON ${GRANT_DATABASE}.* TO '{{.UserName}}'@'localhost'; GRANT ALL PRIVILEGES ON ${GRANT_DATABASE}.* TO '{{.UserName}}'@'%'; EOF +# If we just changed the root password, update MYSQL_CMD to use the new password +if [ "{{.UserName}}" = "{{.DatabaseAdminUsername}}" ]; then + MYSQL_CMD="mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -p${DatabasePassword} -P 3306" +fi # search for the account. not using SHOW CREATE USER to avoid displaying # password hash diff --git a/templates/database.sh b/templates/database.sh index 7f845a9f..f83a2c5e 100755 --- a/templates/database.sh +++ b/templates/database.sh @@ -1,6 +1,6 @@ #!/bin/bash -MYSQL_REMOTE_HOST={{.DatabaseHostname}} source /var/lib/operator-scripts/mysql_root_auth.sh +MYSQL_REMOTE_HOST="{{.DatabaseHostname}}" source /var/lib/operator-scripts/mysql_root_auth.sh mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -P 3306 -e "CREATE DATABASE IF NOT EXISTS {{.DatabaseName}}; ALTER DATABASE {{.DatabaseName}} CHARACTER SET '{{.DefaultCharacterSet}}' COLLATE '{{.DefaultCollation}}';" diff --git a/templates/delete_account.sh b/templates/delete_account.sh index ebc60f13..0c687827 100755 --- a/templates/delete_account.sh +++ b/templates/delete_account.sh @@ -1,5 +1,5 @@ #!/bin/bash -MYSQL_REMOTE_HOST={{.DatabaseHostname}} source /var/lib/operator-scripts/mysql_root_auth.sh +MYSQL_REMOTE_HOST="{{.DatabaseHostname}}" source /var/lib/operator-scripts/mysql_root_auth.sh mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -P 3306 -e "DROP USER IF EXISTS '{{.UserName}}'@'localhost'; DROP USER IF EXISTS '{{.UserName}}'@'%';" diff --git a/templates/delete_database.sh b/templates/delete_database.sh index fc15f137..03443902 100755 --- a/templates/delete_database.sh +++ b/templates/delete_database.sh @@ -1,6 +1,6 @@ #!/bin/bash -MYSQL_REMOTE_HOST={{.DatabaseHostname}} source /var/lib/operator-scripts/mysql_root_auth.sh +MYSQL_REMOTE_HOST="{{.DatabaseHostname}}" source /var/lib/operator-scripts/mysql_root_auth.sh mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -P 3306 -e "DROP DATABASE IF EXISTS {{.DatabaseName}};" diff --git a/templates/galera/bin/mysql_bootstrap.sh b/templates/galera/bin/mysql_bootstrap.sh index c308b568..4128c3c1 100755 --- a/templates/galera/bin/mysql_bootstrap.sh +++ b/templates/galera/bin/mysql_bootstrap.sh @@ -1,13 +1,58 @@ #!/bin/bash set +eux -source /var/lib/operator-scripts/mysql_root_auth.sh +# disable my.cnf caching in mysql_root_auth.sh, so that we definitely +# use the root password defined in the cluster +MYSQL_ROOT_AUTH_BYPASS_CHECKS=true source /var/lib/operator-scripts/mysql_root_auth.sh + +MARIADB_PIDFILE=/var/lib/mysql/mariadb.pid +function kolla_update_db_root_pw { + # update the root password given a set of mariadb datafiles + + # because galera controller generates a new root password if one was + # not sent via pre-existing secret, the root pw has to be updated if + # existing datafiles are present, as they will still store the previous + # root pw which we by definition don't know what it is + + # ported from kolla_extend_start + echo -e "Running with --skip-grant-tables to reset root password" + rm -fv ${MARIADB_PIDFILE} + mysqld_safe --skip-grant-tables --wsrep-on=OFF --pid-file=${MARIADB_PIDFILE} & + + # Wait for the mariadb server to be "Ready" before running root update commands + # NOTE(huikang): the location of mysql's socket file varies depending on the OS distributions. + # Querying the cluster status has to be executed after the existence of mysql.sock and mariadb.pid. + TIMEOUT=${DB_MAX_TIMEOUT:-60} + while [[ ! -S /var/lib/mysql/mysql.sock ]] && \ + [[ ! -S /var/run/mysqld/mysqld.sock ]] || \ + [[ ! -f "${MARIADB_PIDFILE}" ]]; do + if [[ ${TIMEOUT} -gt 0 ]]; then + let TIMEOUT-=1 + sleep 1 + else + echo -e "Surpassed timeout of ${TIMEOUT} without seeing a pidfile" + exit 1 + fi + done + + echo -e "Refreshing root passwords" + mysql -u root <Status->rootDatabaseSecret) and if not working, see if there is a +# different (newer) password in root galera->Spec->rootDatabaseAccount->Secret, +# and try that. This suits the case where a new password was placed in +# galera->Spec->rootDatabaseAccount->Secret, account.sh ran to update the root +# password, but failed to complete, even though the actual password got +# updated. account.sh will run again on a new pod but the password that's in +# galera->Status->rootDatabaseSecret is no longer valid, and would prevent +# account.sh from proceeding a second time. Try the "pending" password just to +# get through, so that account.sh can succeed and +# galera->Status->rootDatabaseSecret can then be updated. + +PASSWORD_VALID=true + +# test password with mysql command if socket exists, or we are remote +if [ "${MYSQL_ROOT_AUTH_BYPASS_CHECKS}" != "true" ] && { [ "${USE_SOCKET}" = "false" ] || [ -S "${MYSQL_SOCKET}" ]; }; then + if ! mysql ${MYSQL_CONN_PARAMS} -uroot -p"${PASSWORD}" -e "SELECT 1;" >/dev/null 2>&1; then + echo "WARNING: primary password retrieved from cluster failed authentication; will try fallback password" >&2 + PASSWORD_VALID=false + fi +fi + +# if password failed, look for alternate password from the mariadbdatabaseaccount +# spec directly. assume we are in root pw flight +if [ "${PASSWORD_VALID}" = "false" ]; then + + MARIADB_ACCOUNT=$(echo "${GALERA_CR}" | python3 -c "import json, sys; print(json.load(sys.stdin)['spec']['rootDatabaseAccount'] or '${GALERA_INSTANCE}-mariadb-root')") -# test again; warn if it doesn't work, however write to my.cnf in any -# case to allow the calling script to continue -if [ "${USE_SOCKET}" = "false" ] || [ -S "${MYSQL_SOCKET}" ]; then + MARIADB_ACCOUNT_CR=$(curl -s \ + --cacert ${CACERT} \ + --header "Content-Type:application/json" \ + --header "Authorization: Bearer ${TOKEN}" \ + "${APISERVER}/${MARIADB_API}/namespaces/${NAMESPACE}/mariadbaccounts/${MARIADB_ACCOUNT}") + + # look in spec.secret + FALLBACK_SECRET_NAME=$(echo "${MARIADB_ACCOUNT_CR}" | python3 -c "import json, sys; print(json.load(sys.stdin)['spec']['secret'])") + + # Get the new password from the fallback secret + PASSWORD=$(curl -s \ + --cacert ${CACERT} \ + --header "Content-Type:application/json" \ + --header "Authorization: Bearer ${TOKEN}" \ + "${APISERVER}/${K8S_API}/namespaces/${NAMESPACE}/secrets/${FALLBACK_SECRET_NAME}" \ + | python3 -c "import json, sys; print(json.load(sys.stdin)['data']['DatabasePassword'])" \ + | base64 -d) + + # test again; warn if it doesn't work, however write to my.cnf in any + # case to allow the calling script to continue if ! mysql ${MYSQL_CONN_PARAMS} -uroot -p"${PASSWORD}" -e "SELECT 1;" >/dev/null 2>&1; then - echo "WARNING: password retrieved from cluster failed authentication" >&2 + echo "WARNING: Both primary and fallback passwords failed authentication, will maintain fallback password" >&2 fi + fi diff --git a/tests/chainsaw/common/galera-assert.yaml b/tests/chainsaw/common/galera-assert.yaml index 3ffdd0ab..6834a81c 100644 --- a/tests/chainsaw/common/galera-assert.yaml +++ b/tests/chainsaw/common/galera-assert.yaml @@ -25,6 +25,10 @@ status: reason: Ready status: "True" type: InputReady + - message: MariaDBAccount System account 'root' creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady - message: RoleBinding created reason: Ready status: "True" diff --git a/tests/chainsaw/tests/update-root-pw/chainsaw-test.yaml b/tests/chainsaw/tests/update-root-pw/chainsaw-test.yaml new file mode 100644 index 00000000..ec7bb03d --- /dev/null +++ b/tests/chainsaw/tests/update-root-pw/chainsaw-test.yaml @@ -0,0 +1,93 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: update-root-pw +spec: + steps: + - name: Deploy 3-node cluster + description: Deploy a 3-node galera cluster for root password update test + bindings: + - name: replicas + value: 3 + try: + - apply: + file: ../../common/galera.yaml + - assert: + file: ../../common/galera-assert.yaml + + - name: Verify root MariaDBAccount is created + description: Verify the root MariaDBAccount exists and has Status.CurrentSecret set + try: + - assert: + file: root-account-initial-assert.yaml + + - name: Verify initial setup + description: Verify the cluster is running with initial root password + try: + - script: + content: | + oc exec -n ${NAMESPACE} -c galera openstack-galera-0 -- /bin/sh -c ' + source /var/lib/operator-scripts/mysql_root_auth.sh + if [ "$MYSQL_PWD" = "newrootpassword123" ]; then + echo "ERROR: password == 'newrootpassword123', should not have changed yet" + exit 1 + fi + echo "PASS: password != 'newrootpassword123' (current: $MYSQL_PWD)" + mysql -e "SELECT 1" > /dev/null + echo "Initial root password works" + ' + + - name: Create new root password secret + description: Create a new secret with updated root password + skipDelete: true + try: + - apply: + file: new-root-secret.yaml + + - name: Update root MariaDBAccount with new secret + description: Apply the new secret to the root MariaDBAccount spec + skipDelete: true + try: + - apply: + file: root-account-updated.yaml + - assert: + file: root-account-assert.yaml + + - name: Verify Galera status updated + description: Wait for Galera status.rootDatabaseSecret to be updated + try: + - assert: + file: galera-status-assert.yaml + + - name: Verify login with new password on all nodes + description: Log into all galera nodes with new root password + try: + - script: + content: | + for pod in openstack-galera-0 openstack-galera-1 openstack-galera-2; do + echo "Testing login on $pod..." + oc exec -n ${NAMESPACE} -c galera $pod -- /bin/sh -c ' + # Clear the cached credentials to force using new password + rm -f $HOME/.my.cnf + source /var/lib/operator-scripts/mysql_root_auth.sh + if [ "$MYSQL_PWD" != "newrootpassword123" ]; then + echo "ERROR: password != 'newrootpassword123' (actual: $MYSQL_PWD)" + exit 1 + fi + echo "PASS: password == 'newrootpassword123'" + mysql -e "SELECT 1" > /dev/null + echo "New root password works on $(hostname)" + ' + done + echo "All nodes verified successfully" + + - name: Verify new password persists + description: Verify the new password still works after cache refresh + try: + - script: + content: | + oc exec -n ${NAMESPACE} -c galera openstack-galera-0 -- /bin/sh -c ' + source /var/lib/operator-scripts/mysql_root_auth.sh + mysql -e "SHOW DATABASES" > /dev/null + echo "New password persists and works correctly" + ' diff --git a/tests/chainsaw/tests/update-root-pw/galera-status-assert.yaml b/tests/chainsaw/tests/update-root-pw/galera-status-assert.yaml new file mode 100644 index 00000000..dc2036c0 --- /dev/null +++ b/tests/chainsaw/tests/update-root-pw/galera-status-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +status: + # Verify that rootDatabaseSecret has been updated to the new secret + rootDatabaseSecret: new-root-secret + bootstrapped: true diff --git a/tests/chainsaw/tests/update-root-pw/new-root-secret.yaml b/tests/chainsaw/tests/update-root-pw/new-root-secret.yaml new file mode 100644 index 00000000..be43bf05 --- /dev/null +++ b/tests/chainsaw/tests/update-root-pw/new-root-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: new-root-secret +type: Opaque +data: + # "newrootpassword123" base64 encoded + DatabasePassword: bmV3cm9vdHBhc3N3b3JkMTIz diff --git a/tests/chainsaw/tests/update-root-pw/root-account-assert.yaml b/tests/chainsaw/tests/update-root-pw/root-account-assert.yaml new file mode 100644 index 00000000..b9ed3d81 --- /dev/null +++ b/tests/chainsaw/tests/update-root-pw/root-account-assert.yaml @@ -0,0 +1,11 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBAccount +metadata: + name: openstack-mariadb-root +spec: + userName: root + secret: new-root-secret + accountType: System +status: + # Verify that the CurrentSecret field has been updated to the new secret + currentSecret: new-root-secret diff --git a/tests/chainsaw/tests/update-root-pw/root-account-initial-assert.yaml b/tests/chainsaw/tests/update-root-pw/root-account-initial-assert.yaml new file mode 100644 index 00000000..866f0419 --- /dev/null +++ b/tests/chainsaw/tests/update-root-pw/root-account-initial-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBAccount +metadata: + name: openstack-mariadb-root +spec: + userName: root + secret: openstack-mariadb-root-db-secret + accountType: System +status: + currentSecret: openstack-mariadb-root-db-secret diff --git a/tests/chainsaw/tests/update-root-pw/root-account-updated.yaml b/tests/chainsaw/tests/update-root-pw/root-account-updated.yaml new file mode 100644 index 00000000..dc3a89f7 --- /dev/null +++ b/tests/chainsaw/tests/update-root-pw/root-account-updated.yaml @@ -0,0 +1,8 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBAccount +metadata: + name: openstack-mariadb-root +spec: + userName: root + secret: new-root-secret + accountType: System diff --git a/tests/kuttl/common/assert_sample_deployment.yaml b/tests/kuttl/common/assert_sample_deployment.yaml index 251435c6..9d87349e 100644 --- a/tests/kuttl/common/assert_sample_deployment.yaml +++ b/tests/kuttl/common/assert_sample_deployment.yaml @@ -32,6 +32,10 @@ status: reason: Ready status: "True" type: InputReady + - message: MariaDBAccount System account 'root' creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady - message: RoleBinding created reason: Ready status: "True" diff --git a/tests/kuttl/tests/account_create/01-assert.yaml b/tests/kuttl/tests/account_create/01-assert.yaml index 98df972c..547b04cf 100644 --- a/tests/kuttl/tests/account_create/01-assert.yaml +++ b/tests/kuttl/tests/account_create/01-assert.yaml @@ -34,6 +34,10 @@ status: reason: Ready status: "True" type: InputReady + - message: MariaDBAccount System account 'root' creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady - message: RoleBinding created reason: Ready status: "True" diff --git a/tests/kuttl/tests/database_create/01-assert.yaml b/tests/kuttl/tests/database_create/01-assert.yaml index 74a26acb..c79ade1a 100644 --- a/tests/kuttl/tests/database_create/01-assert.yaml +++ b/tests/kuttl/tests/database_create/01-assert.yaml @@ -32,6 +32,10 @@ status: reason: Ready status: "True" type: InputReady + - message: MariaDBAccount System account 'root' creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady - message: RoleBinding created reason: Ready status: "True" diff --git a/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml b/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml index 08b5b237..ae2af301 100644 --- a/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml +++ b/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml @@ -27,6 +27,10 @@ status: reason: Ready status: "True" type: InputReady + - message: MariaDBAccount System account 'root' creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady - message: RoleBinding created reason: Ready status: "True" diff --git a/tests/kuttl/tests/galera_log_to_disk/01-assert.yaml b/tests/kuttl/tests/galera_log_to_disk/01-assert.yaml index 74a26acb..c79ade1a 100644 --- a/tests/kuttl/tests/galera_log_to_disk/01-assert.yaml +++ b/tests/kuttl/tests/galera_log_to_disk/01-assert.yaml @@ -32,6 +32,10 @@ status: reason: Ready status: "True" type: InputReady + - message: MariaDBAccount System account 'root' creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady - message: RoleBinding created reason: Ready status: "True"