diff --git a/api/bases/keystone.openstack.org_keystoneapis.yaml b/api/bases/keystone.openstack.org_keystoneapis.yaml index 1ce9da9e..ea238523 100644 --- a/api/bases/keystone.openstack.org_keystoneapis.yaml +++ b/api/bases/keystone.openstack.org_keystoneapis.yaml @@ -89,6 +89,20 @@ spec: description: EnableSecureRBAC - Enable Consistent and Secure RBAC policies type: boolean + fernetMaxActiveKeys: + default: "5" + description: FernetMaxActiveKeys - Maximum number of fernet token + keys after rotation + type: string + fernetRotationContainerImage: + description: Keystone Fernet Rotation Container Image URL (will be + set to environmental default if empty) + type: string + fernetRotationSchedule: + default: 1 0 * * * + description: FernetRotationSchedule - Schedule rotate fernet token + keys + type: string memcachedInstance: default: memcached description: Memcached instance name. @@ -412,6 +426,7 @@ spec: required: - containerImage - databaseInstance + - fernetRotationContainerImage - memcachedInstance - rabbitMqClusterName - secret diff --git a/api/v1beta1/keystoneapi_types.go b/api/v1beta1/keystoneapi_types.go index a2064d7f..0b3c7724 100644 --- a/api/v1beta1/keystoneapi_types.go +++ b/api/v1beta1/keystoneapi_types.go @@ -45,6 +45,9 @@ const ( // KeystoneAPIContainerImage is the fall-back container image for KeystoneAPI KeystoneAPIContainerImage = "quay.io/podified-antelope-centos9/openstack-keystone:current-podified" + + // KeystoneFernetRotationContainerImage is the fall-back container image for Keystone Fernet Rotation + KeystoneFernetRotationContainerImage = "registry.redhat.io/openshift4/ose-cli" ) type KeystoneAPISpec struct { @@ -53,6 +56,9 @@ type KeystoneAPISpec struct { // +kubebuilder:validation:Required // Keystone Container Image URL (will be set to environmental default if empty) ContainerImage string `json:"containerImage"` + // +kubebuilder:validation:Required + // Keystone Fernet Rotation Container Image URL (will be set to environmental default if empty) + FernetRotationContainerImage string `json:"fernetRotationContainerImage"` } // KeystoneAPISpec defines the desired state of KeystoneAPI @@ -119,6 +125,16 @@ type KeystoneAPISpecCore struct { // TrustFlushSuspend - Suspend the cron job to purge trusts TrustFlushSuspend bool `json:"trustFlushSuspend"` + // +kubebuilder:validation:Optional + // +kubebuilder:default="1 0 * * *" + // FernetRotationSchedule - Schedule rotate fernet token keys + FernetRotationSchedule string `json:"fernetRotationSchedule"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="5" + // FernetMaxActiveKeys - Maximum number of fernet token keys after rotation + FernetMaxActiveKeys string `json:"fernetMaxActiveKeys"` + // +kubebuilder:validation:Optional // +kubebuilder:default={admin: AdminPassword} // PasswordSelectors - Selectors to identify the AdminUser password from the Secret @@ -268,11 +284,22 @@ func (instance KeystoneAPI) RbacResourceName() string { return "keystone-" + instance.Name } +// KeystoneAPIFernet - used to create different role for fernet key rotation +type KeystoneAPIFernet struct { + *KeystoneAPI +} + +// RbacResourceName - return the name to be used for rbac objects used for fernet key rotation (serviceaccount, role, rolebinding) +func (instance KeystoneAPIFernet) RbacResourceName() string { + return "keystone-fernet-" + instance.Name +} + // SetupDefaults - initializes any CRD field defaults based on environment variables (the defaulting mechanism itself is implemented via webhooks) func SetupDefaults() { // Acquire environmental defaults and initialize Keystone defaults with them keystoneDefaults := KeystoneAPIDefaults{ ContainerImageURL: util.GetEnvVar("RELATED_IMAGE_KEYSTONE_API_IMAGE_URL_DEFAULT", KeystoneAPIContainerImage), + FernetRotationContainerImageURL: util.GetEnvVar("RELATED_IMAGE_KEYSTONE_FERNET_ROTATION_IMAGE_URL_DEFAULT", KeystoneFernetRotationContainerImage), } SetupKeystoneAPIDefaults(keystoneDefaults) diff --git a/api/v1beta1/keystoneapi_webhook.go b/api/v1beta1/keystoneapi_webhook.go index 21d8a5e7..9ab69d5b 100644 --- a/api/v1beta1/keystoneapi_webhook.go +++ b/api/v1beta1/keystoneapi_webhook.go @@ -38,6 +38,7 @@ import ( // KeystoneAPIDefaults - type KeystoneAPIDefaults struct { ContainerImageURL string + FernetRotationContainerImageURL string } var keystoneAPIDefaults KeystoneAPIDefaults @@ -69,6 +70,9 @@ func (r *KeystoneAPI) Default() { if r.Spec.ContainerImage == "" { r.Spec.ContainerImage = keystoneAPIDefaults.ContainerImageURL } + if r.Spec.FernetRotationContainerImage == "" { + r.Spec.FernetRotationContainerImage = keystoneAPIDefaults.FernetRotationContainerImageURL + } r.Spec.Default() } @@ -151,7 +155,7 @@ func (spec *KeystoneAPISpec) ValidateUpdate(old KeystoneAPISpec, basePath *field return spec.KeystoneAPISpecCore.ValidateUpdate(old.KeystoneAPISpecCore, basePath) } -func (spec *KeystoneAPISpecCore) ValidateUpdate(old KeystoneAPISpecCore, basePath *field.Path) field.ErrorList { +func (spec *KeystoneAPISpecCore) ValidateUpdate(_ KeystoneAPISpecCore, basePath *field.Path) field.ErrorList { var allErrs field.ErrorList // validate the service override key is valid diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index f079bd32..ccff3c7e 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -91,6 +91,26 @@ func (in *KeystoneAPIDefaults) DeepCopy() *KeystoneAPIDefaults { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KeystoneAPIFernet) DeepCopyInto(out *KeystoneAPIFernet) { + *out = *in + if in.KeystoneAPI != nil { + in, out := &in.KeystoneAPI, &out.KeystoneAPI + *out = new(KeystoneAPI) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KeystoneAPIFernet. +func (in *KeystoneAPIFernet) DeepCopy() *KeystoneAPIFernet { + if in == nil { + return nil + } + out := new(KeystoneAPIFernet) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KeystoneAPIList) DeepCopyInto(out *KeystoneAPIList) { *out = *in diff --git a/config/crd/bases/keystone.openstack.org_keystoneapis.yaml b/config/crd/bases/keystone.openstack.org_keystoneapis.yaml index 1ce9da9e..ea238523 100644 --- a/config/crd/bases/keystone.openstack.org_keystoneapis.yaml +++ b/config/crd/bases/keystone.openstack.org_keystoneapis.yaml @@ -89,6 +89,20 @@ spec: description: EnableSecureRBAC - Enable Consistent and Secure RBAC policies type: boolean + fernetMaxActiveKeys: + default: "5" + description: FernetMaxActiveKeys - Maximum number of fernet token + keys after rotation + type: string + fernetRotationContainerImage: + description: Keystone Fernet Rotation Container Image URL (will be + set to environmental default if empty) + type: string + fernetRotationSchedule: + default: 1 0 * * * + description: FernetRotationSchedule - Schedule rotate fernet token + keys + type: string memcachedInstance: default: memcached description: Memcached instance name. @@ -412,6 +426,7 @@ spec: required: - containerImage - databaseInstance + - fernetRotationContainerImage - memcachedInstance - rabbitMqClusterName - secret diff --git a/controllers/#keystoneapi_controller.go# b/controllers/#keystoneapi_controller.go# new file mode 100644 index 00000000..854c9762 --- /dev/null +++ b/controllers/#keystoneapi_controller.go# @@ -0,0 +1,1566 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "time" + + memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + keystone "github.com/openstack-k8s-operators/keystone-operator/pkg/keystone" + "github.com/openstack-k8s-operators/lib-common/modules/common" + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + configmap "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" + cronjob "github.com/openstack-k8s-operators/lib-common/modules/common/cronjob" + deployment "github.com/openstack-k8s-operators/lib-common/modules/common/deployment" + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" + env "github.com/openstack-k8s-operators/lib-common/modules/common/env" + helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + job "github.com/openstack-k8s-operators/lib-common/modules/common/job" + labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" + common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" + oko_secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + util "github.com/openstack-k8s-operators/lib-common/modules/common/util" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" + + "gopkg.in/yaml.v3" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/utils/ptr" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// GetClient - +func (r *KeystoneAPIReconciler) GetClient() client.Client { + return r.Client +} + +// GetKClient - +func (r *KeystoneAPIReconciler) GetKClient() kubernetes.Interface { + return r.Kclient +} + +// GetScheme - +func (r *KeystoneAPIReconciler) GetScheme() *runtime.Scheme { + return r.Scheme +} + +// GetLog returns a logger object with a logging prefix of "controller.name" and additional controller context fields +func (r *KeystoneAPIReconciler) GetLogger(ctx context.Context) logr.Logger { + return log.FromContext(ctx).WithName("Controllers").WithName("KeystoneAPI") +} + +// KeystoneAPIReconciler reconciles a KeystoneAPI object +type KeystoneAPIReconciler struct { + client.Client + Kclient kubernetes.Interface + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=batch,resources=cronjobs,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbdatabases,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbaccounts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbaccounts/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=memcached.openstack.org,resources=memcacheds,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=memcached.openstack.org,resources=memcacheds/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch +// +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete + +// service account, role, rolebinding +// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update;patch +// keystone service account permissions that are needed to grant permission to the above +// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid,resources=securitycontextconstraints,verbs=use +// +kubebuilder:rbac:groups="",resources=pods,verbs=create;delete;get;list;patch;update;watch + +// Reconcile reconcile keystone API requests +func (r *KeystoneAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { + Log := r.GetLogger(ctx) + // Fetch the KeystoneAPI instance + instance := &keystonev1.KeystoneAPI{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. + // For additional cleanup logic use finalizers. Return and don't requeue. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + + helper, err := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + Log, + ) + if err != nil { + return ctrl.Result{}, err + } + + // + // initialize status + // + isNewInstance := instance.Status.Conditions == nil + if isNewInstance { + instance.Status.Conditions = condition.Conditions{} + } + + // Save a copy of the condtions so that we can restore the LastTransitionTime + // when a condition's state doesn't change. + savedConditions := instance.Status.Conditions.DeepCopy() + + // Always patch the instance status when exiting this function so we can persist any changes. + defer func() { + // update the Ready condition based on the sub conditions + if instance.Status.Conditions.AllSubConditionIsTrue() { + instance.Status.Conditions.MarkTrue( + condition.ReadyCondition, condition.ReadyMessage) + } else { + // something is not ready so reset the Ready condition + instance.Status.Conditions.MarkUnknown( + condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage) + // and recalculate it based on the state of the rest of the conditions + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) + err := helper.PatchInstance(ctx, instance) + if err != nil { + _err = err + return + } + }() + + // If we're not deleting this and the service object doesn't have our finalizer, add it. + if instance.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(instance, helper.GetFinalizer()) || isNewInstance { + return ctrl.Result{}, err + } + + // + // Conditions init + // + cl := condition.CreateList( + condition.UnknownCondition(condition.DBReadyCondition, condition.InitReason, condition.DBReadyInitMessage), + condition.UnknownCondition(condition.DBSyncReadyCondition, condition.InitReason, condition.DBSyncReadyInitMessage), + condition.UnknownCondition(condition.RabbitMqTransportURLReadyCondition, condition.InitReason, condition.RabbitMqTransportURLReadyInitMessage), + condition.UnknownCondition(condition.MemcachedReadyCondition, condition.InitReason, condition.MemcachedReadyInitMessage), + condition.UnknownCondition(condition.ExposeServiceReadyCondition, condition.InitReason, condition.ExposeServiceReadyInitMessage), + condition.UnknownCondition(condition.BootstrapReadyCondition, condition.InitReason, condition.BootstrapReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), + condition.UnknownCondition(condition.CronJobReadyCondition, condition.InitReason, condition.CronJobReadyInitMessage), + condition.UnknownCondition(condition.TLSInputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + // service account, role, rolebinding conditions + condition.UnknownCondition(condition.ServiceAccountReadyCondition, condition.InitReason, condition.ServiceAccountReadyInitMessage), + condition.UnknownCondition(condition.RoleReadyCondition, condition.InitReason, condition.RoleReadyInitMessage), + condition.UnknownCondition(condition.RoleBindingReadyCondition, condition.InitReason, condition.RoleBindingReadyInitMessage), + ) + + instance.Status.Conditions.Init(&cl) + instance.Status.ObservedGeneration = instance.Generation + + if instance.Status.Hash == nil { + instance.Status.Hash = map[string]string{} + } + if instance.Status.APIEndpoints == nil { + instance.Status.APIEndpoints = map[string]string{} + } + if instance.Status.NetworkAttachments == nil { + instance.Status.NetworkAttachments = map[string][]string{} + } + + // Handle service delete + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instance, helper) + } + + // Handle non-deleted clusters + return r.reconcileNormal(ctx, instance, helper) +} + +// fields to index to reconcile when change +const ( + passwordSecretField = ".spec.secret" + caBundleSecretNameField = ".spec.tls.caBundleSecretName" + tlsAPIInternalField = ".spec.tls.api.internal.secretName" + tlsAPIPublicField = ".spec.tls.api.public.secretName" +) + +var allWatchFields = []string{ + passwordSecretField, + caBundleSecretNameField, + tlsAPIInternalField, + tlsAPIPublicField, +} + +// SetupWithManager - +func (r *KeystoneAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + Log := r.GetLogger(ctx) + + // index passwordSecretField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &keystonev1.KeystoneAPI{}, passwordSecretField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*keystonev1.KeystoneAPI) + if cr.Spec.Secret == "" { + return nil + } + return []string{cr.Spec.Secret} + }); err != nil { + return err + } + + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &keystonev1.KeystoneAPI{}, caBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*keystonev1.KeystoneAPI) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + + // index tlsAPIInternalField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &keystonev1.KeystoneAPI{}, tlsAPIInternalField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*keystonev1.KeystoneAPI) + if cr.Spec.TLS.API.Internal.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.API.Internal.SecretName} + }); err != nil { + return err + } + + // index tlsAPIPublicField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &keystonev1.KeystoneAPI{}, tlsAPIPublicField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*keystonev1.KeystoneAPI) + if cr.Spec.TLS.API.Public.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.API.Public.SecretName} + }); err != nil { + return err + } + + memcachedFn := func(ctx context.Context, o client.Object) []reconcile.Request { + result := []reconcile.Request{} + + // get all KeystoneAPI CRs + keystoneAPIs := &keystonev1.KeystoneAPIList{} + listOpts := []client.ListOption{ + client.InNamespace(o.GetNamespace()), + } + if err := r.Client.List(ctx, keystoneAPIs, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve KeystoneAPI CRs %w") + return nil + } + + for _, cr := range keystoneAPIs.Items { + if o.GetName() == cr.Spec.MemcachedInstance { + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Memcached %s is used by KeystoneAPI CR %s", o.GetName(), cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + if len(result) > 0 { + return result + } + return nil + } + + return ctrl.NewControllerManagedBy(mgr). + For(&keystonev1.KeystoneAPI{}). + Owns(&mariadbv1.MariaDBDatabase{}). + Owns(&mariadbv1.MariaDBAccount{}). + Owns(&batchv1.Job{}). + Owns(&batchv1.CronJob{}). + Owns(&corev1.Service{}). + Owns(&corev1.Secret{}). + Owns(&corev1.ConfigMap{}). + Owns(&appsv1.Deployment{}). + Owns(&corev1.ServiceAccount{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + Watches(&memcachedv1.Memcached{}, + handler.EnqueueRequestsFromMapFunc(memcachedFn)). + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). + Complete(r) +} + +func (r *KeystoneAPIReconciler) findObjectsForSrc(ctx context.Context, src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + Log := r.GetLogger(context.Background()) + + for _, field := range allWatchFields { + crList := &keystonev1.KeystoneAPIList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.List(ctx, crList, listOps) + if err != nil { + return []reconcile.Request{} + } + + for _, item := range crList.Items { + Log.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + +func (r *KeystoneAPIReconciler) reconcileDelete(ctx context.Context, instance *keystonev1.KeystoneAPI, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info("Reconciling Service delete") + + // We need to allow all KeystoneEndpoint and KeystoneService processing to finish + // in the case of a delete before we remove the finalizers. For instance, in the + // case of the Memcached dependency, if Memcached is deleted before all Keystone + // cleanup has finished, then the Keystone logic will likely hit a 500 error and + // thus its deletion will hang indefinitely. + for _, finalizer := range instance.Finalizers { + // If this finalizer is not our KeystoneAPI finalizer, then it is either + // a KeystoneService or KeystoneEndpointer finalizer, which indicates that + // there is more Keystone processing that needs to finish before we can + // allow our DB and Memcached dependencies to be potentially deleted + // themselves + if finalizer != helper.GetFinalizer() { + return ctrl.Result{}, nil + } + } + + // Remove our finalizer from Memcached + memcached, err := memcachedv1.GetMemcachedByName(ctx, helper, instance.Spec.MemcachedInstance, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if !k8s_errors.IsNotFound(err) && memcached != nil { + if controllerutil.RemoveFinalizer(memcached, helper.GetFinalizer()) { + err := r.Update(ctx, memcached) + if err != nil { + return ctrl.Result{}, err + } + } + } + + // remove db finalizer before the keystone one + db, err := mariadbv1.GetDatabaseByNameAndAccount(ctx, helper, keystone.DatabaseCRName, instance.Spec.DatabaseAccount, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if !k8s_errors.IsNotFound(err) { + if err := db.DeleteFinalizer(ctx, helper); err != nil { + return ctrl.Result{}, err + } + } + + // Service is deleted so remove the finalizer. + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + Log.Info("Reconciled Service delete successfully") + + return ctrl.Result{}, nil +} + +func (r *KeystoneAPIReconciler) reconcileInit( + ctx context.Context, + instance *keystonev1.KeystoneAPI, + helper *helper.Helper, + serviceLabels map[string]string, + serviceAnnotations map[string]string, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info("Reconciling Service init") + // + // Service account, role, binding + // + rbacRules := []rbacv1.PolicyRule{ + { + APIGroups: []string{"security.openshift.io"}, + ResourceNames: []string{"anyuid"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"create", "get", "list", "watch", "update", "patch", "delete"}, + }, + } + rbacResult, err := common_rbac.ReconcileRbac(ctx, helper, instance, rbacRules) + if err != nil { + return rbacResult, err + } else if (rbacResult != ctrl.Result{}) { + return rbacResult, nil + } + + // + // Service account, role, binding for fernet key rotation + // + fernetRbacRules := []rbacv1.PolicyRule{ + { + APIGroups: []string{"security.openshift.io"}, + ResourceNames: []string{"anyuid"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"patch"}, + }, + } + fernetRbacResult, err := common_rbac.ReconcileRbac(ctx, helper, keystonev1.KeystoneAPIFernet{KeystoneAPI: instance}, fernetRbacRules) + if err != nil { + return fernetRbacResult, err + } else if (rbacResult != ctrl.Result{}) { + return fernetRbacResult, nil + } + + // + // run keystone db sync + // + dbSyncHash := instance.Status.Hash[keystonev1.DbSyncHash] + jobDef := keystone.DbSyncJob(instance, serviceLabels, serviceAnnotations) + dbSyncjob := job.NewJob( + jobDef, + keystonev1.DbSyncHash, + instance.Spec.PreserveJobs, + 5*time.Second, + dbSyncHash, + ) + ctrlResult, err := dbSyncjob.DoJob( + ctx, + helper, + ) + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBSyncReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBSyncReadyRunningMessage)) + return ctrlResult, nil + } + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBSyncReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBSyncReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if dbSyncjob.HasChanged() { + instance.Status.Hash[keystonev1.DbSyncHash] = dbSyncjob.GetHash() + Log.Info(fmt.Sprintf("Job %s hash added - %s", jobDef.Name, instance.Status.Hash[keystonev1.DbSyncHash])) + } + instance.Status.Conditions.MarkTrue(condition.DBSyncReadyCondition, condition.DBSyncReadyMessage) + + // run keystone db sync - end + + // + // create service/s + // + keystoneEndpoints := map[service.Endpoint]endpoint.Data{ + service.EndpointPublic: { + Port: keystone.KeystonePublicPort, + }, + service.EndpointInternal: { + Port: keystone.KeystoneInternalPort, + }, + } + + apiEndpoints := make(map[string]string) + for endpointType, data := range keystoneEndpoints { + endpointTypeStr := string(endpointType) + endpointName := instance.Name + "-" + endpointTypeStr + + svcOverride := instance.Spec.Override.Service[endpointType] + if svcOverride.EmbeddedLabelsAnnotations == nil { + svcOverride.EmbeddedLabelsAnnotations = &service.EmbeddedLabelsAnnotations{} + } + + exportLabels := util.MergeStringMaps( + serviceLabels, + map[string]string{ + service.AnnotationEndpointKey: endpointTypeStr, + }, + ) + + // Create the service + svc, err := service.NewService( + service.GenericService(&service.GenericServiceDetails{ + Name: endpointName, + Namespace: instance.Namespace, + Labels: exportLabels, + Selector: serviceLabels, + Port: service.GenericServicePort{ + Name: endpointName, + Port: data.Port, + Protocol: corev1.ProtocolTCP, + }, + }), + 5, + &svcOverride.OverrideSpec, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + + svc.AddAnnotation(map[string]string{ + service.AnnotationEndpointKey: endpointTypeStr, + }) + + // add Annotation to whether creating an ingress is required or not + if endpointType == service.EndpointPublic && svc.GetServiceType() == corev1.ServiceTypeClusterIP { + svc.AddAnnotation(map[string]string{ + service.AnnotationIngressCreateKey: "true", + }) + } else { + svc.AddAnnotation(map[string]string{ + service.AnnotationIngressCreateKey: "false", + }) + if svc.GetServiceType() == corev1.ServiceTypeLoadBalancer { + svc.AddAnnotation(map[string]string{ + service.AnnotationHostnameKey: svc.GetServiceHostname(), // add annotation to register service name in dnsmasq + }) + } + } + + ctrlResult, err := svc.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error())) + + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.ExposeServiceReadyRunningMessage)) + return ctrlResult, nil + } + // create service - end + + // if TLS is enabled + if instance.Spec.TLS.API.Enabled(endpointType) { + // set endpoint protocol to https + data.Protocol = ptr.To(service.ProtocolHTTPS) + } + + apiEndpoints[string(endpointType)], err = svc.GetAPIEndpoint( + svcOverride.EndpointURL, data.Protocol, data.Path) + if err != nil { + return ctrl.Result{}, err + } + } + + instance.Status.Conditions.MarkTrue(condition.ExposeServiceReadyCondition, condition.ExposeServiceReadyMessage) + + // + // Update instance status with service endpoint url from route host information + // + instance.Status.APIEndpoints = apiEndpoints + + // expose service - end + + // + // BootStrap Job + // + jobDef = keystone.BootstrapJob(instance, serviceLabels, serviceAnnotations, instance.Status.APIEndpoints) + bootstrapjob := job.NewJob( + jobDef, + keystonev1.BootstrapHash, + instance.Spec.PreserveJobs, + 5*time.Second, + instance.Status.Hash[keystonev1.BootstrapHash], + ) + ctrlResult, err = bootstrapjob.DoJob( + ctx, + helper, + ) + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.BootstrapReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.BootstrapReadyRunningMessage)) + return ctrlResult, nil + } + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.BootstrapReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.BootstrapReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if bootstrapjob.HasChanged() { + instance.Status.Hash[keystonev1.BootstrapHash] = bootstrapjob.GetHash() + Log.Info(fmt.Sprintf("Job %s hash added - %s", jobDef.Name, instance.Status.Hash[keystonev1.BootstrapHash])) + } + instance.Status.Conditions.MarkTrue(condition.BootstrapReadyCondition, condition.BootstrapReadyMessage) + + // run keystone bootstrap - end + + Log.Info("Reconciled Service init successfully") + return ctrl.Result{}, nil +} + +func (r *KeystoneAPIReconciler) reconcileUpdate(ctx context.Context) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info("Reconciling Service update") + + // TODO: should have minor update tasks if required + // - delete dbsync hash from status to rerun it? + + Log.Info("Reconciled Service update successfully") + return ctrl.Result{}, nil +} + +func (r *KeystoneAPIReconciler) reconcileUpgrade(ctx context.Context) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info("Reconciling Service upgrade") + + // TODO: should have major version upgrade tasks + // -delete dbsync hash from status to rerun it? + + Log.Info("Reconciled Service upgrade successfully") + return ctrl.Result{}, nil +} + +func (r *KeystoneAPIReconciler) reconcileNormal( + ctx context.Context, + instance *keystonev1.KeystoneAPI, + helper *helper.Helper, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info("Reconciling Service") + + serviceLabels := map[string]string{ + common.AppSelector: keystone.ServiceName, + common.OwnerSelector: instance.Name, + } + + // ConfigMap + configMapVars := make(map[string]env.Setter) + + // + // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map + // NOTE: VerifySecret handles the "not found" error and returns RequeueAfter ctrl.Result if so, so we don't + // need to check the error type here + // + hash, result, err := oko_secret.VerifySecret(ctx, types.NamespacedName{Name: instance.Spec.Secret, Namespace: instance.Namespace}, []string{"AdminPassword"}, helper.GetClient(), time.Second*10) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } else if (result != ctrl.Result{}) { + // This case is "secret not found". VerifySecret already logs a message for it + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return result, nil + } + configMapVars[instance.Spec.Secret] = env.SetValue(hash) + + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + + // run check OpenStack secret - end + + // + // create service DB instance + // + db, result, err := r.ensureDB(ctx, helper, instance) + if err != nil { + return ctrl.Result{}, err + } else if (result != ctrl.Result{}) { + return result, nil + } + // create service DB - end + + // + // create RabbitMQ transportURL CR and get the actual URL from the associated secret that is created + // + transportURL, op, err := r.transportURLCreateOrUpdate(ctx, instance, serviceLabels) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.RabbitMqTransportURLReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.RabbitMqTransportURLReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if op != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("TransportURL %s successfully reconciled - operation: %s", transportURL.Name, string(op))) + } + + instance.Status.TransportURLSecret = transportURL.Status.SecretName + + if instance.Status.TransportURLSecret == "" { + Log.Info(fmt.Sprintf("Waiting for TransportURL %s secret to be created", transportURL.Name)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.RabbitMqTransportURLReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.RabbitMqTransportURLReadyRunningMessage)) + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, nil + } + Log.Info(fmt.Sprintf("TransportURL secret name %s", transportURL.Status.SecretName)) + instance.Status.Conditions.MarkTrue(condition.RabbitMqTransportURLReadyCondition, condition.RabbitMqTransportURLReadyMessage) + // run check rabbitmq - end + + // + // Check for required memcached used for caching + // + memcached, err := memcachedv1.GetMemcachedByName(ctx, helper, instance.Spec.MemcachedInstance, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info(fmt.Sprintf("memcached %s not found", instance.Spec.MemcachedInstance)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.MemcachedReadyWaitingMessage)) + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.MemcachedReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // Add finalizer to Memcached to prevent it from being deleted now that we're using it + if controllerutil.AddFinalizer(memcached, helper.GetFinalizer()) { + err := r.Update(ctx, memcached) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.MemcachedReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + } + + if !memcached.IsReady() { + Log.Info(fmt.Sprintf("memcached %s is not ready", memcached.Name)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.MemcachedReadyWaitingMessage)) + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + // Mark the Memcached Service as Ready if we get to this point with no errors + instance.Status.Conditions.MarkTrue( + condition.MemcachedReadyCondition, condition.MemcachedReadyMessage) + // run check memcached - end + + // + // Create ConfigMaps and Secrets required as input for the Service and calculate an overall hash of hashes + // + + // + // create Configmap required for keystone input + // - %-scripts configmap holding scripts to e.g. bootstrap the service + // - %-config configmap holding minimal keystone config required to get the service up, user can add additional files to be added to the service + // - parameters which has passwords gets added from the OpenStack secret via the init container + // + err = r.generateServiceConfigMaps(ctx, instance, helper, &configMapVars, memcached, db) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // + // Create secret holding fernet keys (for token and credential) + // + // TODO key rotation + err = r.ensureFernetKeys(ctx, instance, helper, &configMapVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, err := tls.ValidateCACertSecret( + ctx, + helper.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + fmt.Sprintf(condition.TLSInputReadyWaitingMessage, instance.Spec.TLS.CaBundleSecretName))) + return ctrl.Result{}, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if hash != "" { + configMapVars[tls.CABundleKey] = env.SetValue(hash) + } + } + + // Validate API service certs secrets + certsHash, err := instance.Spec.TLS.API.ValidateCertSecrets(ctx, helper, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + fmt.Sprintf(condition.TLSInputReadyWaitingMessage, err.Error()))) + return ctrl.Result{}, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + configMapVars[tls.TLSHashName] = env.SetValue(certsHash) + + // all cert input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. + inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configMapVars) + if err != nil { + return ctrl.Result{}, err + } else if hashChanged { + // Hash changed and instance status should be updated (which will be done by main defer func), + // so we need to return and reconcile again + return ctrl.Result{}, nil + } + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + + // Create ConfigMaps and Secrets - end + + // + // TODO check when/if Init, Update, or Upgrade should/could be skipped + // + + // networks to attach to + for _, netAtt := range instance.Spec.NetworkAttachments { + _, err := nad.GetNADWithName(ctx, helper, netAtt, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info(fmt.Sprintf("network-attachment-definition %s not found", netAtt)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyWaitingMessage, + netAtt)) + return ctrl.Result{RequeueAfter: time.Second * 10}, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + } + + serviceAnnotations, err := nad.CreateNetworksAnnotation(instance.Namespace, instance.Spec.NetworkAttachments) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed create network annotation from %s: %w", + instance.Spec.NetworkAttachments, err) + } + + // Handle service init + ctrlResult, err := r.reconcileInit(ctx, instance, helper, serviceLabels, serviceAnnotations) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // Handle service update + ctrlResult, err = r.reconcileUpdate(ctx) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // Handle service upgrade + ctrlResult, err = r.reconcileUpgrade(ctx) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // + // normal reconcile tasks + // + + // Define a new Deployment object + deplDef, err := keystone.Deployment(instance, inputHash, serviceLabels, serviceAnnotations) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + depl := deployment.NewDeployment( + deplDef, + 5*time.Second, + ) + + ctrlResult, err = depl.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + return ctrlResult, nil + } + instance.Status.ReadyCount = depl.GetDeployment().Status.ReadyReplicas + + // verify if network attachment matches expectations + networkReady, networkAttachmentStatus, err := nad.VerifyNetworkStatusFromAnnotation(ctx, helper, instance.Spec.NetworkAttachments, serviceLabels, instance.Status.ReadyCount) + if err != nil { + return ctrl.Result{}, err + } + + instance.Status.NetworkAttachments = networkAttachmentStatus + if networkReady { + instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage) + } else { + err := fmt.Errorf("not all pods have interfaces with ips as configured in NetworkAttachments: %s", instance.Spec.NetworkAttachments) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + + if instance.Status.ReadyCount > 0 { + instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + } + // create Deployment - end + + if instance.Status.ReadyCount == *instance.Spec.Replicas { + // remove finalizers from unused MariaDBAccount records + err = mariadbv1.DeleteUnusedMariaDBAccountFinalizers(ctx, helper, keystone.DatabaseName, instance.Spec.DatabaseAccount, instance.Namespace) + if err != nil { + return ctrl.Result{}, err + } + } + + // create Trust Flush CronJob + cronjobDef := keystone.CronJob(instance, serviceLabels, serviceAnnotations) + trustflushjob := cronjob.NewCronJob( + cronjobDef, + 5*time.Second, + ) + + ctrlResult, err = trustflushjob.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CronJobReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CronJobReadyErrorMessage, + err.Error())) + return ctrlResult, err + } + + instance.Status.Conditions.MarkTrue(condition.CronJobReadyCondition, condition.CronJobReadyMessage) + // create Trust Flush CronJob - end + + // create Fernet Key Rotation CronJob + fernetjobDef := keystone.FernetCronJob(instance, serviceLabels, serviceAnnotations) + fernetjob := cronjob.NewCronJob( + fernetjobDef, + 5*time.Second, + ) + + ctrlResult, err = fernetjob.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CronJobReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CronJobReadyErrorMessage, + err.Error())) + return ctrlResult, err + } + + instance.Status.Conditions.MarkTrue(condition.CronJobReadyCondition, condition.CronJobReadyMessage) + // create Fernet Key Rotation CronJob - end + + // + // create OpenStackClient config + // + err = r.reconcileCloudConfig(ctx, helper, instance) + if err != nil { + return ctrl.Result{}, err + } + + Log.Info("Reconciled Service successfully") + return ctrl.Result{}, nil +} + +func (r *KeystoneAPIReconciler) transportURLCreateOrUpdate( + ctx context.Context, + instance *keystonev1.KeystoneAPI, + serviceLabels map[string]string, +) (*rabbitmqv1.TransportURL, controllerutil.OperationResult, error) { + transportURL := &rabbitmqv1.TransportURL{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-keystone-transport", instance.Name), + Namespace: instance.Namespace, + Labels: serviceLabels, + }, + } + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, transportURL, func() error { + transportURL.Spec.RabbitmqClusterName = instance.Spec.RabbitMqClusterName + err := controllerutil.SetControllerReference(instance, transportURL, r.Scheme) + return err + }) + + return transportURL, op, err +} + +// generateServiceConfigMaps - create create configmaps which hold scripts and service configuration +// TODO add DefaultConfigOverwrite +func (r *KeystoneAPIReconciler) generateServiceConfigMaps( + ctx context.Context, + instance *keystonev1.KeystoneAPI, + h *helper.Helper, + envVars *map[string]env.Setter, + mc *memcachedv1.Memcached, + db *mariadbv1.Database, +) error { + // + // create Configmap/Secret required for keystone input + // - %-scripts configmap holding scripts to e.g. bootstrap the service + // - %-config configmap holding minimal keystone config required to get the service up, user can add additional files to be added to the service + // - parameters which has passwords gets added from the ospSecret via the init container + // + + cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(keystone.ServiceName), map[string]string{}) + + var tlsCfg *tls.Service + if instance.Spec.TLS.Ca.CaBundleSecretName != "" { + tlsCfg = &tls.Service{} + } + + // customData hold any customization for the service. + // custom.conf is going to /etc//.conf.d + // all other files get placed into /etc/ to allow overwrite of e.g. policy.json + // TODO: make sure custom.conf can not be overwritten + customData := map[string]string{ + common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig, + "my.cnf": db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf + } + for key, data := range instance.Spec.DefaultConfigOverwrite { + customData[key] = data + } + + transportURLSecret, _, err := oko_secret.GetSecret(ctx, h, instance.Status.TransportURLSecret, instance.Namespace) + if err != nil { + return err + } + + databaseAccount := db.GetAccount() + dbSecret := db.GetSecret() + + templateParameters := map[string]interface{}{ + "memcachedServers": mc.GetMemcachedServerListString(), + "memcachedTLS": mc.GetMemcachedTLSSupport(), + "TransportURL": string(transportURLSecret.Data["transport_url"]), + "DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", + databaseAccount.Spec.UserName, + string(dbSecret.Data[mariadbv1.DatabasePasswordSelector]), + instance.Status.DatabaseHostname, + keystone.DatabaseName, + ), + "enableSecureRBAC": instance.Spec.EnableSecureRBAC, + } + + // create httpd vhost template parameters + httpdVhostConfig := map[string]interface{}{} + for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { + endptConfig := map[string]interface{}{} + endptConfig["ServerName"] = fmt.Sprintf("%s-%s.%s.svc", instance.Name, endpt.String(), instance.Namespace) + endptConfig["TLS"] = false // default TLS to false, and set it bellow to true if enabled + if instance.Spec.TLS.API.Enabled(endpt) { + endptConfig["TLS"] = true + endptConfig["SSLCertificateFile"] = fmt.Sprintf("/etc/pki/tls/certs/%s.crt", endpt.String()) + endptConfig["SSLCertificateKeyFile"] = fmt.Sprintf("/etc/pki/tls/private/%s.key", endpt.String()) + } + httpdVhostConfig[endpt.String()] = endptConfig + } + templateParameters["VHosts"] = httpdVhostConfig + + tmpl := []util.Template{ + // Scripts + { + Name: fmt.Sprintf("%s-scripts", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeScripts, + InstanceType: instance.Kind, + Labels: cmLabels, + }, + // Configs + { + Name: fmt.Sprintf("%s-config-data", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeConfig, + InstanceType: instance.Kind, + CustomData: customData, + ConfigOptions: templateParameters, + Labels: cmLabels, + }, + } + return oko_secret.EnsureSecrets(ctx, h, instance, tmpl, envVars) +} + +// reconcileConfigMap - creates clouds.yaml +// TODO: most likely should be part of the higher openstack operator +func (r *KeystoneAPIReconciler) reconcileCloudConfig( + ctx context.Context, + h *helper.Helper, + instance *keystonev1.KeystoneAPI, +) error { + // clouds.yaml + var openStackConfig keystone.OpenStackConfig + templateParameters := make(map[string]interface{}) + cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(keystone.ServiceName), map[string]string{}) + + authURL, err := instance.GetEndpoint(endpoint.EndpointPublic) + if err != nil { + return err + } + openStackConfig.Clouds.Default.Auth.AuthURL = authURL + openStackConfig.Clouds.Default.Auth.ProjectName = instance.Spec.AdminProject + openStackConfig.Clouds.Default.Auth.UserName = instance.Spec.AdminUser + openStackConfig.Clouds.Default.Auth.UserDomainName = "Default" + openStackConfig.Clouds.Default.Auth.ProjectDomainName = "Default" + openStackConfig.Clouds.Default.RegionName = instance.Spec.Region + + cloudsYamlVal, err := yaml.Marshal(&openStackConfig) + if err != nil { + return err + } + cloudsYaml := map[string]string{ + "clouds.yaml": string(cloudsYamlVal), + "OS_CLOUD": "default", + } + + cms := []util.Template{ + { + Name: "openstack-config", + Namespace: instance.Namespace, + Type: util.TemplateTypeNone, + InstanceType: instance.Kind, + CustomData: cloudsYaml, + ConfigOptions: templateParameters, + Labels: cmLabels, + }, + } + err = configmap.EnsureConfigMaps(ctx, h, instance, cms, nil) + if err != nil { + return err + } + + // secure.yaml + keystoneSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Spec.Secret, + Namespace: instance.Namespace, + }, + Type: "Opaque", + } + + err = r.Client.Get(ctx, types.NamespacedName{Name: keystoneSecret.Name, Namespace: instance.Namespace}, keystoneSecret) + if err != nil { + return err + } + + var openStackConfigSecret keystone.OpenStackConfigSecret + openStackConfigSecret.Clouds.Default.Auth.Password = string(keystoneSecret.Data[instance.Spec.PasswordSelectors.Admin]) + + secretVal, err := yaml.Marshal(&openStackConfigSecret) + if err != nil { + return err + } + cloudrc := keystone.GenerateCloudrc(&openStackConfigSecret, &openStackConfig) + secretString := map[string]string{ + "secure.yaml": string(secretVal), + "cloudrc": string(cloudrc), + } + + secrets := []util.Template{ + { + Name: "openstack-config-secret", + Namespace: instance.Namespace, + Type: util.TemplateTypeNone, + InstanceType: instance.Kind, + CustomData: secretString, + ConfigOptions: templateParameters, + Labels: cmLabels, + }, + } + + return oko_secret.EnsureSecrets(ctx, h, instance, secrets, nil) +} + +// ensureFernetKeys - creates secret with fernet keys +func (r *KeystoneAPIReconciler) ensureFernetKeys( + ctx context.Context, + instance *keystonev1.KeystoneAPI, + helper *helper.Helper, + envVars *map[string]env.Setter, +) error { + labels := labels.GetLabels(instance, labels.GetGroupLabel(keystone.ServiceName), map[string]string{}) + + // + // check if secret already exist + // + secretName := keystone.ServiceName + secret, hash, err := oko_secret.GetSecret(ctx, helper, secretName, instance.Namespace) + fmt.Println("=========================") + fmt.Println(secretName) + fmt.Println(secret) + fmt.Println(hash) + fmt.Println(err) + fmt.Println("=========================") + + fmt.Println(secret.Data) + fernetKeys := map[string]string{} + var numberKeys int + fmt.Sscan(instance.Spec.FernetMaxActiveKeys, &numberKeys) + fmt.Println("===== FOR") + fmt.Println(numberKeys) + for i := 0; i < numberKeys; i++ { + key := fmt.Sprintf("FernetKeys%d", i) + v, exists := secret.Data[key] + fmt.Println(exists) + fmt.Println(v) + if exists { + fernetKeys[key] = string(v[:]) + } else { + fernetKeys[key] = keystone.GenerateFernetKey() + } + } + fmt.Println(fernetKeys) + tmpl := []util.Template{ + { + Name: secretName, + Namespace: instance.Namespace, + Type: util.TemplateTypeNone, + CustomData: fernetKeys, + Labels: labels, + }, + } + err = oko_secret.EnsureSecrets(ctx, helper, instance, tmpl, envVars) + if err != nil { + return err + } + fmt.Println("=========================") + fmt.Println("=========================") + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } else if k8s_errors.IsNotFound(err) { + fernetKeys := map[string]string{ + "CredentialKeys0": keystone.GenerateFernetKey(), + "CredentialKeys1": keystone.GenerateFernetKey(), + } + var numberKeys int + fmt.Sscan(instance.Spec.FernetMaxActiveKeys, &numberKeys) + for i := 0; i < numberKeys; i++ { + fernetKeys[fmt.Sprintf("FernetKeys%d", i)] = keystone.GenerateFernetKey() + } + + tmpl := []util.Template{ + { + Name: secretName, + Namespace: instance.Namespace, + Type: util.TemplateTypeNone, + CustomData: fernetKeys, + Labels: labels, + }, + } + err := oko_secret.EnsureSecrets(ctx, helper, instance, tmpl, envVars) + if err != nil { + return err + } + } else { + // add hash to envVars + (*envVars)[secret.Name] = env.SetValue(hash) + } + + // TODO: fernet key rotation + + return nil +} + +// createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart +// if any of the input resources change, like configs, passwords, ... +// +// returns the hash, whether the hash changed (as a bool) and any error +func (r *KeystoneAPIReconciler) createHashOfInputHashes( + ctx context.Context, + instance *keystonev1.KeystoneAPI, + envVars map[string]env.Setter, +) (string, bool, error) { + Log := r.GetLogger(ctx) + var hashMap map[string]string + changed := false + mergedMapVars := env.MergeEnvs([]corev1.EnvVar{}, envVars) + hash, err := util.ObjectHash(mergedMapVars) + if err != nil { + return hash, changed, err + } + if hashMap, changed = util.SetHash(instance.Status.Hash, common.InputHashName, hash); changed { + instance.Status.Hash = hashMap + Log.Info(fmt.Sprintf("Input maps hash %s - %s", common.InputHashName, hash)) + } + return hash, changed, nil +} + +func (r *KeystoneAPIReconciler) ensureDB( + ctx context.Context, + h *helper.Helper, + instance *keystonev1.KeystoneAPI, +) (*mariadbv1.Database, ctrl.Result, error) { + + // ensure MariaDBAccount exists. This account record may be created by + // openstack-operator or the cloud operator up front without a specific + // MariaDBDatabase configured yet. Otherwise, a MariaDBAccount CR is + // created here with a generated username as well as a secret with + // generated password. The MariaDBAccount is created without being + // yet associated with any MariaDBDatabase. + _, _, err := mariadbv1.EnsureMariaDBAccount( + ctx, h, instance.Spec.DatabaseAccount, + instance.Namespace, false, keystone.DatabaseUsernamePrefix, + ) + + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + mariadbv1.MariaDBAccountReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + mariadbv1.MariaDBAccountNotReadyMessage, + err.Error())) + + return nil, ctrl.Result{}, err + } + instance.Status.Conditions.MarkTrue( + mariadbv1.MariaDBAccountReadyCondition, + mariadbv1.MariaDBAccountReadyMessage) + + // + // create service DB instance + // + db := mariadbv1.NewDatabaseForAccount( + instance.Spec.DatabaseInstance, // mariadb/galera service to target + keystone.DatabaseName, // name used in CREATE DATABASE in mariadb + keystone.DatabaseCRName, // CR name for MariaDBDatabase + instance.Spec.DatabaseAccount, // CR name for MariaDBAccount + instance.Namespace, // namespace + ) + + // create or patch the DB + ctrlResult, err := db.CreateOrPatchAll(ctx, h) + + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBReadyErrorMessage, + err.Error())) + return db, ctrl.Result{}, err + } + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBReadyRunningMessage)) + return db, ctrlResult, nil + } + // wait for the DB to be setup + // (ksambor) should we use WaitForDBCreatedWithTimeout instead? + ctrlResult, err = db.WaitForDBCreated(ctx, h) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBReadyErrorMessage, + err.Error())) + return db, ctrlResult, err + } + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBReadyRunningMessage)) + return db, ctrlResult, nil + } + + // update Status.DatabaseHostname, used to config the service + instance.Status.DatabaseHostname = db.GetDatabaseHostname() + instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage) + return db, ctrlResult, nil +} diff --git a/controllers/keystoneapi_controller.go b/controllers/keystoneapi_controller.go index 904f6e25..8519bb2b 100644 --- a/controllers/keystoneapi_controller.go +++ b/controllers/keystoneapi_controller.go @@ -468,6 +468,29 @@ func (r *KeystoneAPIReconciler) reconcileInit( return rbacResult, nil } + // + // Service account, role, binding for fernet key rotation + // + fernetRbacRules := []rbacv1.PolicyRule{ + { + APIGroups: []string{"security.openshift.io"}, + ResourceNames: []string{"anyuid"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"patch"}, + }, + } + fernetRbacResult, err := common_rbac.ReconcileRbac(ctx, helper, keystonev1.KeystoneAPIFernet{KeystoneAPI: instance}, fernetRbacRules) + if err != nil { + return fernetRbacResult, err + } else if (rbacResult != ctrl.Result{}) { + return fernetRbacResult, nil + } + // // run keystone db sync // @@ -1079,14 +1102,14 @@ func (r *KeystoneAPIReconciler) reconcileNormal( } } - // create CronJob + // create Trust Flush CronJob cronjobDef := keystone.CronJob(instance, serviceLabels, serviceAnnotations) - cronjob := cronjob.NewCronJob( + trustflushjob := cronjob.NewCronJob( cronjobDef, 5*time.Second, ) - ctrlResult, err = cronjob.CreateOrPatch(ctx, helper) + ctrlResult, err = trustflushjob.CreateOrPatch(ctx, helper) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.CronJobReadyCondition, @@ -1098,7 +1121,28 @@ func (r *KeystoneAPIReconciler) reconcileNormal( } instance.Status.Conditions.MarkTrue(condition.CronJobReadyCondition, condition.CronJobReadyMessage) - // create CronJob - end + // create Trust Flush CronJob - end + + // create Fernet Key Rotation CronJob + fernetjobDef := keystone.FernetCronJob(instance, serviceLabels, serviceAnnotations) + fernetjob := cronjob.NewCronJob( + fernetjobDef, + 5*time.Second, + ) + + ctrlResult, err = fernetjob.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CronJobReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CronJobReadyErrorMessage, + err.Error())) + return ctrlResult, err + } + + instance.Status.Conditions.MarkTrue(condition.CronJobReadyCondition, condition.CronJobReadyMessage) + // create Fernet Key Rotation CronJob - end // // create OpenStackClient config @@ -1332,35 +1376,46 @@ func (r *KeystoneAPIReconciler) ensureFernetKeys( // secretName := keystone.ServiceName secret, hash, err := oko_secret.GetSecret(ctx, helper, secretName, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { return err - } else if k8s_errors.IsNotFound(err) { - fernetKeys := map[string]string{ - "FernetKeys0": keystone.GenerateFernetKey(), - "FernetKeys1": keystone.GenerateFernetKey(), - "CredentialKeys0": keystone.GenerateFernetKey(), - "CredentialKeys1": keystone.GenerateFernetKey(), - } - - tmpl := []util.Template{ - { - Name: secretName, - Namespace: instance.Namespace, - Type: util.TemplateTypeNone, - CustomData: fernetKeys, - Labels: labels, - }, - } - err := oko_secret.EnsureSecrets(ctx, helper, instance, tmpl, envVars) - if err != nil { - return err - } } else { // add hash to envVars (*envVars)[secret.Name] = env.SetValue(hash) } - // TODO: fernet key rotation + fernetKeys := map[string]string{ + "CredentialKeys0": keystone.GenerateFernetKey(), + "CredentialKeys1": keystone.GenerateFernetKey(), + } + + var numberKeys int + fmt.Sscan(instance.Spec.FernetMaxActiveKeys, &numberKeys) + + for i := 0; i < numberKeys; i++ { + key := fmt.Sprintf("FernetKeys%d", i) + v, exists := secret.Data[key] + if exists { + fernetKeys[key] = string(v[:]) + } else { + fernetKeys[key] = keystone.GenerateFernetKey() + } + } + + tmpl := []util.Template{ + { + Name: secretName, + Namespace: instance.Namespace, + Type: util.TemplateTypeNone, + CustomData: fernetKeys, + Labels: labels, + }, + } + + err = oko_secret.EnsureSecrets(ctx, helper, instance, tmpl, envVars) + if err != nil { + return err + } return nil } diff --git a/pkg/keystone/bootstrap.go b/pkg/keystone/bootstrap.go index 99c3ed08..1471770d 100644 --- a/pkg/keystone/bootstrap.go +++ b/pkg/keystone/bootstrap.go @@ -60,12 +60,12 @@ func BootstrapJob( } // create Volume and VolumeMounts - volumes := getVolumes(instance.Name) + volumes := getVolumes(instance) volumeMounts := getVolumeMounts() // add CA cert if defined if instance.Spec.TLS.CaBundleSecretName != "" { - volumes = append(getVolumes(instance.Name), instance.Spec.TLS.CreateVolume()) + volumes = append(getVolumes(instance), instance.Spec.TLS.CreateVolume()) volumeMounts = append(getVolumeMounts(), instance.Spec.TLS.CreateVolumeMounts(nil)...) } diff --git a/pkg/keystone/cronjob.go b/pkg/keystone/cronjob.go index 2dbb8b98..573c6f9d 100644 --- a/pkg/keystone/cronjob.go +++ b/pkg/keystone/cronjob.go @@ -46,12 +46,12 @@ func CronJob( completions := int32(1) // create Volume and VolumeMounts - volumes := getVolumes(instance.Name) + volumes := getVolumes(instance) volumeMounts := getVolumeMounts() // add CA cert if defined if instance.Spec.TLS.CaBundleSecretName != "" { - volumes = append(getVolumes(instance.Name), instance.Spec.TLS.CreateVolume()) + volumes = append(getVolumes(instance), instance.Spec.TLS.CreateVolume()) volumeMounts = append(getVolumeMounts(), instance.Spec.TLS.CreateVolumeMounts(nil)...) } diff --git a/pkg/keystone/dbsync.go b/pkg/keystone/dbsync.go index ca23b92f..283f5ad0 100644 --- a/pkg/keystone/dbsync.go +++ b/pkg/keystone/dbsync.go @@ -45,12 +45,13 @@ func DbSyncJob( envVars["KOLLA_BOOTSTRAP"] = env.SetValue("true") // create Volume and VolumeMounts - volumes := getVolumes(instance.Name) + volumes := getVolumes(instance) volumeMounts := getVolumeMounts() // add CA cert if defined if instance.Spec.TLS.CaBundleSecretName != "" { - volumes = append(getVolumes(instance.Name), instance.Spec.TLS.CreateVolume()) + //TODO(afaranha): Why not reuse the 'volumes'? + volumes = append(getVolumes(instance), instance.Spec.TLS.CreateVolume()) volumeMounts = append(getVolumeMounts(), instance.Spec.TLS.CreateVolumeMounts(nil)...) } diff --git a/pkg/keystone/deployment.go b/pkg/keystone/deployment.go index 40e98532..8a02973a 100644 --- a/pkg/keystone/deployment.go +++ b/pkg/keystone/deployment.go @@ -80,7 +80,7 @@ func Deployment( envVars["CONFIG_HASH"] = env.SetValue(configHash) // create Volume and VolumeMounts - volumes := getVolumes(instance.Name) + volumes := getVolumes(instance) volumeMounts := getVolumeMounts() // add CA cert if defined diff --git a/pkg/keystone/fernet.go b/pkg/keystone/fernet.go index 635b77db..8e77c690 100644 --- a/pkg/keystone/fernet.go +++ b/pkg/keystone/fernet.go @@ -19,6 +19,13 @@ import ( "encoding/base64" "math/rand" + + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // GenerateFernetKey - @@ -29,3 +36,160 @@ func GenerateFernetKey() string { } return base64.StdEncoding.EncodeToString(data) } + +const ( + // FernetRotationCommand - + FernetRotationCommand = ` + echo $(date -u) Starting... + case $MAX_ACTIVE_KEYS in + ''|*[!0-9]*) + echo "MAX_ACTIVE_KEYS is not a number, exiting." + exit 1 + ;; + [01]) + echo "MAX_ACTIVE_KEYS ($MAX_ACTIVE_KEYS) -lt 2, exiting." + exit 1 + esac + + cd /var/lib/fernet-keys + mkdir /tmp/keys + for file in FernetKeys[0-9]*; + do + cat "$file" > /tmp/keys/"${file#FernetKeys}" + done + + cd /tmp/keys + + number_of_keys=$(ls -1 | wc -l) + max_key=$(ls -1 | sort -n | tail -1) + + if [ $((number_of_keys - 1)) != $max_key ]; then + echo "Corrupted FernetKeys secret, exiting." + exit 1 + fi + + mv 0 $((max_key + 1)) + dd if=/dev/urandom bs=32 count=1 2>/dev/null | base64 > 0 + + while [ -f "$MAX_ACTIVE_KEYS" ]; do + i=2 + while [ -f "$i" ]; do + mv $i $((i-1)) + i=$((i+1)) + done + done + + echo '{"stringData": {' > /tmp/patch_file.json + i=0 + while [ -f "$((i+1))" ]; do + echo '"FernetKeys'$i'": "'$(cat $i)'",' >> /tmp/patch_file.json + i=$((i+1)) + done + echo '"FernetKeys'$i'": "'$(cat $i)'"' >> /tmp/patch_file.json + echo '}}' >> /tmp/patch_file.json + + kubectl patch secret -n $NAMESPACE $SECRET_NAME \ + --patch-file=/tmp/patch_file.json + echo $(date -u) $((i+1)) keys rotated. + + cd /var/lib/fernet-keys + if [ -f "FernetKeys$MAX_ACTIVE_KEYS" ]; then + echo '[' > /tmp/patch_file.json + i=$((MAX_ACTIVE_KEYS-1)) + while [ -f "FernetKeys$((i+1))" ]; do + echo '{"op": "remove", "path": "/data/FernetKeys'$i'"},' \ + >> /tmp/patch_file.json + i=$((i+1)) + done + echo '{"op": "remove", "path": "/data/FernetKeys'$i'"}' \ + >> /tmp/patch_file.json + echo ']' >> /tmp/patch_file.json + + kubectl patch secret -n $NAMESPACE $SECRET_NAME \ + --type=json --patch-file=/tmp/patch_file.json + echo $(date -u) $MAX_ACTIVE_KEYS through $i keys deleted. + fi +` +) + +// FernetCronJob func +func FernetCronJob( + keystoneapiinstance *keystonev1.KeystoneAPI, + labels map[string]string, + annotations map[string]string, +) *batchv1.CronJob { + instance := &keystonev1.KeystoneAPIFernet{KeystoneAPI: keystoneapiinstance} + runAsUser := int64(0) + suspend := false + successfulJobsHistoryLimit := int32(3) + failedJobsHistoryLimit := int32(1) + + args := []string{"-c", FernetRotationCommand} + + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["SECRET_NAME"] = env.SetValue(ServiceName) + envVars["MAX_ACTIVE_KEYS"] = env.SetValue( + instance.Spec.FernetMaxActiveKeys) + + backoffLimit := int32(0) + parallelism := int32(1) + completions := int32(1) + + // create Volume and VolumeMounts + //volumes := getVolumes(instance) + volumes := getVolumes(keystoneapiinstance) + volumeMounts := getVolumeMounts() + + cronjob := &batchv1.CronJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: ServiceName + "-fernet-cronjob", + Namespace: instance.Namespace, + }, + Spec: batchv1.CronJobSpec{ + Schedule: instance.Spec.FernetRotationSchedule, + Suspend: &suspend, + ConcurrencyPolicy: batchv1.ForbidConcurrent, + SuccessfulJobsHistoryLimit: &successfulJobsHistoryLimit, + FailedJobsHistoryLimit: &failedJobsHistoryLimit, + JobTemplate: batchv1.JobTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + BackoffLimit: &backoffLimit, + Parallelism: ¶llelism, + Completions: &completions, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: ServiceName + "-fernet-job", + Image: instance.Spec.FernetRotationContainerImage, + Command: []string{ + "/bin/bash", + }, + Args: args, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + }, + }, + Volumes: volumes, + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: instance.RbacResourceName(), + }, + }, + }, + }, + }, + } + if instance.Spec.NodeSelector != nil && len(instance.Spec.NodeSelector) > 0 { + cronjob.Spec.JobTemplate.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector + } + + return cronjob +} diff --git a/pkg/keystone/volumes.go b/pkg/keystone/volumes.go index 3055a62d..00ae3092 100644 --- a/pkg/keystone/volumes.go +++ b/pkg/keystone/volumes.go @@ -16,14 +16,35 @@ limitations under the License. package keystone import ( + "fmt" + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" corev1 "k8s.io/api/core/v1" ) // getVolumes - service volumes -func getVolumes(name string) []corev1.Volume { +func getVolumes(keystoneapiinstance *keystonev1.KeystoneAPI) []corev1.Volume { + //func getVolumes(instance *keystonev1.KeystoneAPIFernet) []corev1.Volume { + name := keystoneapiinstance.Name var scriptsVolumeDefaultMode int32 = 0755 var config0640AccessMode int32 = 0640 + instance := &keystonev1.KeystoneAPIFernet{KeystoneAPI: keystoneapiinstance} + + fernetKeys := []corev1.KeyToPath{} + + var numberKeys int + fmt.Sscan(instance.Spec.FernetMaxActiveKeys, &numberKeys) + + for i := 0; i < numberKeys; i++ { + fernetKeys = append( + fernetKeys, + corev1.KeyToPath{ + Key: fmt.Sprintf("FernetKeys%d", i), + Path: fmt.Sprintf("%d", i), + }, + ) + } + return []corev1.Volume{ { Name: "scripts", @@ -48,16 +69,7 @@ func getVolumes(name string) []corev1.Volume { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: ServiceName, - Items: []corev1.KeyToPath{ - { - Key: "FernetKeys0", - Path: "0", - }, - { - Key: "FernetKeys1", - Path: "1", - }, - }, + Items: fernetKeys, }, }, }, diff --git a/test_keystone.yaml b/test_keystone.yaml new file mode 100644 index 00000000..d4f1ee62 --- /dev/null +++ b/test_keystone.yaml @@ -0,0 +1,177 @@ +apiVersion: v1 +items: +- apiVersion: keystone.openstack.org/v1beta1 + kind: KeystoneAPI + metadata: + creationTimestamp: "2024-10-15T09:49:52Z" + finalizers: + - openstack.org/keystoneapi + - openstack.org/keystoneservice-nova + - openstack.org/keystoneservice-swift + - openstack.org/keystoneservice-glance + - openstack.org/keystoneservice-barbican + - openstack.org/keystoneendpoint-swift + - openstack.org/keystoneendpoint-glance-default-single + - openstack.org/keystoneservice-ceilometer + - openstack.org/keystoneendpoint-barbican-api + - openstack.org/keystoneservice-cinderv3 + - openstack.org/keystoneservice-placement + - openstack.org/keystoneendpoint-cinderv3 + - openstack.org/keystoneendpoint-placement + - openstack.org/keystoneservice-neutron + - openstack.org/keystoneendpoint-neutron + - openstack.org/keystoneendpoint-nova + generation: 2 + name: keystone + namespace: openstack + ownerReferences: + - apiVersion: core.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: OpenStackControlPlane + name: openstack-galera + uid: d9262aa4-b233-4aeb-b383-018983f62f33 + resourceVersion: "1273973" + uid: 743bf48e-5697-4f4c-9103-939c9ad47564 + spec: + adminProject: admin + adminUser: admin + containerImage: quay.io/podified-antelope-centos9/openstack-keystone@sha256:a71e5eaf974d4ecc8f3185fa5b76c8d21416d9dd1fac9c02b9f62bc39005d969 + databaseAccount: keystone + databaseInstance: openstack + enableSecureRBAC: true + fernetMaxActiveKeys: "6" + fernetRotationSchedule: 1 0 * * * + memcachedInstance: memcached + override: + service: + internal: + metadata: + labels: + osctlplane: "" + osctlplane-service: keystone + public: + endpointURL: https://keystone-public-openstack.apps-crc.testing + metadata: + labels: + osctlplane: "" + osctlplane-service: keystone + passwordSelectors: + admin: AdminPassword + preserveJobs: false + rabbitMqClusterName: rabbitmq + region: regionOne + replicas: 1 + resources: {} + secret: osp-secret + tls: + api: + internal: + secretName: cert-keystone-internal-svc + public: + secretName: cert-keystone-public-svc + caBundleSecretName: combined-ca-bundle + trustFlushArgs: "" + trustFlushSchedule: 1 * * * * + trustFlushSuspend: false + status: + apiEndpoints: + internal: https://keystone-internal.openstack.svc:5000 + public: https://keystone-public-openstack.apps-crc.testing + conditions: + - lastTransitionTime: "2024-10-16T10:44:12Z" + message: Setup complete + reason: Ready + status: "True" + type: Ready + - lastTransitionTime: "2024-10-16T10:44:12Z" + message: Bootstrap completed + reason: Ready + status: "True" + type: BootstrapReady + - lastTransitionTime: "2024-10-16T10:44:12Z" + message: CronJob completed + reason: Ready + status: "True" + type: CronJobReady + - lastTransitionTime: "2024-10-15T09:50:39Z" + message: DB create completed + reason: Ready + status: "True" + type: DBReady + - lastTransitionTime: "2024-10-16T10:44:12Z" + message: DBsync completed + reason: Ready + status: "True" + type: DBSyncReady + - lastTransitionTime: "2024-10-16T10:44:12Z" + message: Deployment completed + reason: Ready + status: "True" + type: DeploymentReady + - lastTransitionTime: "2024-10-16T10:44:12Z" + message: Exposing service completed + reason: Ready + status: "True" + type: ExposeServiceReady + - lastTransitionTime: "2024-10-15T09:49:52Z" + message: Input data complete + reason: Ready + status: "True" + type: InputReady + - lastTransitionTime: "2024-10-15T09:49:52Z" + message: MariaDBAccount creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady + - lastTransitionTime: "2024-10-15T09:50:53Z" + message: ' Memcached instance has been provisioned' + reason: Ready + status: "True" + type: MemcachedReady + - lastTransitionTime: "2024-10-16T10:44:12Z" + message: NetworkAttachments completed + reason: Ready + status: "True" + type: NetworkAttachmentsReady + - lastTransitionTime: "2024-10-15T09:50:53Z" + message: RabbitMqTransportURL successfully created + reason: Ready + status: "True" + type: RabbitMqTransportURLReady + - lastTransitionTime: "2024-10-16T09:20:03Z" + message: RoleBinding created + reason: Ready + status: "True" + type: RoleBindingReady + - lastTransitionTime: "2024-10-16T09:20:03Z" + message: Role created + reason: Ready + status: "True" + type: RoleReady + - lastTransitionTime: "2024-10-16T09:20:03Z" + message: ServiceAccount created + reason: Ready + status: "True" + type: ServiceAccountReady + - lastTransitionTime: "2024-10-16T09:20:03Z" + message: Service config create completed + reason: Ready + status: "True" + type: ServiceConfigReady + - lastTransitionTime: "2024-10-15T09:50:53Z" + message: Input data complete + reason: Ready + status: "True" + type: TLSInputReady + databaseHostname: openstack.openstack.svc + hash: + bootstrap: n5cfh9ch5f5h59dh599h65bh654h667h676h5ffh588h576h4h89h669h655h678h97h569h5d6h58h5b8h56dh5h544h5fch87hd7hdch694h5d9hbq + dbsync: n546hcch5bbh5b5h646h59fhb5h5d8h54chd9hfh5d7h7ch9fh5cfh5dch5c9h649h694h576h65h547h588h66fh88h66ch56dh54dhc9h698h696h66bq + input: n79h99hb4hbfh55dh8bh54ch554h9bh56ch558h5ch649h696h586h689h74h95h57fh8bhbbhd6h689h66h648h57fh575hfdh547hb7h664h5dfq + observedGeneration: 2 + readyCount: 1 + transportURLSecret: rabbitmq-transport-url-keystone-keystone-transport +kind: List +metadata: + resourceVersion: "" diff --git a/tests/functional/keystoneapi_webhook_test.go b/tests/functional/keystoneapi_webhook_test.go index d648592e..07f2d4a9 100644 --- a/tests/functional/keystoneapi_webhook_test.go +++ b/tests/functional/keystoneapi_webhook_test.go @@ -94,6 +94,9 @@ var _ = Describe("KeystoneAPI Webhook", func() { Expect(KeystoneAPI.Spec.ContainerImage).Should(Equal( keystonev1.KeystoneAPIContainerImage, )) + Expect(KeystoneAPI.Spec.FernetRotationContainerImage).Should(Equal( + keystonev1.KeystoneFernetRotationContainerImage, + )) }) }) @@ -101,6 +104,7 @@ var _ = Describe("KeystoneAPI Webhook", func() { BeforeEach(func() { spec := GetDefaultKeystoneAPISpec() spec["containerImage"] = "api-container-image" + spec["fernetRotationContainerImage"] = "fernet-rotation-container-image" DeferCleanup(th.DeleteInstance, CreateKeystoneAPI(keystoneAPIName, spec)) }) @@ -109,6 +113,9 @@ var _ = Describe("KeystoneAPI Webhook", func() { Expect(KeystoneAPI.Spec.ContainerImage).Should(Equal( "api-container-image", )) + Expect(KeystoneAPI.Spec.FernetRotationContainerImage).Should(Equal( + "fernet-rotation-container-image", + )) }) })