Enable MTLS memcached auth

This commit allows operators to use mtls as an authentication
method against Memcached.
Nova controllers will detect the presence of a purposely-created mtls
certificate (authCertSecret) and use this to configure the
[keystone_authtoken] section accordingly.
Additional volumes/volumemounts will be appended to each pod.

Note that this commit switches from MemcachedServersWithInet to
MemcachedServers as keystone-middleware uses oslo.cache and as such
there is no need to use "[]" to enclose the list of memcached servers even for ipv6.

Author: lmiccini

controllers/novaapi_controller.go

@@ -333,7 +333,7 @@ func (r *NovaAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re
 		return result, err
 	}
 
-	result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations)
+	result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached)
 	if (err != nil || result != ctrl.Result{}) {
 		return result, err
 	}
@@ -539,6 +539,13 @@ func (r *NovaAPIReconciler) generateConfigs(
 		tlsCfg = &tls.Service{}
 	}
 
+	// MTLS
+	if memcachedInstance.GetMemcachedMTLSSecret() != "" {
+		templateParameters["MemcachedAuthCert"] = fmt.Sprint(memcachedv1.CertMountPath())
+		templateParameters["MemcachedAuthKey"] = fmt.Sprint(memcachedv1.KeyMountPath())
+		templateParameters["MemcachedAuthCa"] = fmt.Sprint(memcachedv1.CaMountPath())
+	}
+
 	extraData := map[string]string{
 		"my.cnf": apiDB.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf
 	}
@@ -567,6 +574,7 @@ func (r *NovaAPIReconciler) ensureDeployment(
 	instance *novav1.NovaAPI,
 	inputHash string,
 	annotations map[string]string,
+	memcached *memcachedv1.Memcached,
 ) (ctrl.Result, error) {
 	Log := r.GetLogger(ctx)
 	serviceLabels := getAPIServiceLabels()
@@ -586,7 +594,7 @@ func (r *NovaAPIReconciler) ensureDeployment(
 		return ctrl.Result{}, fmt.Errorf("waiting for Topology requirements: %w", err)
 	}
 
-	ssSpec, err := novaapi.StatefulSet(instance, inputHash, serviceLabels, annotations, topology)
+	ssSpec, err := novaapi.StatefulSet(instance, inputHash, serviceLabels, annotations, topology, memcached)
 	if err != nil {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.DeploymentReadyCondition,

controllers/novaconductor_controller.go

@@ -299,17 +299,17 @@ func (r *NovaConductorReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		return result, err
 	}
 
-	result, err = r.ensureCellDBSynced(ctx, h, instance, serviceAnnotations)
+	result, err = r.ensureCellDBSynced(ctx, h, instance, serviceAnnotations, memcached)
 	if (err != nil || result != ctrl.Result{}) {
 		return result, err
 	}
 
-	result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations)
+	result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached)
 	if (err != nil || result != ctrl.Result{}) {
 		return result, err
 	}
 
-	err = r.ensureDBPurgeCronJob(ctx, h, instance, serviceAnnotations)
+	err = r.ensureDBPurgeCronJob(ctx, h, instance, serviceAnnotations, memcached)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
@@ -486,6 +486,14 @@ func (r *NovaConductorReconciler) generateConfigs(
 	if instance.Spec.TLS.CaBundleSecretName != "" {
 		tlsCfg = &tls.Service{}
 	}
+
+	// MTLS
+	if memcachedInstance.GetMemcachedMTLSSecret() != "" {
+		templateParameters["MemcachedAuthCert"] = fmt.Sprint(memcachedv1.CertMountPath())
+		templateParameters["MemcachedAuthKey"] = fmt.Sprint(memcachedv1.KeyMountPath())
+		templateParameters["MemcachedAuthCa"] = fmt.Sprint(memcachedv1.CaMountPath())
+	}
+
 	extraData := map[string]string{
 		"my.cnf": cellDB.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf
 	}
@@ -506,13 +514,15 @@ func (r *NovaConductorReconciler) ensureCellDBSynced(
 	h *helper.Helper,
 	instance *novav1.NovaConductor,
 	annotations map[string]string,
+	memcached *memcachedv1.Memcached,
 ) (ctrl.Result, error) {
 	serviceLabels := map[string]string{
 		common.AppSelector: NovaConductorLabelPrefix,
 	}
 	Log := r.GetLogger(ctx)
 	dbSyncHash := instance.Status.Hash[DbSyncHash]
-	jobDef := novaconductor.CellDBSyncJob(instance, serviceLabels, annotations)
+
+	jobDef := novaconductor.CellDBSyncJob(instance, serviceLabels, annotations, memcached)
 	dbSyncJob := job.NewJob(jobDef, "dbsync", instance.Spec.PreserveJobs, r.RequeueTimeout, dbSyncHash)
 	ctrlResult, err := dbSyncJob.DoJob(ctx, h)
 	if (ctrlResult != ctrl.Result{}) {
@@ -547,6 +557,7 @@ func (r *NovaConductorReconciler) ensureDeployment(
 	instance *novav1.NovaConductor,
 	inputHash string,
 	annotations map[string]string,
+	memcached *memcachedv1.Memcached,
 ) (ctrl.Result, error) {
 	serviceLabels := map[string]string{
 		common.AppSelector: NovaConductorLabelPrefix,
@@ -569,7 +580,7 @@ func (r *NovaConductorReconciler) ensureDeployment(
 		return ctrl.Result{}, fmt.Errorf("waiting for Topology requirements: %w", err)
 	}
 
-	ss := statefulset.NewStatefulSet(novaconductor.StatefulSet(instance, inputHash, serviceLabels, annotations, topology), r.RequeueTimeout)
+	ss := statefulset.NewStatefulSet(novaconductor.StatefulSet(instance, inputHash, serviceLabels, annotations, topology, memcached), r.RequeueTimeout)
 	ctrlResult, err := ss.CreateOrPatch(ctx, h)
 	if err != nil && !k8s_errors.IsNotFound(err) {
 		Log.Error(err, "Deployment failed")
@@ -642,11 +653,13 @@ func (r *NovaConductorReconciler) ensureDBPurgeCronJob(
 	h *helper.Helper,
 	instance *novav1.NovaConductor,
 	annotations map[string]string,
+	memcached *memcachedv1.Memcached,
 ) error {
 	serviceLabels := map[string]string{
 		common.AppSelector: NovaConductorLabelPrefix,
 	}
-	cronDef := novaconductor.DBPurgeCronJob(instance, serviceLabels, annotations)
+
+	cronDef := novaconductor.DBPurgeCronJob(instance, serviceLabels, annotations, memcached)
 	cronjob := cronjob.NewCronJob(cronDef, r.RequeueTimeout)
 
 	_, err := cronjob.CreateOrPatch(ctx, h)

controllers/novametadata_controller.go

@@ -326,7 +326,7 @@ func (r *NovaMetadataReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return result, err
 	}
 
-	result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations)
+	result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached)
 	if (err != nil || result != ctrl.Result{}) {
 		return result, err
 	}
@@ -545,6 +545,14 @@ func (r *NovaMetadataReconciler) generateConfigs(
 	if instance.Spec.TLS.CaBundleSecretName != "" {
 		tlsCfg = &tls.Service{}
 	}
+
+	// MTLS
+	if memcachedInstance.GetMemcachedMTLSSecret() != "" {
+		templateParameters["MemcachedAuthCert"] = fmt.Sprint(memcachedv1.CertMountPath())
+		templateParameters["MemcachedAuthKey"] = fmt.Sprint(memcachedv1.KeyMountPath())
+		templateParameters["MemcachedAuthCa"] = fmt.Sprint(memcachedv1.CaMountPath())
+	}
+
 	extraData := map[string]string{
 		"my.cnf": db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf
 	}
@@ -572,6 +580,7 @@ func (r *NovaMetadataReconciler) ensureDeployment(
 	instance *novav1.NovaMetadata,
 	inputHash string,
 	annotations map[string]string,
+	memcached *memcachedv1.Memcached,
 ) (ctrl.Result, error) {
 	Log := r.GetLogger(ctx)
 	serviceLabels := getMetadataServiceLabels(instance.Spec.CellName)
@@ -591,7 +600,7 @@ func (r *NovaMetadataReconciler) ensureDeployment(
 		return ctrl.Result{}, fmt.Errorf("waiting for Topology requirements: %w", err)
 	}
 
-	ssSpec, err := novametadata.StatefulSet(instance, inputHash, serviceLabels, annotations, topology)
+	ssSpec, err := novametadata.StatefulSet(instance, inputHash, serviceLabels, annotations, topology, memcached)
 	if err != nil {
 		Log.Error(err, "Deployment failed")
 		instance.Status.Conditions.Set(condition.FalseCondition(

controllers/novanovncproxy_controller.go

@@ -345,7 +345,7 @@ func (r *NovaNoVNCProxyReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 		return result, err
 	}
 
-	result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations)
+	result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached)
 	if (err != nil || result != ctrl.Result{}) {
 		return result, err
 	}
@@ -499,6 +499,14 @@ func (r *NovaNoVNCProxyReconciler) generateConfigs(
 	if instance.Spec.TLS.CaBundleSecretName != "" {
 		tlsCfg = &tls.Service{}
 	}
+
+	// MTLS
+	if memcachedInstance.GetMemcachedMTLSSecret() != "" {
+		templateParameters["MemcachedAuthCert"] = fmt.Sprint(memcachedv1.CertMountPath())
+		templateParameters["MemcachedAuthKey"] = fmt.Sprint(memcachedv1.KeyMountPath())
+		templateParameters["MemcachedAuthCa"] = fmt.Sprint(memcachedv1.CaMountPath())
+	}
+
 	extraData := map[string]string{
 		"my.cnf": cellDB.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf
 	}
@@ -523,6 +531,7 @@ func (r *NovaNoVNCProxyReconciler) ensureDeployment(
 	instance *novav1.NovaNoVNCProxy,
 	inputHash string,
 	annotations map[string]string,
+	memcached *memcachedv1.Memcached,
 ) (ctrl.Result, error) {
 	Log := r.GetLogger(ctx)
 	serviceLabels := getNoVNCProxyServiceLabels(instance.Spec.CellName)
@@ -542,7 +551,7 @@ func (r *NovaNoVNCProxyReconciler) ensureDeployment(
 		return ctrl.Result{}, fmt.Errorf("waiting for Topology requirements: %w", err)
 	}
 
-	ssSpec, err := novncproxy.StatefulSet(instance, inputHash, serviceLabels, annotations, topology)
+	ssSpec, err := novncproxy.StatefulSet(instance, inputHash, serviceLabels, annotations, topology, memcached)
 	if err != nil {
 		Log.Info("Deployment failed")
 		instance.Status.Conditions.Set(condition.FalseCondition(

controllers/novascheduler_controller.go

@@ -309,7 +309,7 @@ func (r *NovaSchedulerReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		return result, err
 	}
 
-	result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations)
+	result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached)
 	if (err != nil || result != ctrl.Result{}) {
 		return result, err
 	}
@@ -605,6 +605,14 @@ func (r *NovaSchedulerReconciler) generateConfigs(
 	if instance.Spec.TLS.CaBundleSecretName != "" {
 		tlsCfg = &tls.Service{}
 	}
+
+	// MTLS
+	if memcachedInstance.GetMemcachedMTLSSecret() != "" {
+		templateParameters["MemcachedAuthCert"] = fmt.Sprint(memcachedv1.CertMountPath())
+		templateParameters["MemcachedAuthKey"] = fmt.Sprint(memcachedv1.KeyMountPath())
+		templateParameters["MemcachedAuthCa"] = fmt.Sprint(memcachedv1.CaMountPath())
+	}
+
 	extraData := map[string]string{
 		"my.cnf": apiDB.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf
 	}
@@ -656,6 +664,7 @@ func (r *NovaSchedulerReconciler) ensureDeployment(
 	instance *novav1.NovaScheduler,
 	inputHash string,
 	annotations map[string]string,
+	memcached *memcachedv1.Memcached,
 ) (ctrl.Result, error) {
 	serviceLabels := map[string]string{
 		common.AppSelector: NovaSchedulerLabelPrefix,
@@ -677,7 +686,7 @@ func (r *NovaSchedulerReconciler) ensureDeployment(
 		return ctrl.Result{}, fmt.Errorf("waiting for Topology requirements: %w", err)
 	}
 
-	ss := statefulset.NewStatefulSet(novascheduler.StatefulSet(instance, inputHash, serviceLabels, annotations, topology), r.RequeueTimeout)
+	ss := statefulset.NewStatefulSet(novascheduler.StatefulSet(instance, inputHash, serviceLabels, annotations, topology, memcached), r.RequeueTimeout)
 	ctrlResult, err := ss.CreateOrPatch(ctx, h)
 	if err != nil && !k8s_errors.IsNotFound(err) {
 		Log.Error(err, "Deployment failed")

pkg/novaapi/deployment.go

@@ -17,6 +17,7 @@ limitations under the License.
 package novaapi
 
 import (
+	memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1"
 	topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1"
 	common "github.com/openstack-k8s-operators/lib-common/modules/common"
 	affinity "github.com/openstack-k8s-operators/lib-common/modules/common/affinity"
@@ -40,6 +41,7 @@ func StatefulSet(
 	labels map[string]string,
 	annotations map[string]string,
 	topology *topologyv1.Topology,
+	memcached *memcachedv1.Memcached,
 ) (*appsv1.StatefulSet, error) {
 	// This allows the pod to start up slowly. The pod will only be killed
 	// if it does not succeed a probe in 60 seconds.
@@ -102,6 +104,12 @@ func StatefulSet(
 		volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 	}
 
+	// add MTLS cert if defined
+	if memcached.Status.MTLSCert != "" {
+		volumes = append(volumes, memcached.CreateMTLSVolume())
+		volumeMounts = append(volumeMounts, memcached.CreateMTLSVolumeMounts(nil, nil)...)
+	}
+
 	for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} {
 		if instance.Spec.TLS.API.Enabled(endpt) {
 			var tlsEndptCfg tls.GenericService

pkg/novaconductor/dbpurge.go

@@ -9,6 +9,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/ptr"
 
+	memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/env"
 	novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1"
 	"github.com/openstack-k8s-operators/nova-operator/pkg/nova"
@@ -18,6 +19,7 @@ func DBPurgeCronJob(
 	instance *novav1.NovaConductor,
 	labels map[string]string,
 	annotations map[string]string,
+	memcached *memcachedv1.Memcached,
 ) *batchv1.CronJob {
 	args := []string{"-c", nova.KollaServiceCommand}
 
@@ -46,6 +48,12 @@ func DBPurgeCronJob(
 		volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 	}
 
+	// add MTLS cert if defined
+	if memcached.Status.MTLSCert != "" {
+		volumes = append(volumes, memcached.CreateMTLSVolume())
+		volumeMounts = append(volumeMounts, memcached.CreateMTLSVolumeMounts(nil, nil)...)
+	}
+
 	// we want to hide the fact that the job is created by the conductor
 	// controller, but we don't have direct access to the Cell CR name, so we
 	// remove the known conductor suffix from the Conductor CR name.

pkg/novaconductor/dbsync.go

@@ -22,6 +22,7 @@ import (
 
 	env "github.com/openstack-k8s-operators/lib-common/modules/common/env"
 
+	memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -33,6 +34,7 @@ func CellDBSyncJob(
 	instance *novav1.NovaConductor,
 	labels map[string]string,
 	annotations map[string]string,
+	memcached *memcachedv1.Memcached,
 ) *batchv1.Job {
 	args := []string{"-c", nova.KollaServiceCommand}
 
@@ -61,6 +63,12 @@ func CellDBSyncJob(
 		volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 	}
 
+	// add MTLS cert if defined
+	if memcached.Status.MTLSCert != "" {
+		volumes = append(volumes, memcached.CreateMTLSVolume())
+		volumeMounts = append(volumeMounts, memcached.CreateMTLSVolumeMounts(nil, nil)...)
+	}
+
 	job := &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        instance.Name + "-db-sync",

pkg/novaconductor/deployment.go

@@ -17,6 +17,7 @@ limitations under the License.
 package novaconductor
 
 import (
+	memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1"
 	topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1"
 	common "github.com/openstack-k8s-operators/lib-common/modules/common"
 	affinity "github.com/openstack-k8s-operators/lib-common/modules/common/affinity"
@@ -37,6 +38,7 @@ func StatefulSet(
 	labels map[string]string,
 	annotations map[string]string,
 	topology *topologyv1.Topology,
+	memcached *memcachedv1.Memcached,
 ) *appsv1.StatefulSet {
 	livenessProbe := &corev1.Probe{
 		// TODO might need tuning
@@ -93,6 +95,12 @@ func StatefulSet(
 		volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 	}
 
+	// add MTLS cert if defined
+	if memcached.Status.MTLSCert != "" {
+		volumes = append(volumes, memcached.CreateMTLSVolume())
+		volumeMounts = append(volumeMounts, memcached.CreateMTLSVolumeMounts(nil, nil)...)
+	}
+
 	statefulset := &appsv1.StatefulSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      instance.Name,

pkg/novametadata/deployment.go

@@ -17,6 +17,7 @@ limitations under the License.
 package novametadata
 
 import (
+	memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1"
 	topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1"
 	common "github.com/openstack-k8s-operators/lib-common/modules/common"
 	affinity "github.com/openstack-k8s-operators/lib-common/modules/common/affinity"
@@ -38,6 +39,7 @@ func StatefulSet(
 	labels map[string]string,
 	annotations map[string]string,
 	topology *topologyv1.Topology,
+	memcached *memcachedv1.Memcached,
 ) (*appsv1.StatefulSet, error) {
 	// This allows the pod to start up slowly. The pod will only be killed
 	// if it does not succeed a probe in 60 seconds.
@@ -100,6 +102,12 @@ func StatefulSet(
 		volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...)
 	}
 
+	// add MTLS cert if defined
+	if memcached.Status.MTLSCert != "" {
+		volumes = append(volumes, memcached.CreateMTLSVolume())
+		volumeMounts = append(volumeMounts, memcached.CreateMTLSVolumeMounts(nil, nil)...)
+	}
+
 	if instance.Spec.TLS.GenericService.Enabled() {
 		svc, err := instance.Spec.TLS.GenericService.ToService()
 		if err != nil {