[tlse] TLS database connection

The my.cnf file gets added to the secret holding the service configs.
The content of my.cnf is centrally managed in the mariadb-operator
and retrieved calling db.GetDatabaseClientConfig(tlsCfg)

Depends-On: openstack-k8s-operators/mariadb-operator#190
Depends-On: openstack-k8s-operators/mariadb-operator#191

Jira: OSPRH-4547

Author: stuggi

controllers/cinder_controller.go

@@ -51,6 +51,7 @@ import (
 	common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/secret"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/service"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/tls"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/util"
 	mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
 
@@ -380,63 +381,6 @@ func (r *CinderReconciler) reconcileInit(
 
 	Log.Info(fmt.Sprintf("Reconciling Service '%s' init", instance.Name))
 
-	//
-	// create service DB instance
-	//
-	db := mariadbv1.NewDatabase(
-		instance.Name,
-		instance.Spec.DatabaseUser,
-		instance.Spec.Secret,
-		map[string]string{
-			"dbName": instance.Spec.DatabaseInstance,
-		},
-	)
-	// create or patch the DB
-	ctrlResult, err := db.CreateOrPatchDB(
-		ctx,
-		helper,
-	)
-	if err != nil {
-		instance.Status.Conditions.Set(condition.FalseCondition(
-			condition.DBReadyCondition,
-			condition.ErrorReason,
-			condition.SeverityWarning,
-			condition.DBReadyErrorMessage,
-			err.Error()))
-		return ctrl.Result{}, err
-	}
-	if (ctrlResult != ctrl.Result{}) {
-		instance.Status.Conditions.Set(condition.FalseCondition(
-			condition.DBReadyCondition,
-			condition.RequestedReason,
-			condition.SeverityInfo,
-			condition.DBReadyRunningMessage))
-		return ctrlResult, nil
-	}
-	// wait for the DB to be setup
-	ctrlResult, err = db.WaitForDBCreated(ctx, helper)
-	if err != nil {
-		instance.Status.Conditions.Set(condition.FalseCondition(
-			condition.DBReadyCondition,
-			condition.ErrorReason,
-			condition.SeverityWarning,
-			condition.DBReadyErrorMessage,
-			err.Error()))
-		return ctrlResult, err
-	}
-	if (ctrlResult != ctrl.Result{}) {
-		instance.Status.Conditions.Set(condition.FalseCondition(
-			condition.DBReadyCondition,
-			condition.RequestedReason,
-			condition.SeverityInfo,
-			condition.DBReadyRunningMessage))
-		return ctrlResult, nil
-	}
-	// update Status.DatabaseHostname, used to config the service
-	instance.Status.DatabaseHostname = db.GetDatabaseHostname()
-	instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage)
-	// create service DB - end
-
 	//
 	// run Cinder db sync
 	//
@@ -450,7 +394,7 @@ func (r *CinderReconciler) reconcileInit(
 		time.Duration(5)*time.Second,
 		dbSyncHash,
 	)
-	ctrlResult, err = dbSyncjob.DoJob(
+	ctrlResult, err := dbSyncjob.DoJob(
 		ctx,
 		helper,
 	)
@@ -615,10 +559,17 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder
 	instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage)
 	// run check OpenStack secret - end
 
+	db, result, err := r.ensureDB(ctx, helper, instance)
+	if err != nil {
+		return ctrl.Result{}, err
+	} else if (result != ctrl.Result{}) {
+		return result, nil
+	}
+
 	//
 	// Create Secrets required as input for the Service and calculate an overall hash of hashes
 	//
-	err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels, memcached)
+	err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels, memcached, db)
 	if err != nil {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.ServiceConfigReadyCondition,
@@ -911,6 +862,7 @@ func (r *CinderReconciler) generateServiceConfigs(
 	envVars *map[string]env.Setter,
 	serviceLabels map[string]string,
 	memcached *memcachedv1.Memcached,
+	db *mariadbv1.Database,
 ) error {
 	//
 	// create Secret required for cinder input
@@ -920,8 +872,20 @@ func (r *CinderReconciler) generateServiceConfigs(
 
 	labels := labels.GetLabels(instance, labels.GetGroupLabel(cinder.ServiceName), serviceLabels)
 
+	db, err := mariadbv1.GetDatabaseByName(ctx, h, cinder.DatabaseName)
+	if err != nil {
+		return err
+	}
+
+	var tlsCfg *tls.Service
+	if instance.Spec.CinderAPI.TLS.Ca.CaBundleSecretName != "" {
+		tlsCfg = &tls.Service{}
+	}
 	// customData hold any customization for all cinder services.
-	customData := map[string]string{cinder.CustomConfigFileName: instance.Spec.CustomServiceConfig}
+	customData := map[string]string{
+		cinder.CustomConfigFileName: instance.Spec.CustomServiceConfig,
+		cinder.MyCnfFileName:        db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf
+	}
 
 	keystoneAPI, err := keystonev1.GetKeystoneAPI(ctx, h, instance.Namespace, map[string]string{})
 	if err != nil {
@@ -952,7 +916,7 @@ func (r *CinderReconciler) generateServiceConfigs(
 	templateParameters["KeystoneInternalURL"] = keystoneInternalURL
 	templateParameters["KeystonePublicURL"] = keystonePublicURL
 	templateParameters["TransportURL"] = string(transportURLSecret.Data["transport_url"])
-	templateParameters["DatabaseConnection"] = fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s",
+	templateParameters["DatabaseConnection"] = fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf",
 		instance.Spec.DatabaseUser,
 		string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]),
 		instance.Status.DatabaseHostname,
@@ -1272,3 +1236,70 @@ func (r *CinderReconciler) volumeCleanupDeployments(ctx context.Context, instanc
 
 	return nil
 }
+
+func (r *CinderReconciler) ensureDB(
+	ctx context.Context,
+	h *helper.Helper,
+	instance *cinderv1beta1.Cinder,
+) (*mariadbv1.Database, ctrl.Result, error) {
+	//
+	// create service DB instance
+	//
+	db := mariadbv1.NewDatabase(
+		instance.Name,
+		instance.Spec.DatabaseUser,
+		instance.Spec.Secret,
+		map[string]string{
+			"dbName": instance.Spec.DatabaseInstance,
+		},
+	)
+
+	// create or patch the DB
+	ctrlResult, err := db.CreateOrPatchDBByName(
+		ctx,
+		h,
+		instance.Spec.DatabaseInstance,
+	)
+	if err != nil {
+		instance.Status.Conditions.Set(condition.FalseCondition(
+			condition.DBReadyCondition,
+			condition.ErrorReason,
+			condition.SeverityWarning,
+			condition.DBReadyErrorMessage,
+			err.Error()))
+		return db, ctrl.Result{}, err
+	}
+	if (ctrlResult != ctrl.Result{}) {
+		instance.Status.Conditions.Set(condition.FalseCondition(
+			condition.DBReadyCondition,
+			condition.RequestedReason,
+			condition.SeverityInfo,
+			condition.DBReadyRunningMessage))
+		return db, ctrlResult, nil
+	}
+	// wait for the DB to be setup
+	// (ksambor) should we use WaitForDBCreatedWithTimeout instead?
+	ctrlResult, err = db.WaitForDBCreated(ctx, h)
+	if err != nil {
+		instance.Status.Conditions.Set(condition.FalseCondition(
+			condition.DBReadyCondition,
+			condition.ErrorReason,
+			condition.SeverityWarning,
+			condition.DBReadyErrorMessage,
+			err.Error()))
+		return db, ctrlResult, err
+	}
+	if (ctrlResult != ctrl.Result{}) {
+		instance.Status.Conditions.Set(condition.FalseCondition(
+			condition.DBReadyCondition,
+			condition.RequestedReason,
+			condition.SeverityInfo,
+			condition.DBReadyRunningMessage))
+		return db, ctrlResult, nil
+	}
+
+	// update Status.DatabaseHostname, used to config the service
+	instance.Status.DatabaseHostname = db.GetDatabaseHostname()
+	instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage)
+	return db, ctrlResult, nil
+}

pkg/cinder/const.go

@@ -41,6 +41,8 @@ const (
 	CustomServiceConfigFileName = "03-service-custom.conf"
 	// CustomServiceConfigSecretsFileName -
 	CustomServiceConfigSecretsFileName = "04-service-custom-secrets.conf"
+	// MyCnfFileName -
+	MyCnfFileName = "my.cnf"
 
 	// CinderPublicPort -
 	CinderPublicPort int32 = 8776

pkg/cinder/cronjob.go

@@ -17,6 +17,7 @@ package cinder
 
 import (
 	cinderv1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1"
+	cinderv1beta1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1"
 
 	"fmt"
 
@@ -75,6 +76,12 @@ func CronJob(
 			MountPath: "/etc/cinder/cinder.conf.d",
 			ReadOnly:  true,
 		},
+		{
+			Name:      "config-data",
+			MountPath: "/etc/my.cnf",
+			SubPath:   MyCnfFileName,
+			ReadOnly:  true,
+		},
 	}
 
 	// add CA cert if defined
@@ -83,6 +90,8 @@ func CronJob(
 		cronJobVolumeMounts = append(cronJobVolumeMounts, instance.Spec.CinderAPI.TLS.CreateVolumeMounts(nil)...)
 	}
 
+	cronJobExtraMounts := []cinderv1beta1.CinderExtraVolMounts{}
+
 	cronjob := &batchv1.CronJob{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        fmt.Sprintf("%s-db-purge", ServiceName),
@@ -122,7 +131,7 @@ func CronJob(
 									},
 								},
 							},
-							Volumes:            cronJobVolumes,
+							Volumes:            append(GetVolumes(instance.Name, false, cronJobExtraMounts, DbsyncPropagation), cronJobVolumes...),
 							RestartPolicy:      corev1.RestartPolicyNever,
 							ServiceAccountName: instance.RbacResourceName(),
 						},

pkg/cinder/volumes.go

@@ -172,6 +172,12 @@ func GetVolumeMounts(storageSvc bool, extraVol []cinderv1beta1.CinderExtraVolMou
 			MountPath: "/var/lib/config-data/merged",
 			ReadOnly:  true,
 		},
+		{
+			Name:      "config-data",
+			MountPath: "/etc/my.cnf",
+			SubPath:   MyCnfFileName,
+			ReadOnly:  true,
+		},
 	}
 
 	// Volume and backup services require extra directories

test/functional/cinder_controller_test.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/utils/ptr"
 
 	cinderv1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1"
+	"github.com/openstack-k8s-operators/cinder-operator/pkg/cinder"
 	memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1"
 	condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
 	util "github.com/openstack-k8s-operators/lib-common/modules/common/util"
@@ -218,13 +219,17 @@ var _ = Describe("Cinder controller", func() {
 			infra.SimulateTransportURLReady(cinderTest.CinderTransportURL)
 			DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(namespace, "memcached", memcachedSpec))
 			infra.SimulateMemcachedReady(cinderTest.CinderMemcached)
+			mariadb.SimulateMariaDBAccountCompleted(cinderTest.Instance)
+			mariadb.SimulateMariaDBDatabaseCompleted(cinderTest.Instance)
 		})
 		It("should create config-data and scripts ConfigMaps", func() {
 			keystoneAPI := keystone.CreateKeystoneAPI(cinderTest.Instance.Namespace)
 			DeferCleanup(keystone.DeleteKeystoneAPI, keystoneAPI)
-			Eventually(func() corev1.Secret {
-				return th.GetSecret(cinderTest.CinderConfigSecret)
-			}, timeout, interval).ShouldNot(BeNil())
+			cf := th.GetSecret(cinderTest.CinderConfigSecret)
+			Expect(cf).ShouldNot(BeNil())
+			conf := cf.Data[cinder.MyCnfFileName]
+			Expect(conf).To(
+				ContainSubstring("[client]\nssl=0"))
 			Eventually(func() corev1.Secret {
 				return th.GetSecret(cinderTest.CinderConfigScripts)
 			}, timeout, interval).ShouldNot(BeNil())
@@ -461,7 +466,7 @@ var _ = Describe("Cinder controller", func() {
 			infra.SimulateMemcachedReady(cinderTest.CinderMemcached)
 			DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(cinderTest.Instance.Namespace))
 			mariadb.SimulateMariaDBAccountCompleted(cinderTest.Instance)
-			mariadb.SimulateMariaDBDatabaseCompleted(cinderTest.Instance)
+			mariadb.SimulateMariaDBTLSDatabaseCompleted(cinderTest.Instance)
 			th.SimulateJobSuccess(cinderTest.CinderDBSync)
 		})
 
@@ -510,6 +515,19 @@ var _ = Describe("Cinder controller", func() {
 			)
 		})
 
+		It("should create config-data and scripts ConfigMaps", func() {
+			keystoneAPI := keystone.CreateKeystoneAPI(cinderTest.Instance.Namespace)
+			DeferCleanup(keystone.DeleteKeystoneAPI, keystoneAPI)
+			cf := th.GetSecret(cinderTest.CinderConfigSecret)
+			Expect(cf).ShouldNot(BeNil())
+			conf := cf.Data[cinder.MyCnfFileName]
+			Expect(conf).To(
+				ContainSubstring("[client]\nssl-ca=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem\nssl=1"))
+			Eventually(func() corev1.Secret {
+				return th.GetSecret(cinderTest.CinderConfigScripts)
+			}, timeout, interval).ShouldNot(BeNil())
+		})
+
 		It("Creates CinderAPI", func() {
 			DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(cinderTest.CABundleSecret))
 			DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(cinderTest.InternalCertSecret))

test/kuttl/common/assert_sample_deployment.yaml

@@ -121,6 +121,10 @@ spec:
         - mountPath: /var/lib/config-data/merged
           name: config-data
           readOnly: true
+        - mountPath: /etc/my.cnf
+          name: config-data
+          readOnly: true
+          subPath: my.cnf
         - mountPath: /etc/cinder/cinder.conf.d
           name: config-data-custom
           readOnly: true

test/kuttl/common/assert_tls_sample_deployment.yaml

@@ -83,6 +83,10 @@ spec:
         - mountPath: /var/lib/config-data/merged
           name: config-data
           readOnly: true
+        - mountPath: /etc/my.cnf
+          name: config-data
+          readOnly: true
+          subPath: my.cnf
         - mountPath: /etc/cinder/cinder.conf.d
           name: config-data-custom
           readOnly: true
@@ -175,6 +179,10 @@ spec:
         - mountPath: /var/lib/config-data/merged
           name: config-data
           readOnly: true
+        - mountPath: /etc/my.cnf
+          name: config-data
+          readOnly: true
+          subPath: my.cnf
         - mountPath: /etc/cinder/cinder.conf.d
           name: config-data-custom
           readOnly: true
@@ -203,6 +211,10 @@ spec:
         - mountPath: /var/lib/config-data/merged
           name: config-data
           readOnly: true
+        - mountPath: /etc/my.cnf
+          name: config-data
+          readOnly: true
+          subPath: my.cnf
         - mountPath: /etc/cinder/cinder.conf.d
           name: config-data-custom
           readOnly: true
@@ -267,6 +279,10 @@ spec:
         - mountPath: /var/lib/config-data/merged
           name: config-data
           readOnly: true
+        - mountPath: /etc/my.cnf
+          name: config-data
+          readOnly: true
+          subPath: my.cnf
         - mountPath: /var/lib/cinder
           name: var-lib-cinder
         - mountPath: /etc/nvme
@@ -315,6 +331,10 @@ spec:
         - mountPath: /var/lib/config-data/merged
           name: config-data
           readOnly: true
+        - mountPath: /etc/my.cnf
+          name: config-data
+          readOnly: true
+          subPath: my.cnf
         - mountPath: /var/lib/cinder
           name: var-lib-cinder
         - mountPath: /etc/nvme