Skip to content

Commit 095609b

Browse files
olliewalsholliewalsh
committed
Configure rabbitmq inter-node TLS
Based on the example in https://github.com/rabbitmq/cluster-operator/tree/0bc9a2a0d3b360021f02bd6386bbcee371c9d1fd/docs/examples/mtls-inter-node Creates and mounts a configmap with the required config and sets the required args in the env vars. Had to list the individual statefulset hostnames in the cert SAN, rabbitmqctl does not work when wildcards are used. Closes: OSPRH-7110
1 parent 9351c6f commit 095609b

File tree

3 files changed

+170
-19
lines changed

3 files changed

+170
-19
lines changed

‎pkg/openstack/rabbitmq.go‎

Lines changed: 88 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
networkv1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1"
1010
"github.com/openstack-k8s-operators/lib-common/modules/certmanager"
1111
condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
12+
"github.com/openstack-k8s-operators/lib-common/modules/common/configmap"
1213
"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
1314
"github.com/openstack-k8s-operators/lib-common/modules/common/ocp"
1415
"github.com/openstack-k8s-operators/lib-common/modules/common/util"
@@ -22,6 +23,7 @@ import (
2223

2324
corev1beta1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1"
2425
corev1 "k8s.io/api/core/v1"
26+
"k8s.io/utils/ptr"
2527

2628
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2729
ctrl "sigs.k8s.io/controller-runtime"
@@ -153,17 +155,56 @@ func reconcileRabbitMQ(
153155
if err != nil {
154156
return mqFailed, ctrl.Result{}, err
155157
}
158+
clusterNodeTLSArgs := "-proto_dist inet_tls -ssl_dist_optfile /etc/rabbitmq/inter-node-tls.config"
156159
if fipsEnabled {
157-
fipsModeStr := "-crypto fips_mode true"
158-
159-
envVars = append(envVars, corev1.EnvVar{
160-
Name: "RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS",
161-
Value: fipsModeStr,
162-
}, corev1.EnvVar{
163-
Name: "RABBITMQ_CTL_ERL_ARGS",
164-
Value: fipsModeStr,
165-
})
160+
clusterNodeTLSArgs += " -crypto fips_mode true"
166161
}
162+
163+
envVars = append(envVars, corev1.EnvVar{
164+
Name: "RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS",
165+
Value: clusterNodeTLSArgs,
166+
}, corev1.EnvVar{
167+
Name: "RABBITMQ_CTL_ERL_ARGS",
168+
Value: clusterNodeTLSArgs,
169+
})
170+
}
171+
172+
cms := []util.Template{
173+
{
174+
Name: fmt.Sprintf("%s-config-data", rabbitmq.Name),
175+
Namespace: rabbitmq.Namespace,
176+
Type: util.TemplateTypeConfig,
177+
InstanceType: "rabbitmq",
178+
Labels: map[string]string{},
179+
CustomData: map[string]string{
180+
"inter_node_tls.config": `[
181+
{server, [
182+
{cacertfile,"/etc/rabbitmq-tls/ca.crt"},
183+
{certfile,"/etc/rabbitmq-tls/tls.crt"},
184+
{keyfile,"/etc/rabbitmq-tls/tls.key"},
185+
{secure_renegotiate, true},
186+
{fail_if_no_peer_cert, true},
187+
{verify, verify_peer},
188+
{versions, ['tlsv1.2','tlsv1.3']}
189+
]},
190+
{client, [
191+
{cacertfile,"/etc/rabbitmq-tls/ca.crt"},
192+
{certfile,"/etc/rabbitmq-tls/tls.crt"},
193+
{keyfile,"/etc/rabbitmq-tls/tls.key"},
194+
{secure_renegotiate, true},
195+
{verify, verify_peer},
196+
{versions, ['tlsv1.2','tlsv1.3']}
197+
]}
198+
].
199+
`,
200+
},
201+
},
202+
}
203+
204+
err := configmap.EnsureConfigMaps(ctx, helper, instance, cms, nil)
205+
if err != nil {
206+
Log.Error(err, "Unable to create rabbitmq config maps")
207+
return mqFailed, ctrl.Result{}, err
167208
}
168209

169210
defaultStatefulSet := rabbitmqv2.StatefulSet{
@@ -196,6 +237,15 @@ func reconcileRabbitMQ(
196237

197238
hostname := fmt.Sprintf("%s.%s.svc", name, instance.Namespace)
198239
hostnameHeadless := fmt.Sprintf("%s-nodes.%s.svc", name, instance.Namespace)
240+
hostnames := []string{
241+
hostname,
242+
fmt.Sprintf("%s.%s", hostname, ClusterInternalDomain),
243+
hostnameHeadless,
244+
fmt.Sprintf("%s.%s", hostnameHeadless, ClusterInternalDomain),
245+
}
246+
for i := 0; i < int(*spec.Replicas); i++ {
247+
hostnames = append(hostnames, fmt.Sprintf("%s-server-%d.%s-nodes.%s", name, i, name, instance.Namespace))
248+
}
199249

200250
tlsCert := ""
201251
commonName := fmt.Sprintf("%s.%s", hostname, ClusterInternalDomain)
@@ -205,14 +255,7 @@ func reconcileRabbitMQ(
205255
IssuerName: instance.GetInternalIssuer(),
206256
CertName: fmt.Sprintf("%s-svc", rabbitmq.Name),
207257
CommonName: &commonName,
208-
Hostnames: []string{
209-
hostname,
210-
fmt.Sprintf("%s.%s", hostname, ClusterInternalDomain),
211-
hostnameHeadless,
212-
fmt.Sprintf("%s.%s", hostnameHeadless, ClusterInternalDomain),
213-
fmt.Sprintf("*.%s", hostnameHeadless),
214-
fmt.Sprintf("*.%s.%s", hostnameHeadless, ClusterInternalDomain),
215-
},
258+
Hostnames: hostnames,
216259
Subject: &certmgrv1.X509Subject{
217260
Organizations: []string{fmt.Sprintf("%s.%s", rabbitmq.Namespace, ClusterInternalDomain)},
218261
},
@@ -345,6 +388,34 @@ func reconcileRabbitMQ(
345388
]}
346389
].
347390
`
391+
392+
rabbitmq.Spec.Override.StatefulSet.Spec.Template.Spec.Volumes = []corev1.Volume{
393+
{
394+
Name: "config-data",
395+
VolumeSource: corev1.VolumeSource{
396+
ConfigMap: &corev1.ConfigMapVolumeSource{
397+
LocalObjectReference: corev1.LocalObjectReference{
398+
Name: fmt.Sprintf("%s-config-data", rabbitmq.Name),
399+
},
400+
DefaultMode: ptr.To[int32](0o420),
401+
Items: []corev1.KeyToPath{
402+
{
403+
Key: "inter_node_tls.config",
404+
Path: "inter_node_tls.config",
405+
},
406+
},
407+
},
408+
},
409+
},
410+
}
411+
rabbitmq.Spec.Override.StatefulSet.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
412+
{
413+
MountPath: "/etc/rabbitmq/inter-node-tls.config",
414+
ReadOnly: true,
415+
Name: "config-data",
416+
SubPath: "inter_node_tls.config",
417+
},
418+
}
348419
}
349420

350421
// overrides

‎tests/functional/base_test.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"github.com/openstack-k8s-operators/lib-common/modules/common/condition"
3030
openstackclientv1 "github.com/openstack-k8s-operators/openstack-operator/apis/client/v1beta1"
3131
corev1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1"
32+
rabbitmqv2 "github.com/rabbitmq/cluster-operator/v2/api/v1beta1"
3233
)
3334

3435
type Names struct {
@@ -456,3 +457,11 @@ func CreateClusterConfigCM() client.Object {
456457

457458
return cm
458459
}
460+
461+
func GetRabbitMQCluster(name types.NamespacedName) *rabbitmqv2.RabbitmqCluster {
462+
instance := &rabbitmqv2.RabbitmqCluster{}
463+
Eventually(func(g Gomega) {
464+
g.Expect(k8sClient.Get(ctx, name, instance)).Should(Succeed())
465+
}, timeout, interval).Should(Succeed())
466+
return instance
467+
}

‎tests/functional/openstackoperator_controller_test.go‎

Lines changed: 73 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -546,7 +546,12 @@ var _ = Describe("OpenStackOperator controller", func() {
546546
g.Expect(memcached.Spec.Replicas).Should(Equal(ptr.To[int32](1)))
547547
}, timeout, interval).Should(Succeed())
548548

549-
// TODO rabbitmq exists
549+
// rabbitmq exists
550+
Eventually(func(g Gomega) {
551+
rabbitmq := GetRabbitMQCluster(names.RabbitMQName)
552+
g.Expect(rabbitmq).Should(Not(BeNil()))
553+
g.Expect(rabbitmq.Spec.Replicas).Should(Equal(ptr.To[int32](1)))
554+
}, timeout, interval).Should(Succeed())
550555

551556
// keystone exists
552557
Eventually(func(g Gomega) {
@@ -759,14 +764,73 @@ var _ = Describe("OpenStackOperator controller", func() {
759764
g.Expect(memcached.Spec.Replicas).Should(Equal(ptr.To[int32](1)))
760765
}, timeout, interval).Should(Succeed())
761766

762-
// TODO rabbitmq exists
767+
// rabbitmq exists
768+
Eventually(func(g Gomega) {
769+
rabbitmq := GetRabbitMQCluster(names.RabbitMQName)
770+
g.Expect(rabbitmq).Should(Not(BeNil()))
771+
g.Expect(rabbitmq.Spec.Replicas).Should(Equal(ptr.To[int32](1)))
772+
}, timeout, interval).Should(Succeed())
763773

764774
// keystone exists
765775
Eventually(func(g Gomega) {
766776
keystoneAPI := keystone.GetKeystoneAPI(names.KeystoneAPIName)
767777
g.Expect(keystoneAPI).Should(Not(BeNil()))
768778
}, timeout, interval).Should(Succeed())
769779
})
780+
Describe("Rabbitmq inter node TLS config", func() {
781+
It("should create and mount inter node tls config file", func() {
782+
Eventually(func(g Gomega) {
783+
cm := th.GetConfigMap(types.NamespacedName{
784+
Namespace: names.RabbitMQName.Namespace,
785+
Name: names.RabbitMQName.Name + "-config-data",
786+
})
787+
g.Expect(cm).ShouldNot(BeNil())
788+
g.Expect(cm.Data).Should(HaveKey("inter_node_tls.config"))
789+
}, timeout, interval).Should(Succeed())
790+
Eventually(func(g Gomega) {
791+
rabbitmq := GetRabbitMQCluster(names.RabbitMQName)
792+
g.Expect(rabbitmq).Should(Not(BeNil()))
793+
794+
var vFindings []k8s_corev1.Volume
795+
g.Expect(rabbitmq.Spec.Override.StatefulSet.Spec.Template.Spec.Volumes).Should(
796+
ContainElement(HaveField("Name", "config-data"), &vFindings),
797+
)
798+
g.Expect(vFindings[0].VolumeSource.ConfigMap.Items[0]).Should(And(
799+
HaveField("Key", "inter_node_tls.config"),
800+
HaveField("Path", "inter_node_tls.config"),
801+
))
802+
803+
var vmFindings []k8s_corev1.VolumeMount
804+
g.Expect(rabbitmq.Spec.Override.StatefulSet.Spec.Template.Spec.Containers[0].VolumeMounts).Should(
805+
ContainElement(HaveField("Name", "config-data"), &vmFindings),
806+
)
807+
g.Expect(vmFindings[0]).Should(And(
808+
HaveField("SubPath", "inter_node_tls.config"),
809+
HaveField("MountPath", "/etc/rabbitmq/inter-node-tls.config"),
810+
))
811+
}, timeout, interval).Should(Succeed())
812+
})
813+
It("should set the TLS arg env vars", func() {
814+
Eventually(func(g Gomega) {
815+
rabbitmq := GetRabbitMQCluster(names.RabbitMQName)
816+
g.Expect(rabbitmq).Should(Not(BeNil()))
817+
818+
var findings []k8s_corev1.EnvVar
819+
g.Expect(rabbitmq.Spec.Override.StatefulSet.Spec.Template.Spec.Containers[0].Env).Should(
820+
ContainElement(HaveField("Name", "RABBITMQ_CTL_ERL_ARGS"), &findings),
821+
)
822+
g.Expect(findings[0].Value).Should(ContainSubstring(
823+
"-proto_dist inet_tls -ssl_dist_optfile /etc/rabbitmq/inter-node-tls.config",
824+
))
825+
g.Expect(rabbitmq.Spec.Override.StatefulSet.Spec.Template.Spec.Containers[0].Env).Should(
826+
ContainElement(HaveField("Name", "RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS"), &findings),
827+
)
828+
g.Expect(findings[0].Value).Should(ContainSubstring(
829+
"-proto_dist inet_tls -ssl_dist_optfile /etc/rabbitmq/inter-node-tls.config",
830+
))
831+
}, timeout, interval).Should(Succeed())
832+
})
833+
})
770834
// Default route timeouts are set
771835
It("should have default timeout for the routes set", func() {
772836
OSCtlplane := GetOpenStackControlPlane(names.OpenStackControlplaneName)
@@ -1194,6 +1258,13 @@ var _ = Describe("OpenStackOperator controller", func() {
11941258
g.Expect(memcached.Spec.Replicas).Should(Equal(ptr.To[int32](1)))
11951259
}, timeout, interval).Should(Succeed())
11961260

1261+
// rabbitmq exists
1262+
Eventually(func(g Gomega) {
1263+
rabbitmq := GetRabbitMQCluster(names.RabbitMQName)
1264+
g.Expect(rabbitmq).Should(Not(BeNil()))
1265+
g.Expect(rabbitmq.Spec.Replicas).Should(Equal(ptr.To[int32](1)))
1266+
}, timeout, interval).Should(Succeed())
1267+
11971268
// keystone exists
11981269
Eventually(func(g Gomega) {
11991270
keystoneAPI := keystone.GetKeystoneAPI(names.KeystoneAPIName)

0 commit comments

Comments
 (0)