Skip to content

Commit d066bc5

Browse files
dprincedprince
committed
minor update: update rabbit/galara/keystone in sequence
Change the minor update workflow so that RabbitMQ, Galara, and Keystone are updated in sequence ahead of the rest of the controlplane services. Jira: OSPRH-16109
1 parent b0944fc commit d066bc5

File tree

6 files changed

+262
-43
lines changed

6 files changed

+262
-43
lines changed

‎apis/core/v1beta1/conditions.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -467,14 +467,20 @@ const (
467467
OpenStackControlPlaneInstanceHaCMReadyMessage = "OpenStackControlPlane InstanceHa CM is available"
468468
)
469469

470-
// Version Conditions used by API objects.
470+
// Version Conditions used by to drive minor updates
471471
const (
472472
OpenStackVersionInitialized condition.Type = "Initialized"
473473

474474
OpenStackVersionMinorUpdateOVNDataplane condition.Type = "MinorUpdateOVNDataplane"
475475

476476
OpenStackVersionMinorUpdateOVNControlplane condition.Type = "MinorUpdateOVNControlplane"
477477

478+
OpenStackVersionMinorUpdateRabbitMQ condition.Type = "MinorUpdateRabbitMQ"
479+
480+
OpenStackVersionMinorUpdateMariaDB condition.Type = "MinorUpdateMariaDB"
481+
482+
OpenStackVersionMinorUpdateKeystone condition.Type = "MinorUpdateKeystone"
483+
478484
OpenStackVersionMinorUpdateControlplane condition.Type = "MinorUpdateControlplane"
479485

480486
OpenStackVersionMinorUpdateDataplane condition.Type = "MinorUpdateDataplane"

‎controllers/core/openstackcontrolplane_controller.go‎

Lines changed: 64 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,8 @@ func (r *OpenStackControlPlaneReconciler) Reconcile(ctx context.Context, req ctr
168168

169169
// Always patch the instance status when exiting this function so we can persist any changes.
170170
defer func() {
171+
//Log all the conditions
172+
171173
// update the Ready condition based on the sub conditions
172174
if instance.Status.Conditions.AllSubConditionIsTrue() {
173175
instance.Status.Conditions.MarkTrue(
@@ -242,33 +244,80 @@ func (r *OpenStackControlPlaneReconciler) Reconcile(ctx context.Context, req ctr
242244
instance.Status.DeployedVersion = &version.Spec.TargetVersion
243245
return ctrl.Result{}, nil
244246
} else {
245-
if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateOVNControlplane) {
246-
Log.Info("Minor update OVN on the ControlPlane")
247-
ctrlResult, err := r.reconcileOVNControllers(ctx, instance, version, helper)
247+
248+
// OVN
249+
Log.Info("Minor update OVN on the ControlPlane")
250+
ctrlResult, err := r.reconcileOVNControllers(ctx, instance, version, helper)
251+
if err != nil {
252+
return ctrl.Result{}, err
253+
} else if (ctrlResult != ctrl.Result{}) {
254+
return ctrlResult, nil
255+
} else {
256+
instance.Status.DeployedOVNVersion = &version.Spec.TargetVersion
257+
if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateOVNControlplane) {
258+
return ctrlResult, nil
259+
}
260+
}
261+
262+
// only if OVN dataplane is updated
263+
if version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateOVNDataplane) {
264+
Log.Info("Minor update in progress")
265+
266+
// RabbitMQ
267+
ctrlResult, err := openstack.ReconcileRabbitMQs(ctx, instance, version, helper)
248268
if err != nil {
249269
return ctrl.Result{}, err
250270
} else if (ctrlResult != ctrl.Result{}) {
251271
return ctrlResult, nil
272+
} else {
273+
if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateRabbitMQ) {
274+
Log.Info("Returning for RabbitMQ minor update reconcile")
275+
return ctrlResult, nil
276+
}
252277
}
253-
instance.Status.DeployedOVNVersion = &version.Spec.TargetVersion
254-
return ctrl.Result{}, nil
255-
} else if version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateOVNDataplane) &&
256-
!version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateControlplane) {
257278

258-
Log.Info("Minor update on the ControlPlane")
259-
ctrlResult, err := r.reconcileNormal(ctx, instance, version, helper)
279+
// Galara
280+
ctrlResult, err = openstack.ReconcileGaleras(ctx, instance, version, helper)
260281
if err != nil {
261282
return ctrl.Result{}, err
262283
} else if (ctrlResult != ctrl.Result{}) {
263284
return ctrlResult, nil
285+
} else {
286+
if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateMariaDB) {
287+
Log.Info("Returning for Galara minor update reconcile")
288+
return ctrlResult, nil
289+
}
290+
}
291+
// Keystone API
292+
ctrlResult, err = openstack.ReconcileKeystoneAPI(ctx, instance, version, helper)
293+
if err != nil {
294+
return ctrl.Result{}, err
295+
} else if (ctrlResult != ctrl.Result{}) {
296+
return ctrlResult, nil
297+
} else {
298+
if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateKeystone) {
299+
Log.Info("Returning for KeystoneAPI minor update reconcile")
300+
return ctrlResult, nil
301+
}
302+
}
303+
304+
// the rest of the controlplane
305+
ctrlResult, err = r.reconcileNormal(ctx, instance, version, helper)
306+
if err != nil {
307+
return ctrl.Result{}, err
308+
} else if (ctrlResult != ctrl.Result{}) {
309+
return ctrlResult, nil
310+
} else {
311+
// this will allow reconcileNormal to proceed in subsequent reconciles
312+
instance.Status.DeployedVersion = &version.Spec.TargetVersion
313+
if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateControlplane) {
314+
Log.Info("Returning for ControlPlane minor update reconcile")
315+
return ctrlResult, nil
316+
}
264317
}
265-
// this will allow reconcileNormal to proceed in subsequent reconciles
266-
instance.Status.DeployedVersion = &version.Spec.TargetVersion
267-
return ctrl.Result{}, nil
268-
} else {
269-
Log.Info("Skipping reconcile. Waiting on minor update to proceed.")
270-
return ctrl.Result{}, nil
271318
}
319+
return ctrl.Result{}, nil
320+
272321
}
273322
}
274323

‎controllers/core/openstackversion_controller.go‎

Lines changed: 52 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,8 +157,12 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req
157157
)
158158
// no minor update conditions unless we have a deployed version
159159
if instance.Status.DeployedVersion != nil && instance.Spec.TargetVersion != *instance.Status.DeployedVersion {
160-
cl = append(cl, *condition.UnknownCondition(corev1beta1.OpenStackVersionMinorUpdateOVNControlplane, condition.InitReason, string(corev1beta1.OpenStackVersionMinorUpdateInitMessage)),
160+
cl = append(cl,
161+
*condition.UnknownCondition(corev1beta1.OpenStackVersionMinorUpdateOVNControlplane, condition.InitReason, string(corev1beta1.OpenStackVersionMinorUpdateInitMessage)),
161162
*condition.UnknownCondition(corev1beta1.OpenStackVersionMinorUpdateOVNDataplane, condition.InitReason, string(corev1beta1.OpenStackVersionMinorUpdateInitMessage)),
163+
*condition.UnknownCondition(corev1beta1.OpenStackVersionMinorUpdateRabbitMQ, condition.InitReason, string(corev1beta1.OpenStackVersionMinorUpdateInitMessage)),
164+
*condition.UnknownCondition(corev1beta1.OpenStackVersionMinorUpdateMariaDB, condition.InitReason, string(corev1beta1.OpenStackVersionMinorUpdateInitMessage)),
165+
*condition.UnknownCondition(corev1beta1.OpenStackVersionMinorUpdateKeystone, condition.InitReason, string(corev1beta1.OpenStackVersionMinorUpdateInitMessage)),
162166
*condition.UnknownCondition(corev1beta1.OpenStackVersionMinorUpdateControlplane, condition.InitReason, string(corev1beta1.OpenStackVersionMinorUpdateInitMessage)),
163167
*condition.UnknownCondition(corev1beta1.OpenStackVersionMinorUpdateDataplane, condition.InitReason, string(corev1beta1.OpenStackVersionMinorUpdateInitMessage)),
164168
)
@@ -259,6 +263,51 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req
259263
corev1beta1.OpenStackVersionMinorUpdateOVNDataplane,
260264
corev1beta1.OpenStackVersionMinorUpdateReadyMessage)
261265

266+
// minor update for RabbitMQ
267+
if !openstack.RabbitmqImageMatch(ctx, controlPlane, instance) ||
268+
!controlPlane.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneRabbitMQReadyCondition) {
269+
instance.Status.Conditions.Set(condition.FalseCondition(
270+
corev1beta1.OpenStackVersionMinorUpdateRabbitMQ,
271+
condition.RequestedReason,
272+
condition.SeverityInfo,
273+
corev1beta1.OpenStackVersionMinorUpdateReadyRunningMessage))
274+
Log.Info("Minor update for RabbitMQ in progress")
275+
return ctrl.Result{}, nil
276+
}
277+
instance.Status.Conditions.MarkTrue(
278+
corev1beta1.OpenStackVersionMinorUpdateRabbitMQ,
279+
corev1beta1.OpenStackVersionMinorUpdateReadyMessage)
280+
281+
// minor update for MariaDB
282+
if !openstack.GaleraImageMatch(ctx, controlPlane, instance) ||
283+
!controlPlane.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneMariaDBReadyCondition) {
284+
instance.Status.Conditions.Set(condition.FalseCondition(
285+
corev1beta1.OpenStackVersionMinorUpdateMariaDB,
286+
condition.RequestedReason,
287+
condition.SeverityInfo,
288+
corev1beta1.OpenStackVersionMinorUpdateReadyRunningMessage))
289+
Log.Info("Minor update for MariaDB in progress")
290+
return ctrl.Result{}, nil
291+
}
292+
instance.Status.Conditions.MarkTrue(
293+
corev1beta1.OpenStackVersionMinorUpdateMariaDB,
294+
corev1beta1.OpenStackVersionMinorUpdateReadyMessage)
295+
296+
// minor update for Keystone API
297+
if !openstack.KeystoneImageMatch(ctx, controlPlane, instance) ||
298+
!controlPlane.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneKeystoneAPIReadyCondition) {
299+
instance.Status.Conditions.Set(condition.FalseCondition(
300+
corev1beta1.OpenStackVersionMinorUpdateKeystone,
301+
condition.RequestedReason,
302+
condition.SeverityInfo,
303+
corev1beta1.OpenStackVersionMinorUpdateReadyRunningMessage))
304+
Log.Info("Minor update for Keystone in progress")
305+
return ctrl.Result{}, nil
306+
}
307+
instance.Status.Conditions.MarkTrue(
308+
corev1beta1.OpenStackVersionMinorUpdateKeystone,
309+
corev1beta1.OpenStackVersionMinorUpdateReadyMessage)
310+
262311
// minor update for Controlplane in progress
263312
if !controlPlane.IsReady() {
264313
instance.Status.Conditions.Set(condition.FalseCondition(
@@ -281,10 +330,10 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req
281330
errMsgBadMatches))
282331

283332
}
284-
285333
instance.Status.Conditions.MarkTrue(
286334
corev1beta1.OpenStackVersionMinorUpdateControlplane,
287335
corev1beta1.OpenStackVersionMinorUpdateReadyMessage)
336+
Log.Info("Minor update for ControlPlane completed")
288337

289338
if !openstack.DataplaneNodesetsDeployed(instance, dataplaneNodesets) {
290339
instance.Status.Conditions.Set(condition.FalseCondition(
@@ -299,6 +348,7 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req
299348
instance.Status.Conditions.MarkTrue(
300349
corev1beta1.OpenStackVersionMinorUpdateDataplane,
301350
corev1beta1.OpenStackVersionMinorUpdateReadyMessage)
351+
302352
}
303353

304354
if controlPlane.IsReady() {

‎pkg/openstack/rabbitmq.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,7 @@ func reconcileRabbitMQ(
285285
}
286286

287287
if rabbitmq.Status.ObservedGeneration == rabbitmq.Generation && rabbitmq.IsReady() {
288-
instance.Status.ContainerImages.InfraMemcachedImage = version.Status.ContainerImages.InfraMemcachedImage
288+
instance.Status.ContainerImages.RabbitmqImage = version.Status.ContainerImages.RabbitmqImage
289289
return mqReady, ctrl.Result{}, nil
290290
}
291291

‎tests/functional/ctlplane/base_test.go‎

Lines changed: 35 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -654,13 +654,38 @@ func GetRabbitMQCluster(name types.NamespacedName) *rabbitmqv1.RabbitMq {
654654
return instance
655655
}
656656

657-
func SimulateControlplaneReady() {
657+
func SimulateRabbitmqReady() {
658+
658659
instance := GetOpenStackControlPlane(names.OpenStackControlplaneName)
660+
if instance.Spec.Rabbitmq.Enabled {
661+
// FIXME add helpers to infra-operator to simulate rabbitmq
662+
Eventually(func(g Gomega) {
659663

660-
if instance.Spec.Keystone.Enabled {
661-
keystone.SimulateKeystoneAPIReady(names.KeystoneAPIName)
664+
for rabbitName := range *instance.Spec.Rabbitmq.Templates {
665+
rabbitmqNamespacedName := types.NamespacedName{
666+
Namespace: names.Namespace,
667+
Name: rabbitName,
668+
}
669+
670+
rabbit := &rabbitmqv1.RabbitMq{}
671+
g.Expect(th.K8sClient.Get(th.Ctx, rabbitmqNamespacedName, rabbit)).Should(Succeed())
672+
rabbit.Status.ObservedGeneration = rabbit.Generation
673+
rabbit.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage)
674+
675+
g.Expect(th.K8sClient.Status().Update(th.Ctx, rabbit)).To(Succeed())
676+
677+
th.Logger.Info("Simulated RabbitMq ready", "on", rabbit.Name)
678+
679+
}
680+
681+
}, timeout, interval).Should(Succeed())
662682
}
663683

684+
}
685+
686+
func SimulateGalaraReady() {
687+
688+
instance := GetOpenStackControlPlane(names.OpenStackControlplaneName)
664689
if instance.Spec.Galera.Enabled {
665690
// FIXME add helpers to mariadb-operator to simulate galera!
666691
Eventually(func(g Gomega) {
@@ -682,25 +707,16 @@ func SimulateControlplaneReady() {
682707
}, timeout, interval).Should(Succeed())
683708
}
684709

685-
if instance.Spec.Rabbitmq.Enabled {
686-
// FIXME add helpers to infra-operator to simulate rabbitmq
687-
Eventually(func(g Gomega) {
710+
}
688711

689-
for rabbitName := range *instance.Spec.Rabbitmq.Templates {
690-
rabbitmqNamespacedName := types.NamespacedName{
691-
Namespace: names.Namespace,
692-
Name: rabbitName,
693-
}
712+
func SimulateControlplaneReady() {
713+
instance := GetOpenStackControlPlane(names.OpenStackControlplaneName)
694714

695-
rabbit := &rabbitmqv1.RabbitMq{}
696-
g.Expect(th.K8sClient.Get(th.Ctx, rabbitmqNamespacedName, rabbit)).Should(Succeed())
697-
rabbit.Status.ObservedGeneration = rabbit.Generation
698-
rabbit.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage)
699-
g.Expect(th.K8sClient.Status().Update(th.Ctx, rabbit)).To(Succeed())
700-
th.Logger.Info("Simulated RabbitMq ready", "on", rabbitName)
701-
}
715+
SimulateRabbitmqReady()
716+
SimulateGalaraReady()
702717

703-
}, timeout, interval).Should(Succeed())
718+
if instance.Spec.Keystone.Enabled {
719+
keystone.SimulateKeystoneAPIReady(names.KeystoneAPIName)
704720
}
705721

706722
if instance.Spec.Memcached.Enabled {

0 commit comments

Comments
 (0)