From b6ccb30af1d44e240e5f51306fdb66f95a32f755 Mon Sep 17 00:00:00 2001 From: Francesco Pantano Date: Wed, 18 Jun 2025 11:09:32 +0200 Subject: [PATCH] Support single GlanceAPI StatefulSet with regular backends Starting with RHOSO 19, Nova and Cinder will adopt Glance's new location API, eliminating the need for default splitting. This change enables single GlanceAPI deployments with supported backends (S3, Ceph, Cinder, Swift) for both greenfield and existing environments. This enables a single Glance StatefulSet deployment with several benefits: - Reduces PVC resource requirements (e.g., halving staging area storage) - Simplify deployment topologies and use cases - Maintains split model for upgrade compatibility Existing split deployments cannot automatically migrate to single layout. Manual migration procedures will be documented separately. Signed-off-by: Francesco Pantano --- .../glance.openstack.org_glanceapis.yaml | 2 +- api/bases/glance.openstack.org_glances.yaml | 2 +- api/v1beta1/common_types.go | 2 +- api/v1beta1/glance_webhook.go | 5 - .../glance.openstack.org_glanceapis.yaml | 2 +- .../bases/glance.openstack.org_glances.yaml | 2 +- test/functional/glance_controller_test.go | 91 +++++++------------ test/functional/glanceapi_controller_test.go | 30 +++--- test/functional/validation_webhook_test.go | 23 ++--- 9 files changed, 56 insertions(+), 103 deletions(-) diff --git a/api/bases/glance.openstack.org_glanceapis.yaml b/api/bases/glance.openstack.org_glanceapis.yaml index 7dbfddc7..0696b45f 100644 --- a/api/bases/glance.openstack.org_glanceapis.yaml +++ b/api/bases/glance.openstack.org_glanceapis.yaml @@ -715,7 +715,7 @@ spec: type: string type: object type: - default: split + default: single enum: - split - single diff --git a/api/bases/glance.openstack.org_glances.yaml b/api/bases/glance.openstack.org_glances.yaml index f85c689c..0ed19be2 100644 --- a/api/bases/glance.openstack.org_glances.yaml +++ b/api/bases/glance.openstack.org_glances.yaml @@ -705,7 +705,7 @@ spec: type: string type: object type: - default: split + default: single enum: - split - single diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index 491fd5cf..86c77d3e 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -95,7 +95,7 @@ type GlanceAPITemplate struct { Storage Storage `json:"storage,omitempty"` // +kubebuilder:validation:Enum=split;single;edge - // +kubebuilder:default:=split + // +kubebuilder:default:=single // Type - represents the layout of the glanceAPI deployment. Type string `json:"type,omitempty"` diff --git a/api/v1beta1/glance_webhook.go b/api/v1beta1/glance_webhook.go index 701f0b8d..d2339a9c 100644 --- a/api/v1beta1/glance_webhook.go +++ b/api/v1beta1/glance_webhook.go @@ -203,11 +203,6 @@ func (r *GlanceSpecCore) isInvalidBackend(glanceAPI GlanceAPITemplate, topLevel if glanceAPI.Type == "split" && isFileBackend(glanceAPI.CustomServiceConfig, topLevel) { return true, InvalidBackendErrorMessageSplit } - // Do not allow to deploy a glanceAPI with "type: single" and a backend - // different than File (Cinder, Swift, Ceph): we must split in that case - if glanceAPI.Type == APISingle && !isFileBackend(glanceAPI.CustomServiceConfig, topLevel) { - return true, InvalidBackendErrorMessageSingle - } return false, "" } diff --git a/config/crd/bases/glance.openstack.org_glanceapis.yaml b/config/crd/bases/glance.openstack.org_glanceapis.yaml index 7dbfddc7..0696b45f 100644 --- a/config/crd/bases/glance.openstack.org_glanceapis.yaml +++ b/config/crd/bases/glance.openstack.org_glanceapis.yaml @@ -715,7 +715,7 @@ spec: type: string type: object type: - default: split + default: single enum: - split - single diff --git a/config/crd/bases/glance.openstack.org_glances.yaml b/config/crd/bases/glance.openstack.org_glances.yaml index f85c689c..0ed19be2 100644 --- a/config/crd/bases/glance.openstack.org_glances.yaml +++ b/config/crd/bases/glance.openstack.org_glances.yaml @@ -705,7 +705,7 @@ spec: type: string type: object type: - default: split + default: single enum: - split - single diff --git a/test/functional/glance_controller_test.go b/test/functional/glance_controller_test.go index 47a58d77..22de74e3 100644 --- a/test/functional/glance_controller_test.go +++ b/test/functional/glance_controller_test.go @@ -32,7 +32,6 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/condition" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" mariadb_test "github.com/openstack-k8s-operators/mariadb-operator/api/test/helpers" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" ) @@ -546,6 +545,7 @@ var _ = Describe("Glance controller", func() { "customServiceConfig": GlanceDummyBackend, "glanceAPIs": map[string]interface{}{ "default": map[string]interface{}{ + "type": "split", "containerImage": glancev1.GlanceAPIContainerImage, "networkAttachments": []string{"internalapi"}, "override": map[string]interface{}{ @@ -565,7 +565,6 @@ var _ = Describe("Glance controller", func() { }, ), ) - //infra.SimulateTransportURLReady(glanceTest.GlanceTransportURL) mariadb.SimulateMariaDBDatabaseCompleted(glanceTest.GlanceDatabaseName) mariadb.SimulateMariaDBAccountCompleted(glanceTest.GlanceDatabaseAccount) th.SimulateJobSuccess(glanceTest.GlanceDBSync) @@ -638,27 +637,21 @@ var _ = Describe("Glance controller", func() { keystone.SimulateKeystoneServiceReady(glanceTest.KeystoneService) }) It("Check the extraMounts of the resulting StatefulSets", func() { - th.SimulateStatefulSetReplicaReady(glanceTest.GlanceInternalStatefulSet) - th.SimulateStatefulSetReplicaReady(glanceTest.GlanceExternalStatefulSet) - // Retrieve the generated resources and the two internal/external - // instances that are split behind the scenes - ssInternal := th.GetStatefulSet(glanceTest.GlanceInternalStatefulSet) - ssExternal := th.GetStatefulSet(glanceTest.GlanceExternalStatefulSet) - - for _, ss := range []*appsv1.StatefulSet{ssInternal, ssExternal} { - // Check the resulting deployment fields - Expect(ss.Spec.Template.Spec.Volumes).To(HaveLen(6)) - Expect(ss.Spec.Template.Spec.Containers).To(HaveLen(2)) - // Get the glance-httpd container - container := ss.Spec.Template.Spec.Containers[1] - // Fail if glance-httpd doesn't have the right number of VolumeMounts - // entries - Expect(container.VolumeMounts).To(HaveLen(8)) - // Inspect VolumeMounts and make sure we have the Ceph MountPath - // provided through extraMounts - th.AssertVolumeMountPathExists(GlanceCephExtraMountsSecretName, - GlanceCephExtraMountsPath, "", container.VolumeMounts) - } + th.SimulateStatefulSetReplicaReady(glanceTest.GlanceSingle) + // Retrieve the generated resources and the glanceAPI StatefulSet + ss := th.GetStatefulSet(glanceTest.GlanceSingle) + // Check the resulting deployment fields + Expect(ss.Spec.Template.Spec.Volumes).To(HaveLen(6)) + Expect(ss.Spec.Template.Spec.Containers).To(HaveLen(2)) + // Get the glance-httpd container + container := ss.Spec.Template.Spec.Containers[1] + // Fail if glance-httpd doesn't have the right number of VolumeMounts + // entries + Expect(container.VolumeMounts).To(HaveLen(8)) + // Inspect VolumeMounts and make sure we have the Ceph MountPath + // provided through extraMounts + th.AssertVolumeMountPathExists(GlanceCephExtraMountsSecretName, + GlanceCephExtraMountsPath, "", container.VolumeMounts) }) }) @@ -710,8 +703,7 @@ var _ = Describe("Glance controller", func() { }) It("Check the topology has been applied to the resulting StatefulSets", func() { - th.SimulateStatefulSetReplicaReady(glanceTest.GlanceInternalStatefulSet) - th.SimulateStatefulSetReplicaReady(glanceTest.GlanceExternalStatefulSet) + th.SimulateStatefulSetReplicaReady(glanceTest.GlanceSingle) Eventually(func(g Gomega) { tp := infra.GetTopology(types.NamespacedName{ Name: topologyRef.Name, @@ -719,16 +711,12 @@ var _ = Describe("Glance controller", func() { }) finalizers := tp.GetFinalizers() g.Expect(finalizers).To(HaveLen(1)) - internalAPI := GetGlanceAPI(glanceTest.GlanceInternal) - externalAPI := GetGlanceAPI(glanceTest.GlanceExternal) - g.Expect(internalAPI.Status.LastAppliedTopology).ToNot(BeNil()) - g.Expect(internalAPI.Status.LastAppliedTopology).To(Equal(topologyRef)) + glanceAPI := GetGlanceAPI(glanceTest.GlanceSingle) + g.Expect(glanceAPI.Status.LastAppliedTopology).ToNot(BeNil()) + g.Expect(glanceAPI.Status.LastAppliedTopology).To(Equal(topologyRef)) g.Expect(finalizers).To(ContainElement( - fmt.Sprintf("openstack.org/glanceapi-%s", internalAPI.APIName()))) - g.Expect(externalAPI.Status.LastAppliedTopology).ToNot(BeNil()) - g.Expect(externalAPI.Status.LastAppliedTopology).To(Equal(topologyRef)) - g.Expect(finalizers).To(ContainElement( - fmt.Sprintf("openstack.org/glanceapi-%s", externalAPI.APIName()))) + fmt.Sprintf("openstack.org/glanceapi-%s", glanceAPI.APIName()))) + }, timeout, interval).Should(Succeed()) }) @@ -739,8 +727,7 @@ var _ = Describe("Glance controller", func() { g.Expect(k8sClient.Update(ctx, glance)).To(Succeed()) }, timeout, interval).Should(Succeed()) - th.SimulateStatefulSetReplicaReady(glanceTest.GlanceInternalStatefulSet) - th.SimulateStatefulSetReplicaReady(glanceTest.GlanceExternalStatefulSet) + th.SimulateStatefulSetReplicaReady(glanceTest.GlanceSingle) Eventually(func(g Gomega) { tp := infra.GetTopology(types.NamespacedName{ @@ -750,16 +737,11 @@ var _ = Describe("Glance controller", func() { finalizers := tp.GetFinalizers() g.Expect(finalizers).To(HaveLen(1)) - internalAPI := GetGlanceAPI(glanceTest.GlanceInternal) - externalAPI := GetGlanceAPI(glanceTest.GlanceExternal) - g.Expect(internalAPI.Status.LastAppliedTopology).ToNot(BeNil()) - g.Expect(internalAPI.Status.LastAppliedTopology).To(Equal(topologyRefAlt)) - g.Expect(finalizers).To(ContainElement( - fmt.Sprintf("openstack.org/glanceapi-%s", internalAPI.APIName()))) - g.Expect(externalAPI.Status.LastAppliedTopology).ToNot(BeNil()) - g.Expect(externalAPI.Status.LastAppliedTopology).To(Equal(topologyRefAlt)) + glanceAPI := GetGlanceAPI(glanceTest.GlanceSingle) + g.Expect(glanceAPI.Status.LastAppliedTopology).ToNot(BeNil()) + g.Expect(glanceAPI.Status.LastAppliedTopology).To(Equal(topologyRefAlt)) g.Expect(finalizers).To(ContainElement( - fmt.Sprintf("openstack.org/glanceapi-%s", externalAPI.APIName()))) + fmt.Sprintf("openstack.org/glanceapi-%s", glanceAPI.APIName()))) // Verify the previous referenced topology has no finalizers tp = infra.GetTopology(types.NamespacedName{ Name: topologyRef.Name, @@ -770,7 +752,7 @@ var _ = Describe("Glance controller", func() { }, timeout, interval).Should(Succeed()) }) - It("Remove the topology reference", func() { + It("Remove topology reference", func() { Eventually(func(g Gomega) { glance := GetGlance(glanceTest.Instance) // Remove the TopologyRef from the existing Glance .Spec @@ -779,23 +761,18 @@ var _ = Describe("Glance controller", func() { }, timeout, interval).Should(Succeed()) Eventually(func(g Gomega) { - internalAPI := GetGlanceAPI(glanceTest.GlanceInternal) - externalAPI := GetGlanceAPI(glanceTest.GlanceExternal) - g.Expect(internalAPI.Status.LastAppliedTopology).Should(BeNil()) - g.Expect(externalAPI.Status.LastAppliedTopology).Should(BeNil()) + glanceAPI := GetGlanceAPI(glanceTest.GlanceSingle) + g.Expect(glanceAPI.Status.LastAppliedTopology).Should(BeNil()) }, timeout, interval).Should(Succeed()) // Check the statefulSet has a default Affinity and no TopologySpreadConstraints: // Affinity is applied by DistributePods function provided by lib-common, while // TopologySpreadConstraints is part of the sample Topology used to test Glance Eventually(func(g Gomega) { - ssInternal := th.GetStatefulSet(glanceTest.GlanceInternalStatefulSet) - ssExternal := th.GetStatefulSet(glanceTest.GlanceExternalStatefulSet) - for _, ss := range []*appsv1.StatefulSet{ssInternal, ssExternal} { - // Check the resulting deployment fields - g.Expect(ss.Spec.Template.Spec.Affinity).ToNot(BeNil()) - g.Expect(ss.Spec.Template.Spec.TopologySpreadConstraints).To(BeNil()) - } + ss := th.GetStatefulSet(glanceTest.GlanceSingle) + // Check the resulting deployment fields + g.Expect(ss.Spec.Template.Spec.Affinity).ToNot(BeNil()) + g.Expect(ss.Spec.Template.Spec.TopologySpreadConstraints).To(BeNil()) }, timeout, interval).Should(Succeed()) // Verify the existing topologies have no finalizer anymore diff --git a/test/functional/glanceapi_controller_test.go b/test/functional/glanceapi_controller_test.go index 6233e7d4..f928fcb8 100644 --- a/test/functional/glanceapi_controller_test.go +++ b/test/functional/glanceapi_controller_test.go @@ -20,7 +20,6 @@ import ( "fmt" "os" - appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/types" . "github.com/onsi/ginkgo/v2" //revive:disable:dot-imports @@ -1103,26 +1102,22 @@ var _ = Describe("Glanceapi controller", func() { }) It("Checks the Topology has been applied to the resulting StatefulSets", func() { - th.SimulateStatefulSetReplicaReady(glanceTest.GlanceInternalStatefulSet) - th.SimulateStatefulSetReplicaReady(glanceTest.GlanceExternalStatefulSet) + th.SimulateStatefulSetReplicaReady(glanceTest.GlanceSingle) Eventually(func(g Gomega) { - internalAPI := GetGlanceAPI(glanceTest.GlanceInternal) - g.Expect(internalAPI.Status.LastAppliedTopology).ShouldNot(BeNil()) - g.Expect(internalAPI.Status.LastAppliedTopology).To(Equal(topologyRefAlt)) + glanceAPI := GetGlanceAPI(glanceTest.GlanceSingle) + g.Expect(glanceAPI.Status.LastAppliedTopology).ShouldNot(BeNil()) + g.Expect(glanceAPI.Status.LastAppliedTopology).To(Equal(topologyRefAlt)) }, timeout, interval).Should(Succeed()) // Check the statefulSet has a default TopologySpreadConstraints and no Affinity // TopologySpreadConstraints is part of the sample Topology used to test Glance, // and is referenced using the Topology CR passed to the GlanceAPI Eventually(func(g Gomega) { - ssInternal := th.GetStatefulSet(glanceTest.GlanceInternalStatefulSet) - ssExternal := th.GetStatefulSet(glanceTest.GlanceExternalStatefulSet) + ss := th.GetStatefulSet(glanceTest.GlanceSingle) _, topologySpecObj := GetSampleTopologySpec(topologyRefAlt.Name) - for _, ss := range []*appsv1.StatefulSet{ssInternal, ssExternal} { - // Check the resulting deployment fields - g.Expect(ss.Spec.Template.Spec.Affinity).To(BeNil()) - g.Expect(ss.Spec.Template.Spec.TopologySpreadConstraints).ToNot(BeNil()) - g.Expect(ss.Spec.Template.Spec.TopologySpreadConstraints).To(Equal(topologySpecObj)) - } + // Check the resulting deployment fields + g.Expect(ss.Spec.Template.Spec.Affinity).To(BeNil()) + g.Expect(ss.Spec.Template.Spec.TopologySpreadConstraints).ToNot(BeNil()) + g.Expect(ss.Spec.Template.Spec.TopologySpreadConstraints).To(Equal(topologySpecObj)) }, timeout, interval).Should(Succeed()) Eventually(func(g Gomega) { @@ -1132,12 +1127,9 @@ var _ = Describe("Glanceapi controller", func() { }) finalizers := tp.GetFinalizers() g.Expect(finalizers).To(HaveLen(1)) - internalAPI := GetGlanceAPI(glanceTest.GlanceInternal) - g.Expect(finalizers).To(ContainElement( - fmt.Sprintf("openstack.org/glanceapi-%s", internalAPI.APIName()))) - externalAPI := GetGlanceAPI(glanceTest.GlanceExternal) + glanceAPI := GetGlanceAPI(glanceTest.GlanceSingle) g.Expect(finalizers).To(ContainElement( - fmt.Sprintf("openstack.org/glanceapi-%s", externalAPI.APIName()))) + fmt.Sprintf("openstack.org/glanceapi-%s", glanceAPI.APIName()))) }, timeout, interval).Should(Succeed()) }) }) diff --git a/test/functional/validation_webhook_test.go b/test/functional/validation_webhook_test.go index 8def23a0..8b556ca0 100644 --- a/test/functional/validation_webhook_test.go +++ b/test/functional/validation_webhook_test.go @@ -58,26 +58,15 @@ var _ = Describe("Glance validation", func() { spec := GetGlanceDefaultSpec() gapis := map[string]interface{}{ - "glanceAPIs": map[string]interface{}{ - "default": map[string]interface{}{ - "replicas": 1, - "type": "split", - }, - "edge1": map[string]interface{}{ - "replicas": 1, - "type": "edge", - }, - // Webhooks catch that a backend != File is set for an instance - // that has type: single - "api1": map[string]interface{}{ - "customServiceConfig": GetDummyBackend(), - "replicas": 1, - "type": "single", - }, + // Webhooks catch that a backend == File is set for an instance + // that has type: split, which is invalid + "default": map[string]interface{}{ + "replicas": 1, + "type": "split", }, } - spec["keystoneEndpoint"] = "edge1" + spec["keystoneEndpoint"] = "default" spec["glanceAPIs"] = gapis raw := map[string]interface{}{