From 752e8b3afc0aa1a8435f05c19c36496cd31329ea Mon Sep 17 00:00:00 2001 From: Damien Ciabrini Date: Wed, 25 Jun 2025 16:10:36 +0200 Subject: [PATCH] Expose pod property for resource QoS Add the ability to configure required/allowed CPU and memory consumption for galera pod, by exposing the standard QoS Resources from pod API. The galera type now exposes an optional Resources field to configure galera pod's QoS, similarly to the RabbitMQCluster type. Unlike rabbitmq, no default QoS is enforce, so the default galera pod behaviour doesn't change (QoS Class is BestEffort). Jira: OSPRH-17410 --- api/bases/mariadb.openstack.org_galeras.yaml | 55 +++++++++ api/v1beta1/galera_types.go | 8 +- api/v1beta1/zz_generated.deepcopy.go | 1 + .../bases/mariadb.openstack.org_galeras.yaml | 55 +++++++++ hack/crd-schema-checker.sh | 3 +- pkg/mariadb/statefulset.go | 2 + tests/chainsaw/tests/qos/chainsaw-test.yaml | 113 ++++++++++++++++++ 7 files changed, 234 insertions(+), 3 deletions(-) create mode 100644 tests/chainsaw/tests/qos/chainsaw-test.yaml diff --git a/api/bases/mariadb.openstack.org_galeras.yaml b/api/bases/mariadb.openstack.org_galeras.yaml index 7715bf46..3754d3af 100644 --- a/api/bases/mariadb.openstack.org_galeras.yaml +++ b/api/bases/mariadb.openstack.org_galeras.yaml @@ -78,6 +78,61 @@ spec: maximum: 3 minimum: 0 type: integer + resources: + description: Resources QoS configuration for pods + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object secret: description: Name of the secret to look for password keys type: string diff --git a/api/v1beta1/galera_types.go b/api/v1beta1/galera_types.go index 5267c8ee..7767d3f3 100644 --- a/api/v1beta1/galera_types.go +++ b/api/v1beta1/galera_types.go @@ -17,12 +17,13 @@ limitations under the License. package v1beta1 import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" - topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" - "k8s.io/apimachinery/pkg/util/validation/field" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation/field" ) const ( @@ -87,6 +88,9 @@ type GaleraSpecCore struct { // TopologyRef to apply the Topology defined by the associated CR referenced // by name TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` + // +kubebuilder:validation:Optional + // Resources QoS configuration for pods + Resources corev1.ResourceRequirements `json:"resources,omitempty"` } // GaleraAttributes holds startup information for a Galera host diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 993b0609..e22b9b3c 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -194,6 +194,7 @@ func (in *GaleraSpecCore) DeepCopyInto(out *GaleraSpecCore) { *out = new(topologyv1beta1.TopoRef) **out = **in } + in.Resources.DeepCopyInto(&out.Resources) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GaleraSpecCore. diff --git a/config/crd/bases/mariadb.openstack.org_galeras.yaml b/config/crd/bases/mariadb.openstack.org_galeras.yaml index 7715bf46..3754d3af 100644 --- a/config/crd/bases/mariadb.openstack.org_galeras.yaml +++ b/config/crd/bases/mariadb.openstack.org_galeras.yaml @@ -78,6 +78,61 @@ spec: maximum: 3 minimum: 0 type: integer + resources: + description: Resources QoS configuration for pods + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object secret: description: Name of the secret to look for password keys type: string diff --git a/hack/crd-schema-checker.sh b/hack/crd-schema-checker.sh index 3a252acd..b8298f60 100755 --- a/hack/crd-schema-checker.sh +++ b/hack/crd-schema-checker.sh @@ -17,6 +17,7 @@ for crd in config/crd/bases/*.yaml; do if git show "$BASE_REF:$crd" > "$TMP_DIR/$crd"; then $CHECKER check-manifests \ --existing-crd-filename="$TMP_DIR/$crd" \ - --new-crd-filename="$crd" + --new-crd-filename="$crd" \ + --disabled-validators NoMaps fi done diff --git a/pkg/mariadb/statefulset.go b/pkg/mariadb/statefulset.go index d3207c9a..ce3202b0 100644 --- a/pkg/mariadb/statefulset.go +++ b/pkg/mariadb/statefulset.go @@ -115,6 +115,7 @@ func getGaleraInitContainers(g *mariadbv1.Galera) []corev1.Container { }, }, }}, + Resources: g.Spec.Resources, VolumeMounts: getGaleraInitVolumeMounts(g), }} } @@ -149,6 +150,7 @@ func getGaleraContainers(g *mariadbv1.Galera, configHash string) []corev1.Contai ContainerPort: 4567, Name: "galera", }}, + Resources: g.Spec.Resources, VolumeMounts: getGaleraVolumeMounts(g), StartupProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ diff --git a/tests/chainsaw/tests/qos/chainsaw-test.yaml b/tests/chainsaw/tests/qos/chainsaw-test.yaml new file mode 100644 index 00000000..ad179907 --- /dev/null +++ b/tests/chainsaw/tests/qos/chainsaw-test.yaml @@ -0,0 +1,113 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: qos +spec: + steps: + - name: Deploy 1-node cluster without pod QoS + description: Deploy a 1-node cluster and wait for readiness + bindings: + - name: replicas + value: 1 + try: + - apply: + file: ../../common/galera.yaml + - assert: + # check if all nodes are started and one pod is reacheable via the service endpoint + file: ../../common/galera-assert.yaml + - script: &check + # check if galera can be accessed and cluster is correct + content: | + ../../scripts/mysql-cli.sh -sNEe "show status like 'wsrep_cluster_size';" | tail -1 | tr -d '\n' + check: + (to_number($stdout)): ($replicas) + + - name: Update QoS to Burstable + description: Ensure that requested resource is configured in the pod + try: + - patch: + resource: + apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + metadata: + name: openstack + spec: + resources: &res_burstable + requests: + memory: "128M" + cpu: "700m" + - assert: + resource: + apiVersion: v1 + kind: Pod + metadata: + name: openstack-galera-0 + spec: + initContainers: + - resources: *res_burstable + containers: + - name: galera + resources: *res_burstable + status: + qosClass: Burstable + containerStatuses: + - name: galera + ready: true + + - name: Update QoS to Guaranteed + description: Ensure that requested and limit resources are configured in the pod + try: + - patch: + resource: + apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + metadata: + name: openstack + spec: + resources: &res_guaranteed + limits: + memory: "256M" + cpu: "700m" + requests: + memory: "256M" + cpu: "700m" + - assert: + resource: + apiVersion: v1 + kind: Pod + metadata: + name: openstack-galera-0 + spec: + initContainers: + - resources: *res_guaranteed + containers: + - name: galera + resources: *res_guaranteed + status: + qosClass: Guaranteed + containerStatuses: + - name: galera + ready: true + + - name: Remove pod QoS + description: Ensure that a pod QoS can be removed + try: + - patch: + resource: + apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + metadata: + name: openstack + spec: + resources: + - assert: + resource: + apiVersion: v1 + kind: Pod + metadata: + name: openstack-galera-0 + status: + qosClass: BestEffort + containerStatuses: + - name: galera + ready: true