Skip to content

Commit c55ec27

Browse files
stuggistuggi
committed
Add tolerations customization interface for service operators
Adds ability for service operators to customize pod tolerations similar to how resource limits/requests are currently handled. Features: - Add Tolerations field to ContainerSpec API type - Implement merge behavior: custom tolerations are merged with defaults, overriding by key when same key exists - Set global default tolerations (node.kubernetes.io/not-ready and node.kubernetes.io/unreachable with 120s timeout) in controller - Update deployment templates (managers.yaml, operator.yaml) to render custom tolerations from Deployment struct - Add test coverage for merge logic and override behavior Example usage: ```yaml operatorOverrides: - name: "keystone" controllerManager: tolerations: - key: "node.kubernetes.io/not-ready" # Override default timeout operator: "Exists" effect: "NoExecute" tolerationSeconds: 600 - key: "node.example.com/gpu" # Add new toleration operator: "Equal" value: "nvidia" effect: "NoSchedule" ``` The merge behavior ensures operators get both default tolerations (unless overridden by matching key) plus any additional custom ones, providing flexibility while maintaining safe defaults. Jira: OSPRH-18693 Assisted-by: claude-4-sonnet Signed-off-by: Martin Schuppert <[email protected]>
1 parent 69d50de commit c55ec27

File tree

13 files changed

+616
-43
lines changed

13 files changed

+616
-43
lines changed

‎Makefile‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,7 @@ bindata: kustomize yq ## Call sync bindata script
156156
sed -i bindata/operator/operator.yaml -e "/customLimits/c\\ cpu: {{ .OpenStackOperator.Deployment.Manager.Resources.Limits.CPU }}\n memory: {{ .OpenStackOperator.Deployment.Manager.Resources.Limits.Memory }}"
157157
sed -i bindata/operator/operator.yaml -e "/customRequests/c\\ cpu: {{ .OpenStackOperator.Deployment.Manager.Resources.Requests.CPU }}\n memory: {{ .OpenStackOperator.Deployment.Manager.Resources.Requests.Memory }}"
158158
sed -i bindata/operator/operator.yaml -e "s|kube-rbac-proxy:replace_me.*|'{{ .OpenStackOperator.Deployment.KubeRbacProxy.Image }}'|"
159+
sed -i bindata/operator/operator.yaml -e "/customTolerations/c\\ tolerations:\n{{- range .OpenStackOperator.Deployment.Tolerations }}\n - key: \"{{ .Key }}\"\n{{- if .Operator }}\n operator: \"{{ .Operator }}\"\n{{- end }}\n{{- if .Value }}\n value: \"{{ .Value }}\"\n{{- end }}\n{{- if .Effect }}\n effect: \"{{ .Effect }}\"\n{{- end }}\n{{- if .TolerationSeconds }}\n tolerationSeconds: {{ .TolerationSeconds }}\n{{- end }}\n{{- end }}"
159160
cp config/operator/managers.yaml bindata/operator/
160161
cp config/operator/rabbit.yaml bindata/operator/
161162
$(KUSTOMIZE) build config/rbac > bindata/rbac/rbac.yaml
@@ -203,7 +204,7 @@ ginkgo-run: ## Run ginkgo.
203204
source hack/export_related_images.sh && \
204205
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) -v debug --bin-dir $(LOCALBIN) use $(ENVTEST_K8S_VERSION) -p path)" \
205206
OPERATOR_TEMPLATES="$(PWD)/templates" \
206-
$(GINKGO) --trace --cover --coverpkg=./pkg/operator,./pkg/openstack,./pkg/openstackclient,./pkg/util,./pkg/dataplane/...,./controllers/...,./apis/client/v1beta1,./apis/core/v1beta1,./apis/dataplane/v1beta1 --coverprofile cover.out --covermode=atomic ${PROC_CMD} $(GINKGO_ARGS) $(GINKGO_TESTS)
207+
$(GINKGO) --trace --cover --coverpkg=./pkg/...,./controllers/...,./apis/... --coverprofile cover.out --covermode=atomic ${PROC_CMD} $(GINKGO_ARGS) $(GINKGO_TESTS)
207208

208209
.PHONY: test-all
209210
test-all: test golint golangci golangci-lint ## Run all tests.
@@ -302,7 +303,7 @@ CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
302303
ENVTEST ?= $(LOCALBIN)/setup-envtest
303304
CRD_MARKDOWN ?= $(LOCALBIN)/crd-to-markdown
304305
GINKGO ?= $(LOCALBIN)/ginkgo
305-
GINKGO_TESTS ?= ./tests/... ./apis/client/... ./apis/core/... ./apis/dataplane/... ./pkg/dataplane/...
306+
GINKGO_TESTS ?= ./tests/... ./apis/client/... ./apis/core/... ./apis/dataplane/... ./pkg/...
306307

307308
KUTTL ?= $(LOCALBIN)/kubectl-kuttl
308309

‎apis/bases/operator.openstack.org_openstacks.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,22 @@ spec:
7070
x-kubernetes-int-or-string: true
7171
type: object
7272
type: object
73+
tolerations:
74+
items:
75+
properties:
76+
effect:
77+
type: string
78+
key:
79+
type: string
80+
operator:
81+
type: string
82+
tolerationSeconds:
83+
format: int64
84+
type: integer
85+
value:
86+
type: string
87+
type: object
88+
type: array
7389
type: object
7490
name:
7591
enum:

‎apis/operator/v1beta1/openstack_types.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
corev1 "k8s.io/api/core/v1"
2222
"k8s.io/apimachinery/pkg/api/resource"
2323
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24+
"k8s.io/utils/ptr"
2425
)
2526

2627
const (
@@ -69,6 +70,23 @@ var (
6970
DefaultRbacProxyMemoryLimit resource.Quantity = resource.MustParse("128Mi")
7071
// DefaultRbacProxyMemoryRequests - Default kube rbac proxy container memory requests
7172
DefaultRbacProxyMemoryRequests resource.Quantity = resource.MustParse("64Mi")
73+
74+
// DefaultTolerations - Default tolerations for all operators
75+
DefaultTolerations = []corev1.Toleration{
76+
{
77+
Key: corev1.TaintNodeNotReady, // "node.kubernetes.io/not-ready"
78+
Operator: corev1.TolerationOpExists,
79+
Effect: corev1.TaintEffectNoExecute,
80+
TolerationSeconds: ptr.To[int64](120),
81+
},
82+
{
83+
Key: corev1.TaintNodeUnreachable, // "node.kubernetes.io/unreachable"
84+
Operator: corev1.TolerationOpExists,
85+
Effect: corev1.TaintEffectNoExecute,
86+
TolerationSeconds: ptr.To[int64](120),
87+
},
88+
}
89+
7290
// OperatorList - list of all operators with optional different defaults then the above.
7391
// NOTE: test-operator was deployed as a independant package so it may or may not be installed
7492
// NOTE: depending on how watcher-operator is released for FR2 and then in FR3 it may need to be
@@ -210,6 +228,11 @@ type ContainerSpec struct {
210228
// Resources - Compute Resources for the service operator controller manager
211229
// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
212230
Resources corev1.ResourceRequirements `json:"resources,omitempty"`
231+
232+
// +kubebuilder:validation:Optional
233+
// Tolerations - Tolerations for the service operator controller manager
234+
// https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/
235+
Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
213236
}
214237

215238
// OpenStackStatus defines the observed state of OpenStack

‎apis/operator/v1beta1/zz_generated.deepcopy.go‎

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bindata/operator/managers.yaml‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -86,14 +86,21 @@ spec:
8686
serviceAccountName: {{ .Name }}-operator-controller-manager
8787
terminationGracePeriodSeconds: 10
8888
tolerations:
89-
- key: "node.kubernetes.io/not-ready"
90-
operator: "Exists"
91-
effect: "NoExecute"
92-
tolerationSeconds: 120
93-
- key: "node.kubernetes.io/unreachable"
94-
operator: "Exists"
95-
effect: "NoExecute"
96-
tolerationSeconds: 120
89+
{{- range .Deployment.Tolerations }}
90+
- key: "{{ .Key }}"
91+
{{- if .Operator }}
92+
operator: "{{ .Operator }}"
93+
{{- end }}
94+
{{- if .Value }}
95+
value: "{{ .Value }}"
96+
{{- end }}
97+
{{- if .Effect }}
98+
effect: "{{ .Effect }}"
99+
{{- end }}
100+
{{- if .TolerationSeconds }}
101+
tolerationSeconds: {{ .TolerationSeconds }}
102+
{{- end }}
103+
{{- end }}
97104
{{- if isEnvVarTrue .Deployment.Manager.Env "ENABLE_WEBHOOKS" }}
98105
volumes:
99106
- name: cert

‎bindata/operator/operator.yaml‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -129,19 +129,26 @@ spec:
129129
memory: 64Mi
130130
securityContext:
131131
allowPrivilegeEscalation: false
132+
tolerations:
133+
{{- range .OpenStackOperator.Deployment.Tolerations }}
134+
- key: "{{ .Key }}"
135+
{{- if .Operator }}
136+
operator: "{{ .Operator }}"
137+
{{- end }}
138+
{{- if .Value }}
139+
value: "{{ .Value }}"
140+
{{- end }}
141+
{{- if .Effect }}
142+
effect: "{{ .Effect }}"
143+
{{- end }}
144+
{{- if .TolerationSeconds }}
145+
tolerationSeconds: {{ .TolerationSeconds }}
146+
{{- end }}
147+
{{- end }}
132148
securityContext:
133149
runAsNonRoot: true
134150
serviceAccountName: openstack-operator-controller-manager
135151
terminationGracePeriodSeconds: 10
136-
tolerations:
137-
- effect: NoExecute
138-
key: node.kubernetes.io/not-ready
139-
operator: Exists
140-
tolerationSeconds: 120
141-
- effect: NoExecute
142-
key: node.kubernetes.io/unreachable
143-
operator: Exists
144-
tolerationSeconds: 120
145152
volumes:
146153
- name: cert
147154
secret:

‎config/crd/bases/operator.openstack.org_openstacks.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,22 @@ spec:
7070
x-kubernetes-int-or-string: true
7171
type: object
7272
type: object
73+
tolerations:
74+
items:
75+
properties:
76+
effect:
77+
type: string
78+
key:
79+
type: string
80+
operator:
81+
type: string
82+
tolerationSeconds:
83+
format: int64
84+
type: integer
85+
value:
86+
type: string
87+
type: object
88+
type: array
7389
type: object
7490
name:
7591
enum:

‎config/manager/manager.yaml‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -72,12 +72,4 @@ spec:
7272
customRequests: replace_me #NOTE: this is used via the Makefile to inject a custom template that kustomize won't allow
7373
serviceAccountName: openstack-operator-controller-manager
7474
terminationGracePeriodSeconds: 10
75-
tolerations:
76-
- key: "node.kubernetes.io/not-ready"
77-
operator: "Exists"
78-
effect: "NoExecute"
79-
tolerationSeconds: 120
80-
- key: "node.kubernetes.io/unreachable"
81-
operator: "Exists"
82-
effect: "NoExecute"
83-
tolerationSeconds: 120
75+
customTolerations: replace_me #NOTE: this is used via the Makefile to inject a custom template that kustomize won't allow

‎config/operator/managers.yaml‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -86,14 +86,21 @@ spec:
8686
serviceAccountName: {{ .Name }}-operator-controller-manager
8787
terminationGracePeriodSeconds: 10
8888
tolerations:
89-
- key: "node.kubernetes.io/not-ready"
90-
operator: "Exists"
91-
effect: "NoExecute"
92-
tolerationSeconds: 120
93-
- key: "node.kubernetes.io/unreachable"
94-
operator: "Exists"
95-
effect: "NoExecute"
96-
tolerationSeconds: 120
89+
{{- range .Deployment.Tolerations }}
90+
- key: "{{ .Key }}"
91+
{{- if .Operator }}
92+
operator: "{{ .Operator }}"
93+
{{- end }}
94+
{{- if .Value }}
95+
value: "{{ .Value }}"
96+
{{- end }}
97+
{{- if .Effect }}
98+
effect: "{{ .Effect }}"
99+
{{- end }}
100+
{{- if .TolerationSeconds }}
101+
tolerationSeconds: {{ .TolerationSeconds }}
102+
{{- end }}
103+
{{- end }}
97104
{{- if isEnvVarTrue .Deployment.Manager.Env "ENABLE_WEBHOOKS" }}
98105
volumes:
99106
- name: cert

‎config/samples/operator_v1beta1_openstack_tolerations_example.yaml‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
apiVersion: operator.openstack.org/v1beta1
2+
kind: OpenStack
3+
metadata:
4+
labels:
5+
app.kubernetes.io/name: openstack
6+
app.kubernetes.io/instance: openstack-sample
7+
app.kubernetes.io/part-of: openstack-operator
8+
app.kubernetes.io/managed-by: kustomize
9+
app.kubernetes.io/created-by: openstack-operator
10+
name: openstack-sample
11+
spec:
12+
# Example showing how to customize tolerations for different service operators
13+
#
14+
# MERGE BEHAVIOR:
15+
# - Custom tolerations are MERGED with the default tolerations
16+
# - If a custom toleration has the same KEY as a default, it OVERRIDES the default
17+
# - If a custom toleration has a different KEY, it is ADDED to the defaults
18+
#
19+
# Default tolerations (applied automatically unless overridden):
20+
# - key: "node.kubernetes.io/not-ready"
21+
# operator: "Exists"
22+
# effect: "NoExecute"
23+
# tolerationSeconds: 120
24+
# - key: "node.kubernetes.io/unreachable"
25+
# operator: "Exists"
26+
# effect: "NoExecute"
27+
# tolerationSeconds: 120
28+
operatorOverrides:
29+
- name: "keystone"
30+
# Custom tolerations for keystone operator pods
31+
controllerManager:
32+
tolerations:
33+
- key: "example.com/special-nodes"
34+
operator: "Equal"
35+
value: "keystone"
36+
effect: "NoSchedule"
37+
- key: "node.kubernetes.io/memory-pressure"
38+
operator: "Exists"
39+
effect: "NoExecute"
40+
tolerationSeconds: 300
41+
- name: "nova"
42+
# Example: Override default tolerations and add new ones
43+
# Result will be:
44+
# 1. node.kubernetes.io/not-ready (OVERRIDDEN - 600s instead of 120s)
45+
# 2. node.kubernetes.io/unreachable (OVERRIDDEN - 400s instead of 120s)
46+
# 3. node.example.com/compute (ADDED - new toleration)
47+
controllerManager:
48+
tolerations:
49+
- key: "node.kubernetes.io/not-ready"
50+
operator: "Exists"
51+
effect: "NoExecute"
52+
tolerationSeconds: 600 # Override default 120s
53+
- key: "node.kubernetes.io/unreachable"
54+
operator: "Exists"
55+
effect: "NoExecute"
56+
tolerationSeconds: 400 # Override default 120s
57+
- key: "node.example.com/compute" # Add new toleration
58+
operator: "Equal"
59+
value: "true"
60+
effect: "NoSchedule"
61+
- name: "glance"
62+
# Custom resource limits AND tolerations example
63+
controllerManager:
64+
resources:
65+
limits:
66+
cpu: "2"
67+
memory: "4Gi"
68+
requests:
69+
cpu: "1"
70+
memory: "2Gi"
71+
tolerations:
72+
- key: "storage-node"
73+
operator: "Equal"
74+
value: "true"
75+
effect: "NoSchedule"
76+
# Note: Operators not listed (like mariadb, neutron, etc.) will use the default tolerations

0 commit comments

Comments
 (0)