Skip to content

Commit 2aa884a

Browse files
stuggiopenshift-cherrypick-robot
stuggi
authored and
openshift-cherrypick-robot
committed
Add interface to allow customize of operator deployments
Adds an interface which allows to customize the following parameters of a controller-manager operator deployment - change the replica (0/1) - change the resources (requests/limits) of the manager container It keeps the current default resource requests as is. For easier gotemplate it this adds an internal custom struct type, which allows to just loop over a list of operator.Operator for service operators and a dedicated instance for the openstack-operator-controller-manager and the rabbitmq-cluster-operator-manager. E.g. changing the default cpu limits for the infra operator and disabling the openstack-baremetal operator would be: ~~~ ... spec: operatorOverrides: - name: infra controllerManager: resources: limits: cpu: 600m memory: 2Gi replicas: 1 - name: openstack-baremetal replicas: 0 ~~~ To add a new service operator has the following high level steps: - in the openstack_types.go api - add a new const with the name - add an entry for the defaults in the OperatorList var - add the name also to the Enum of the OperatorSpec.Name - (optional) if any specific env var need be passed to the operator, e.g. related images list, or to enable webhooks, check the applyOperator() in the openstack_controller.go . Jira: OSPRH-17785 Signed-off-by: Martin Schuppert <[email protected]>
1 parent bb1d7bd commit 2aa884a

File tree

16 files changed

+1154
-207
lines changed

16 files changed

+1154
-207
lines changed

‎Makefile‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -151,8 +151,11 @@ bindata: kustomize yq ## Call sync bindata script
151151
mkdir -p bindata/crds bindata/rbac bindata/operator
152152
$(KUSTOMIZE) build config/crd > bindata/crds/crds.yaml
153153
$(KUSTOMIZE) build config/default > bindata/operator/operator.yaml
154-
sed -i bindata/operator/operator.yaml -e "/envCustomImage/c\\{{ range \$$envName, \$$envValue := .OpenStackServiceRelatedImages }}\n - name: {{ \$$envName }}\n value: {{ \$$envValue }}\n{{ end }}"
155-
sed -i bindata/operator/operator.yaml -e "s|kube-rbac-proxy:replace_me.*|'{{ .KubeRbacProxyImage }}'|"
154+
sed -i bindata/operator/operator.yaml -e "s|replicas:.*|replicas: {{ .OpenStackOperator.Deployment.Replicas }}|"
155+
sed -i bindata/operator/operator.yaml -e "/envCustom/c\\{{- range .OpenStackOperator.Deployment.Manager.Env }}\n - name: '{{ .Name }}'\n value: '{{ .Value }}'\n{{- end }}"
156+
sed -i bindata/operator/operator.yaml -e "/customLimits/c\\ cpu: {{ .OpenStackOperator.Deployment.Manager.Resources.Limits.CPU }}\n memory: {{ .OpenStackOperator.Deployment.Manager.Resources.Limits.Memory }}"
157+
sed -i bindata/operator/operator.yaml -e "/customRequests/c\\ cpu: {{ .OpenStackOperator.Deployment.Manager.Resources.Requests.CPU }}\n memory: {{ .OpenStackOperator.Deployment.Manager.Resources.Requests.Memory }}"
158+
sed -i bindata/operator/operator.yaml -e "s|kube-rbac-proxy:replace_me.*|'{{ .OpenStackOperator.Deployment.KubeRbacProxy.Image }}'|"
156159
cp config/operator/managers.yaml bindata/operator/
157160
cp config/operator/rabbit.yaml bindata/operator/
158161
$(KUSTOMIZE) build config/rbac > bindata/rbac/rbac.yaml
@@ -200,7 +203,7 @@ ginkgo-run: ## Run ginkgo.
200203
source hack/export_related_images.sh && \
201204
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) -v debug --bin-dir $(LOCALBIN) use $(ENVTEST_K8S_VERSION) -p path)" \
202205
OPERATOR_TEMPLATES="$(PWD)/templates" \
203-
$(GINKGO) --trace --cover --coverpkg=./pkg/openstack,./pkg/openstackclient,./pkg/util,./pkg/dataplane/...,./controllers/...,./apis/client/v1beta1,./apis/core/v1beta1,./apis/dataplane/v1beta1 --coverprofile cover.out --covermode=atomic ${PROC_CMD} $(GINKGO_ARGS) $(GINKGO_TESTS)
206+
$(GINKGO) --trace --cover --coverpkg=./pkg/operator,./pkg/openstack,./pkg/openstackclient,./pkg/util,./pkg/dataplane/...,./controllers/...,./apis/client/v1beta1,./apis/core/v1beta1,./apis/dataplane/v1beta1 --coverprofile cover.out --covermode=atomic ${PROC_CMD} $(GINKGO_ARGS) $(GINKGO_TESTS)
204207

205208
.PHONY: test-all
206209
test-all: test golint golangci golangci-lint ## Run all tests.

‎apis/bases/operator.openstack.org_openstacks.yaml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,82 @@ spec:
3333
metadata:
3434
type: object
3535
spec:
36+
properties:
37+
operatorOverrides:
38+
items:
39+
properties:
40+
controllerManager:
41+
properties:
42+
resources:
43+
properties:
44+
claims:
45+
items:
46+
properties:
47+
name:
48+
type: string
49+
required:
50+
- name
51+
type: object
52+
type: array
53+
x-kubernetes-list-map-keys:
54+
- name
55+
x-kubernetes-list-type: map
56+
limits:
57+
additionalProperties:
58+
anyOf:
59+
- type: integer
60+
- type: string
61+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
62+
x-kubernetes-int-or-string: true
63+
type: object
64+
requests:
65+
additionalProperties:
66+
anyOf:
67+
- type: integer
68+
- type: string
69+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
70+
x-kubernetes-int-or-string: true
71+
type: object
72+
type: object
73+
type: object
74+
name:
75+
enum:
76+
- openstack
77+
- barbican
78+
- cinder
79+
- designate
80+
- glance
81+
- heat
82+
- horizon
83+
- infra
84+
- ironic
85+
- keystone
86+
- manila
87+
- mariadb
88+
- neutron
89+
- nova
90+
- octavia
91+
- openstack-baremetal
92+
- ovn
93+
- placement
94+
- rabbitmq-cluster
95+
- swift
96+
- telemetry
97+
- test
98+
minLength: 1
99+
type: string
100+
replicas:
101+
format: int32
102+
maximum: 1
103+
minimum: 0
104+
type: integer
105+
required:
106+
- name
107+
type: object
108+
type: array
109+
x-kubernetes-list-map-keys:
110+
- name
111+
x-kubernetes-list-type: map
36112
type: object
37113
status:
38114
properties:
@@ -62,9 +138,15 @@ spec:
62138
type: string
63139
deployedOperatorCount:
64140
type: integer
141+
disabledOperatorCount:
142+
type: integer
143+
enabledOperatorCount:
144+
type: integer
65145
observedGeneration:
66146
format: int64
67147
type: integer
148+
totalOperatorCount:
149+
type: integer
68150
type: object
69151
type: object
70152
served: true

‎apis/operator/v1beta1/openstack_types.go‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,198 @@ package v1beta1
1818

1919
import (
2020
condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
21+
corev1 "k8s.io/api/core/v1"
22+
"k8s.io/apimachinery/pkg/api/resource"
2123
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2224
)
2325

26+
const (
27+
OpenStackOperatorName = "openstack"
28+
BarbicanOperatorName = "barbican"
29+
CinderOperatorName = "cinder"
30+
DesignateOperatorName = "designate"
31+
GlanceOperatorName = "glance"
32+
HeatOperatorName = "heat"
33+
HorizonOperatorName = "horizon"
34+
InfraOperatorName = "infra"
35+
IronicOperatorName = "ironic"
36+
KeystoneOperatorName = "keystone"
37+
ManilaOperatorName = "manila"
38+
MariaDBOperatorName = "mariadb"
39+
NeutronOperatorName = "neutron"
40+
NovaOperatorName = "nova"
41+
OctaviaOperatorName = "octavia"
42+
OpenStackBaremetalOperatorName = "openstack-baremetal"
43+
OvnOperatorName = "ovn"
44+
PlacementOperatorName = "placement"
45+
RabbitMQOperatorName = "rabbitmq-cluster"
46+
SwiftOperatorName = "swift"
47+
TelemetryOperatorName = "telemetry"
48+
TestOperatorName = "test"
49+
// ReplicasEnabled - default replicas count when enabled
50+
ReplicasEnabled int32 = 1
51+
// ReplicasDisabled - replicas when disabled
52+
ReplicasDisabled int32 = 0
53+
)
54+
55+
var (
56+
// DefaultManagerCPULimit - Default controller manager container CPU limit
57+
DefaultManagerCPULimit resource.Quantity = resource.MustParse("500m")
58+
// DefaultManagerCPURequests - Default controller manager container CPU requests
59+
DefaultManagerCPURequests resource.Quantity = resource.MustParse("10m")
60+
// DefaultManagerMemoryLimit - Default controller manager container memory limit
61+
DefaultManagerMemoryLimit resource.Quantity = resource.MustParse("256Mi")
62+
// DefaultManagerMemoryRequests - Default controller manager container memory requests
63+
DefaultManagerMemoryRequests resource.Quantity = resource.MustParse("128Mi")
64+
// DefaultRbacProxyCPULimit - Default kube rbac proxy container CPU limit
65+
DefaultRbacProxyCPULimit resource.Quantity = resource.MustParse("500m")
66+
// DefaultRbacProxyCPURequests - Default kube rbac proxy container CPU requests
67+
DefaultRbacProxyCPURequests resource.Quantity = resource.MustParse("5m")
68+
// DefaultRbacProxyMemoryLimit - Default kube rbac proxy container memory limit
69+
DefaultRbacProxyMemoryLimit resource.Quantity = resource.MustParse("128Mi")
70+
// DefaultRbacProxyMemoryRequests - Default kube rbac proxy container memory requests
71+
DefaultRbacProxyMemoryRequests resource.Quantity = resource.MustParse("64Mi")
72+
// OperatorList - list of all operators with optional different defaults then the above.
73+
// NOTE: test-operator was deployed as a independant package so it may or may not be installed
74+
// NOTE: depending on how watcher-operator is released for FR2 and then in FR3 it may need to be
75+
// added into this list in the future
76+
// IMPORTANT: have this list in sync with the Enum in OperatorSpec.Name parameter
77+
OperatorList []OperatorSpec = []OperatorSpec{
78+
{
79+
Name: OpenStackOperatorName,
80+
ControllerManager: ContainerSpec{
81+
Resources: corev1.ResourceRequirements{
82+
Requests: corev1.ResourceList{
83+
corev1.ResourceMemory: resource.MustParse("512Mi"),
84+
},
85+
Limits: corev1.ResourceList{
86+
corev1.ResourceMemory: resource.MustParse("1Gi"),
87+
},
88+
},
89+
},
90+
},
91+
{
92+
Name: BarbicanOperatorName,
93+
},
94+
{
95+
Name: CinderOperatorName,
96+
},
97+
{
98+
Name: DesignateOperatorName,
99+
},
100+
{
101+
Name: GlanceOperatorName,
102+
},
103+
{
104+
Name: HeatOperatorName,
105+
},
106+
{
107+
Name: HorizonOperatorName,
108+
},
109+
{
110+
Name: InfraOperatorName,
111+
ControllerManager: ContainerSpec{
112+
Resources: corev1.ResourceRequirements{
113+
Requests: corev1.ResourceList{
114+
corev1.ResourceMemory: resource.MustParse("512Mi"),
115+
},
116+
Limits: corev1.ResourceList{
117+
corev1.ResourceMemory: resource.MustParse("1Gi"),
118+
},
119+
},
120+
},
121+
},
122+
{
123+
Name: IronicOperatorName,
124+
},
125+
{
126+
Name: KeystoneOperatorName,
127+
},
128+
{
129+
Name: ManilaOperatorName,
130+
},
131+
{
132+
Name: MariaDBOperatorName,
133+
},
134+
{
135+
Name: NeutronOperatorName,
136+
},
137+
{
138+
Name: NovaOperatorName,
139+
},
140+
{
141+
Name: OctaviaOperatorName,
142+
},
143+
{
144+
Name: OpenStackBaremetalOperatorName,
145+
},
146+
{
147+
Name: OvnOperatorName,
148+
},
149+
{
150+
Name: PlacementOperatorName,
151+
},
152+
{
153+
Name: RabbitMQOperatorName,
154+
ControllerManager: ContainerSpec{
155+
Resources: corev1.ResourceRequirements{
156+
Requests: corev1.ResourceList{
157+
corev1.ResourceCPU: resource.MustParse("5m"),
158+
corev1.ResourceMemory: resource.MustParse("64Mi"),
159+
},
160+
Limits: corev1.ResourceList{
161+
corev1.ResourceCPU: resource.MustParse("200m"),
162+
corev1.ResourceMemory: resource.MustParse("500Mi"),
163+
},
164+
},
165+
},
166+
},
167+
{
168+
Name: SwiftOperatorName,
169+
},
170+
{
171+
Name: TelemetryOperatorName,
172+
},
173+
{
174+
Name: TestOperatorName,
175+
},
176+
}
177+
)
178+
24179
// OpenStackSpec defines the desired state of OpenStack
25180
type OpenStackSpec struct {
181+
// +kubebuilder:validation:Optional
182+
// +listType=map
183+
// +listMapKey=name
184+
// OperatorOverrides - list of OperatorSpec which allows to customize operator deployments
185+
OperatorOverrides []OperatorSpec `json:"operatorOverrides"`
186+
}
187+
188+
// OperatorSpec - customization for the operator deployment
189+
type OperatorSpec struct {
190+
// +kubebuilder:validation:Required
191+
// +kubebuilder:validation:MinLength=1
192+
// +kubebuilder:validation:Enum:=openstack;barbican;cinder;designate;glance;heat;horizon;infra;ironic;keystone;manila;mariadb;neutron;nova;octavia;openstack-baremetal;ovn;placement;rabbitmq-cluster;swift;telemetry;test
193+
// Name of the service operators.
194+
Name string `json:"name"`
195+
196+
// +kubebuilder:validation:Optional
197+
// +kubebuilder:validation:Maximum=1
198+
// +kubebuilder:validation:Minimum=0
199+
// Replicas of the operator deployment
200+
Replicas *int32 `json:"replicas,omitempty"`
201+
202+
// +kubebuilder:validation:Optional
203+
// ControllerManager - tunings for the controller manager container
204+
ControllerManager ContainerSpec `json:"controllerManager,omitempty"`
205+
}
206+
207+
// ContainerSpec - customization for the container spec
208+
type ContainerSpec struct {
209+
// +kubebuilder:validation:Optional
210+
// Resources - Compute Resources for the service operator controller manager
211+
// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
212+
Resources corev1.ResourceRequirements `json:"resources,omitempty"`
26213
}
27214

28215
// OpenStackStatus defines the observed state of OpenStack
@@ -35,6 +222,15 @@ type OpenStackStatus struct {
35222
// DeployedOperatorCount - the number of operators deployed
36223
DeployedOperatorCount *int `json:"deployedOperatorCount,omitempty"`
37224

225+
// DisabledOperatorCount - the number of operators which has replicas set to 0
226+
DisabledOperatorCount *int `json:"disabledOperatorCount,omitempty"`
227+
228+
// EnabledOperatorCount - the number of operators which has replicas set to 1
229+
EnabledOperatorCount *int `json:"enabledOperatorCount,omitempty"`
230+
231+
// TotalOperatorCount - the number all operators available
232+
TotalOperatorCount *int `json:"totalOperatorCount,omitempty"`
233+
38234
// ObservedGeneration - the most recent generation observed for this object.
39235
ObservedGeneration int64 `json:"observedGeneration,omitempty"` // no spec yet so maybe we don't need this
40236

0 commit comments

Comments
 (0)