Skip to content

Commit 31671e9

Browse files
nikParasyrnikParasyr
committed
Add e2e tests for autoscaling
1 parent 4dcc029 commit 31671e9

File tree

5 files changed

+343
-0
lines changed

5 files changed

+343
-0
lines changed

‎controllers/openstackmachinetemplate_controller_test.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,19 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
117
package controllers
218

319
import (

‎test/e2e/data/autoscaler/autoscaler-to-workload-workload.yaml‎

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
# This yaml deploys the autoscaler on a workload cluster and configures it to match
2+
# against the corresponding Cluster API cluster which is defined into the management cluster.
3+
---
4+
apiVersion: v1
5+
kind: Namespace
6+
metadata:
7+
name: cluster-autoscaler-system
8+
labels:
9+
pod-security.kubernetes.io/enforce: privileged
10+
pod-security.kubernetes.io/warn: privileged
11+
pod-security.kubernetes.io/audit: privileged
12+
---
13+
# Specify kubeconfig for management cluster
14+
apiVersion: v1
15+
kind: Secret
16+
metadata:
17+
name: kubeconfig-management-cluster
18+
namespace: cluster-autoscaler-system
19+
stringData:
20+
kubeconfig: |
21+
apiVersion: v1
22+
kind: Config
23+
clusters:
24+
- name: management-cluster
25+
cluster:
26+
certificate-authority-data: ${MANAGEMENT_CLUSTER_CA}
27+
server: ${MANAGEMENT_CLUSTER_ADDRESS}
28+
contexts:
29+
- name: management-context
30+
context:
31+
cluster: management-cluster
32+
namespace: ${CLUSTER_NAMESPACE}
33+
user: cluster-autoscaler-sa
34+
current-context: management-context
35+
users:
36+
- name: cluster-autoscaler-sa
37+
user:
38+
token: "${MANAGEMENT_CLUSTER_TOKEN}"
39+
---
40+
# Defines the service used by the cluster autoscaler and gives it
41+
# RBAC permissions to look at all the workloads running in this cluster.
42+
apiVersion: v1
43+
kind: ServiceAccount
44+
metadata:
45+
name: cluster-autoscaler
46+
namespace: cluster-autoscaler-system
47+
---
48+
kind: ClusterRoleBinding
49+
apiVersion: rbac.authorization.k8s.io/v1
50+
metadata:
51+
name: cluster-autoscaler-workload
52+
roleRef:
53+
apiGroup: rbac.authorization.k8s.io
54+
kind: ClusterRole
55+
name: cluster-autoscaler-workload
56+
subjects:
57+
- kind: ServiceAccount
58+
name: cluster-autoscaler
59+
namespace: cluster-autoscaler-system
60+
---
61+
kind: ClusterRole
62+
apiVersion: rbac.authorization.k8s.io/v1
63+
metadata:
64+
name: cluster-autoscaler-workload
65+
rules:
66+
- apiGroups:
67+
- ""
68+
resources:
69+
- namespaces
70+
- persistentvolumeclaims
71+
- persistentvolumes
72+
- pods
73+
- replicationcontrollers
74+
- services
75+
verbs:
76+
- get
77+
- list
78+
- watch
79+
- apiGroups:
80+
- "storage.k8s.io"
81+
resources:
82+
- volumeattachments
83+
verbs:
84+
- get
85+
- list
86+
- watch
87+
- apiGroups:
88+
- ""
89+
resources:
90+
- nodes
91+
verbs:
92+
- get
93+
- list
94+
- update
95+
- watch
96+
- apiGroups:
97+
- ""
98+
resources:
99+
- pods/eviction
100+
verbs:
101+
- create
102+
- apiGroups:
103+
- policy
104+
resources:
105+
- poddisruptionbudgets
106+
verbs:
107+
- list
108+
- watch
109+
- apiGroups:
110+
- storage.k8s.io
111+
resources:
112+
- csinodes
113+
- storageclasses
114+
- csidrivers
115+
- csistoragecapacities
116+
verbs:
117+
- get
118+
- list
119+
- watch
120+
- apiGroups:
121+
- batch
122+
resources:
123+
- jobs
124+
verbs:
125+
- list
126+
- watch
127+
- apiGroups:
128+
- apps
129+
resources:
130+
- daemonsets
131+
- replicasets
132+
- statefulsets
133+
verbs:
134+
- list
135+
- watch
136+
- apiGroups:
137+
- ""
138+
resources:
139+
- events
140+
verbs:
141+
- create
142+
- patch
143+
- apiGroups:
144+
- ""
145+
resources:
146+
- configmaps
147+
verbs:
148+
- create
149+
- delete
150+
- get
151+
- update
152+
- apiGroups:
153+
- coordination.k8s.io
154+
resources:
155+
- leases
156+
verbs:
157+
- create
158+
- get
159+
- update
160+
---
161+
apiVersion: apps/v1
162+
kind: Deployment
163+
metadata:
164+
name: cluster-autoscaler
165+
namespace: cluster-autoscaler-system
166+
labels:
167+
app: cluster-autoscaler
168+
spec:
169+
selector:
170+
matchLabels:
171+
app: cluster-autoscaler
172+
replicas: 1
173+
template:
174+
metadata:
175+
labels:
176+
app: cluster-autoscaler
177+
spec:
178+
containers:
179+
- image: registry.k8s.io/autoscaling/cluster-autoscaler:${AUTOSCALER_VERSION}
180+
name: cluster-autoscaler
181+
command:
182+
- /cluster-autoscaler
183+
args:
184+
- --cloud-provider=clusterapi
185+
# Specify kubeconfig for management cluster
186+
- --cloud-config=/management-cluster/kubeconfig
187+
# Limit cluster autoscaler to only match against resources belonging to a single Cluster API cluster
188+
- --node-group-auto-discovery=clusterapi:namespace=${CLUSTER_NAMESPACE},clusterName=${CLUSTER_NAME}
189+
# Set a short scale down unneeded time, so we don't have to wait too long during e2e testing
190+
- --scale-down-unneeded-time=1m
191+
# Set a short scale down delay after add time, so we don't have to wait too long during e2e testing
192+
- --scale-down-delay-after-add=1m
193+
# Set a short scale down delay after delete time, so we don't have to wait too long during e2e testing
194+
- --scale-down-delay-after-delete=1m
195+
# Set a short scale down delay after failure time, so we don't have to wait too long during e2e testing
196+
- --scale-down-delay-after-failure=1m
197+
# Disable special handling for pods in kube-system to prevent flakes when one of the core-dns pod is scheduled
198+
# on one of the MachineDeployment machines and the test is trying to scale down to zero.
199+
- --skip-nodes-with-system-pods=false
200+
# Set a max nodes limit as safeguard so that the test does not scale up unbounded.
201+
# Note: The E2E test should only go up to 4 (assuming it starts with a min node group size of 2).
202+
# Using 6 for additional some buffer and to allow different starting min node group sizes.
203+
- --max-nodes-total=6
204+
# CABPK sets this taint on startup, so we should configure it here accordingly
205+
- --startup-taint=node.cluster.x-k8s.io/uninitialized
206+
# Set the log verbosity
207+
- --v=4
208+
volumeMounts:
209+
- name: kubeconfig-management-cluster
210+
mountPath: /management-cluster
211+
readOnly: true
212+
# Run the autoscaler on control plane Machines to avoid disruptions when scaling to 0.
213+
nodeSelector:
214+
node-role.kubernetes.io/control-plane: ""
215+
tolerations:
216+
- key: node-role.kubernetes.io/control-plane
217+
effect: NoSchedule
218+
operator: Exists
219+
serviceAccountName: cluster-autoscaler
220+
terminationGracePeriodSeconds: 10
221+
volumes:
222+
- name: kubeconfig-management-cluster
223+
secret:
224+
secretName: kubeconfig-management-cluster
225+
optional: false

‎test/e2e/data/autoscaler/cluster-template-topology-autoscaler.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
apiVersion: cluster.x-k8s.io/v1beta2
2+
kind: Cluster
3+
metadata:
4+
name: ${CLUSTER_NAME}
5+
spec:
6+
topology:
7+
classRef:
8+
name: dev-test
9+
controlPlane:
10+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
11+
variables:
12+
- name: identityRef
13+
value:
14+
cloudName: ${OPENSTACK_CLOUD:=capo-e2e}
15+
name: ${CLOUD_CONFIG_SECRET:=dev-test-cloud-config}
16+
- name: imageRef
17+
value: ${IMAGE_REF:=node-image}
18+
- name: addImageVersion
19+
value: ${ADD_IMAGE_VERSION:=false}
20+
- name: injectIgnitionSysext
21+
value: ${INJECT_IGNITION_SYSEXT:=true}
22+
- name: allowedCIDRs
23+
value: ${OPENSTACK_API_SERVER_ALLOWED_CIDRS:=[]}
24+
- name: bastion
25+
value:
26+
enabled: ${OPENSTACK_BASTION_ENABLED:=false}
27+
spec:
28+
flavor: ${OPENSTACK_BASTION_FLAVOR:=m1.small}
29+
image:
30+
imageRef:
31+
name: ${OPENSTACK_BASTION_IMAGE_NAME:=ubuntu-24.04}
32+
sshKeyName: ${OPENSTACK_SSH_KEY_NAME:=""}
33+
version: ${KUBERNETES_VERSION}
34+
workers:
35+
machineDeployments:
36+
- class: default-worker
37+
name: md-0
38+
deletion:
39+
nodeDeletionTimeoutSeconds: 60
40+
metadata:
41+
annotations:
42+
cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "3"
43+
cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "0"

‎test/e2e/data/e2e_conf.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ providers:
124124
files:
125125
- sourcePath: "../data/shared/provider/metadata.yaml"
126126
- sourcePath: "./infrastructure-openstack-no-artifact/cluster-template.yaml"
127+
- sourcePath: "../data/autoscaler/cluster-template-topology-autoscaler.yaml"
127128
replacements:
128129
- old: "imagePullPolicy: Always"
129130
new: "imagePullPolicy: IfNotPresent"
@@ -213,6 +214,8 @@ variables:
213214
KUBERNETES_VERSION: "v1.34.2"
214215
KUBERNETES_VERSION_UPGRADE_FROM: "v1.33.1"
215216
KUBERNETES_VERSION_UPGRADE_TO: "v1.34.2"
217+
AUTOSCALER_VERSION: "v1.34.2"
218+
AUTOSCALER_WORKLOAD: "./data/autoscaler/autoscaler-to-workload-workload.yaml"
216219
# NOTE: To see default images run kubeadm config images list (optionally with --kubernetes-version=vX.Y.Z)
217220
ETCD_VERSION_UPGRADE_TO: "3.5.21-0"
218221
COREDNS_VERSION_UPGRADE_TO: "v1.12.0"
@@ -222,6 +225,7 @@ variables:
222225
CCM: "../../data/ccm/cloud-controller-manager.yaml"
223226
EXP_CLUSTER_RESOURCE_SET: "true"
224227
EXP_CAPO_PRIORITY_QUEUE: "false"
228+
EXP_CAPO_AUTOSCALE_FROM_ZERO: "true"
225229
IP_FAMILY: "ipv4"
226230
OPENSTACK_BASTION_IMAGE_NAME: "cirros-0.6.1-x86_64-disk"
227231
OPENSTACK_BASTION_IMAGE_URL: https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/cirros/2022-12-05/cirros-0.6.1-x86_64-disk.img

‎test/e2e/suites/e2e/autoscaler_test.go‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
//go:build e2e
2+
3+
/*
4+
Copyright 2025 The Kubernetes Authors.
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
19+
package e2e
20+
21+
import (
22+
"context"
23+
24+
. "github.com/onsi/ginkgo/v2"
25+
capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
26+
27+
infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1"
28+
)
29+
30+
// Currently marked as PR-blocking to ensure it runs
31+
// might need to remove it.
32+
var _ = Describe("Autoscaler on ClusterClass cluster [PR-Blocking] [Autoscaler] [ClusterClass]", func() {
33+
capi_e2e.AutoscalerSpec(context.TODO(), func() capi_e2e.AutoscalerSpecInput {
34+
infraAPIGroup := infrav1.GroupName
35+
autoscalerFlavor := "topology-autoscaler"
36+
37+
return capi_e2e.AutoscalerSpecInput{
38+
E2EConfig: e2eCtx.E2EConfig,
39+
ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath,
40+
BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy,
41+
ArtifactFolder: e2eCtx.Settings.ArtifactFolder,
42+
SkipCleanup: false,
43+
Flavor: &autoscalerFlavor,
44+
InfrastructureMachineTemplateKind: "openstackmachinetemplates",
45+
// CAPO does not support machinePools
46+
InfrastructureMachinePoolTemplateKind: "",
47+
InfrastructureMachinePoolKind: "",
48+
InfrastructureAPIGroup: infraAPIGroup,
49+
AutoscalerVersion: e2eCtx.E2EConfig.MustGetVariable("AUTOSCALER_VERSION"),
50+
InstallOnManagementCluster: false,
51+
ScaleToAndFromZero: true,
52+
PostNamespaceCreated: nil,
53+
}
54+
})
55+
})

0 commit comments

Comments
 (0)