Skip to content

Commit 88face9

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #8514 from ykakarap/pr-e2e-autoscaler
🌱 E2E: autoscaler e2e test
2 parents 2be4e65 + 14d3cbb commit 88face9

File tree

12 files changed

+1014
-33
lines changed

12 files changed

+1014
-33
lines changed

‎Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -514,6 +514,7 @@ generate-e2e-templates-main: $(KUSTOMIZE)
514514
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-kcp-scale-in --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-kcp-scale-in.yaml
515515
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-ipv6 --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-ipv6.yaml
516516
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-single-node-cluster --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-single-node-cluster.yaml
517+
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-autoscaler --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-autoscaler.yaml
517518
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology.yaml
518519
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-ignition --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-ignition.yaml
519520

‎api/v1beta1/common_types.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,26 @@ const (
148148
// This annotation can be used to inform MachinePool status during in-progress scaling scenarios.
149149
ReplicasManagedByAnnotation = "cluster.x-k8s.io/replicas-managed-by"
150150

151+
// AutoscalerMinSizeAnnotation defines the minimum node group size.
152+
// The annotation is used by autoscaler.
153+
// The annotation is copied from kubernetes/autoscaler.
154+
// Ref:https://github.com/kubernetes/autoscaler/blob/d8336cca37dbfa5d1cb7b7e453bd511172d6e5e7/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_utils.go#L256-L259
155+
// Note: With the Kubernetes autoscaler it is possible to use different annotations by configuring a different
156+
// "Cluster API group" than "cluster.x-k8s.io" via the "CAPI_GROUP" environment variable.
157+
// We only handle the default group in our implementation.
158+
// Note: It can be used by setting as top level annotation on MachineDeployment and MachineSets.
159+
AutoscalerMinSizeAnnotation = "cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"
160+
161+
// AutoscalerMaxSizeAnnotation defines the maximum node group size.
162+
// The annotations is used by the autoscaler.
163+
// The annotation definition is copied from kubernetes/autoscaler.
164+
// Ref:https://github.com/kubernetes/autoscaler/blob/d8336cca37dbfa5d1cb7b7e453bd511172d6e5e7/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_utils.go#L264-L267
165+
// Note: With the Kubernetes autoscaler it is possible to use different annotations by configuring a different
166+
// "Cluster API group" than "cluster.x-k8s.io" via the "CAPI_GROUP" environment variable.
167+
// We only handle the default group in our implementation.
168+
// Note: It can be used by setting as top level annotation on MachineDeployment and MachineSets.
169+
AutoscalerMaxSizeAnnotation = "cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size"
170+
151171
// VariableDefinitionFromInline indicates a patch or variable was defined in the `.spec` of a ClusterClass
152172
// rather than from an external patch extension.
153173
VariableDefinitionFromInline = "inline"

‎api/v1beta1/machinedeployment_webhook.go

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -267,14 +267,6 @@ func (m *MachineDeployment) validate(old *MachineDeployment) error {
267267
return apierrors.NewInvalid(GroupVersion.WithKind("MachineDeployment").GroupKind(), m.Name, allErrs)
268268
}
269269

270-
// With the Kubernetes autoscaler it is possible to use different annotations by configuring a different
271-
// "Cluster API group" than "cluster.x-k8s.io" via the "CAPI_GROUP" environment variable.
272-
// We only handle the default group in our implementation.
273-
const (
274-
autoscalerMinSize = "cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"
275-
autoscalerMaxSize = "cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size"
276-
)
277-
278270
// calculateMachineDeploymentReplicas calculates the default value of the replicas field.
279271
// The value will be calculated based on the following logic:
280272
// * if replicas is already set on newMD, keep the current value
@@ -314,23 +306,23 @@ func calculateMachineDeploymentReplicas(ctx context.Context, oldMD *MachineDeplo
314306
log := ctrl.LoggerFrom(ctx).WithValues("MachineDeployment", klog.KObj(newMD))
315307

316308
// If both autoscaler annotations are set, use them to calculate the default value.
317-
minSizeString, hasMinSizeAnnotation := newMD.Annotations[autoscalerMinSize]
318-
maxSizeString, hasMaxSizeAnnotation := newMD.Annotations[autoscalerMaxSize]
309+
minSizeString, hasMinSizeAnnotation := newMD.Annotations[AutoscalerMinSizeAnnotation]
310+
maxSizeString, hasMaxSizeAnnotation := newMD.Annotations[AutoscalerMaxSizeAnnotation]
319311
if hasMinSizeAnnotation && hasMaxSizeAnnotation {
320312
minSize, err := strconv.ParseInt(minSizeString, 10, 32)
321313
if err != nil {
322-
return 0, errors.Wrapf(err, "failed to caculate MachineDeployment replicas value: could not parse the value of the %q annotation", autoscalerMinSize)
314+
return 0, errors.Wrapf(err, "failed to caculate MachineDeployment replicas value: could not parse the value of the %q annotation", AutoscalerMinSizeAnnotation)
323315
}
324316
maxSize, err := strconv.ParseInt(maxSizeString, 10, 32)
325317
if err != nil {
326-
return 0, errors.Wrapf(err, "failed to caculate MachineDeployment replicas value: could not parse the value of the %q annotation", autoscalerMaxSize)
318+
return 0, errors.Wrapf(err, "failed to caculate MachineDeployment replicas value: could not parse the value of the %q annotation", AutoscalerMaxSizeAnnotation)
327319
}
328320

329321
// If it's a new MachineDeployment => Use the min size.
330322
// Note: This will result in a scale up to get into the range where autoscaler takes over.
331323
if oldMD == nil {
332324
if !dryRun {
333-
log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (MD is a new MD)", minSize, autoscalerMinSize))
325+
log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (MD is a new MD)", minSize, AutoscalerMinSizeAnnotation))
334326
}
335327
return int32(minSize), nil
336328
}
@@ -344,21 +336,21 @@ func calculateMachineDeploymentReplicas(ctx context.Context, oldMD *MachineDeplo
344336
// We only have this handling to be 100% safe against panics.
345337
case oldMD.Spec.Replicas == nil:
346338
if !dryRun {
347-
log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MD didn't have replicas set)", minSize, autoscalerMinSize))
339+
log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MD didn't have replicas set)", minSize, AutoscalerMinSizeAnnotation))
348340
}
349341
return int32(minSize), nil
350342
// If the old MachineDeployment replicas are lower than min size => Use the min size.
351343
// Note: This will result in a scale up to get into the range where autoscaler takes over.
352344
case *oldMD.Spec.Replicas < int32(minSize):
353345
if !dryRun {
354-
log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MD had replicas below min size)", minSize, autoscalerMinSize))
346+
log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MD had replicas below min size)", minSize, AutoscalerMinSizeAnnotation))
355347
}
356348
return int32(minSize), nil
357349
// If the old MachineDeployment replicas are higher than max size => Use the max size.
358350
// Note: This will result in a scale down to get into the range where autoscaler takes over.
359351
case *oldMD.Spec.Replicas > int32(maxSize):
360352
if !dryRun {
361-
log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MD had replicas above max size)", maxSize, autoscalerMaxSize))
353+
log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MD had replicas above max size)", maxSize, AutoscalerMaxSizeAnnotation))
362354
}
363355
return int32(maxSize), nil
364356
// If the old MachineDeployment replicas are between min and max size => Keep the current value.

‎api/v1beta1/machinedeployment_webhook_test.go

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
105105
newMD: &MachineDeployment{
106106
ObjectMeta: metav1.ObjectMeta{
107107
Annotations: map[string]string{
108-
autoscalerMinSize: "3",
108+
AutoscalerMinSizeAnnotation: "3",
109109
},
110110
},
111111
},
@@ -116,7 +116,7 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
116116
newMD: &MachineDeployment{
117117
ObjectMeta: metav1.ObjectMeta{
118118
Annotations: map[string]string{
119-
autoscalerMaxSize: "7",
119+
AutoscalerMaxSizeAnnotation: "7",
120120
},
121121
},
122122
},
@@ -127,8 +127,8 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
127127
newMD: &MachineDeployment{
128128
ObjectMeta: metav1.ObjectMeta{
129129
Annotations: map[string]string{
130-
autoscalerMinSize: "abc",
131-
autoscalerMaxSize: "7",
130+
AutoscalerMinSizeAnnotation: "abc",
131+
AutoscalerMaxSizeAnnotation: "7",
132132
},
133133
},
134134
},
@@ -139,8 +139,8 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
139139
newMD: &MachineDeployment{
140140
ObjectMeta: metav1.ObjectMeta{
141141
Annotations: map[string]string{
142-
autoscalerMinSize: "3",
143-
autoscalerMaxSize: "abc",
142+
AutoscalerMinSizeAnnotation: "3",
143+
AutoscalerMaxSizeAnnotation: "abc",
144144
},
145145
},
146146
},
@@ -151,8 +151,8 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
151151
newMD: &MachineDeployment{
152152
ObjectMeta: metav1.ObjectMeta{
153153
Annotations: map[string]string{
154-
autoscalerMinSize: "3",
155-
autoscalerMaxSize: "7",
154+
AutoscalerMinSizeAnnotation: "3",
155+
AutoscalerMaxSizeAnnotation: "7",
156156
},
157157
},
158158
},
@@ -163,8 +163,8 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
163163
newMD: &MachineDeployment{
164164
ObjectMeta: metav1.ObjectMeta{
165165
Annotations: map[string]string{
166-
autoscalerMinSize: "3",
167-
autoscalerMaxSize: "7",
166+
AutoscalerMinSizeAnnotation: "3",
167+
AutoscalerMaxSizeAnnotation: "7",
168168
},
169169
},
170170
},
@@ -176,8 +176,8 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
176176
newMD: &MachineDeployment{
177177
ObjectMeta: metav1.ObjectMeta{
178178
Annotations: map[string]string{
179-
autoscalerMinSize: "3",
180-
autoscalerMaxSize: "7",
179+
AutoscalerMinSizeAnnotation: "3",
180+
AutoscalerMaxSizeAnnotation: "7",
181181
},
182182
},
183183
},
@@ -193,8 +193,8 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
193193
newMD: &MachineDeployment{
194194
ObjectMeta: metav1.ObjectMeta{
195195
Annotations: map[string]string{
196-
autoscalerMinSize: "3",
197-
autoscalerMaxSize: "7",
196+
AutoscalerMinSizeAnnotation: "3",
197+
AutoscalerMaxSizeAnnotation: "7",
198198
},
199199
},
200200
},
@@ -210,8 +210,8 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
210210
newMD: &MachineDeployment{
211211
ObjectMeta: metav1.ObjectMeta{
212212
Annotations: map[string]string{
213-
autoscalerMinSize: "3",
214-
autoscalerMaxSize: "7",
213+
AutoscalerMinSizeAnnotation: "3",
214+
AutoscalerMaxSizeAnnotation: "7",
215215
},
216216
},
217217
},

0 commit comments

Comments
 (0)