Skip to content

Commit c9b9276

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #4654 from calvix/do-not-set-desired-when-updating-asg-with-external-scaler
🐛 Update ASG - do not set desired value for machinepool which have externally managed replicas
2 parents b4506f8 + 4d33ed1 commit c9b9276

File tree

6 files changed

+62
-48
lines changed

6 files changed

+62
-48
lines changed

‎exp/controllers/awsmachinepool_controller.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ import (
4949
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
5050
expclusterv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
5151
"sigs.k8s.io/cluster-api/util"
52+
"sigs.k8s.io/cluster-api/util/annotations"
5253
"sigs.k8s.io/cluster-api/util/conditions"
5354
"sigs.k8s.io/cluster-api/util/predicates"
5455
)
@@ -276,7 +277,7 @@ func (r *AWSMachinePoolReconciler) reconcileNormal(ctx context.Context, machineP
276277
return nil
277278
}
278279

279-
if scope.ReplicasExternallyManaged(machinePoolScope.MachinePool) {
280+
if annotations.ReplicasManagedByExternalAutoscaler(machinePoolScope.MachinePool) {
280281
// Set MachinePool replicas to the ASG DesiredCapacity
281282
if *machinePoolScope.MachinePool.Spec.Replicas != *asg.DesiredCapacity {
282283
machinePoolScope.Info("Setting MachinePool replicas to ASG DesiredCapacity",
@@ -504,7 +505,7 @@ func (r *AWSMachinePoolReconciler) findASG(machinePoolScope *scope.MachinePoolSc
504505
func diffASG(machinePoolScope *scope.MachinePoolScope, existingASG *expinfrav1.AutoScalingGroup) string {
505506
detectedMachinePoolSpec := machinePoolScope.MachinePool.Spec.DeepCopy()
506507

507-
if !scope.ReplicasExternallyManaged(machinePoolScope.MachinePool) {
508+
if !annotations.ReplicasManagedByExternalAutoscaler(machinePoolScope.MachinePool) {
508509
detectedMachinePoolSpec.Replicas = existingASG.DesiredCapacity
509510
}
510511
if diff := cmp.Diff(machinePoolScope.MachinePool.Spec, *detectedMachinePoolSpec); diff != "" {

‎exp/controllers/awsmachinepool_controller_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -380,7 +380,7 @@ func TestAWSMachinePoolReconciler(t *testing.T) {
380380
ec2Svc.EXPECT().ReconcileTags(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
381381

382382
ms.MachinePool.Annotations = map[string]string{
383-
scope.ReplicasManagedByAnnotation: scope.ExternalAutoscalerReplicasManagedByAnnotationValue,
383+
clusterv1.ReplicasManagedByAnnotation: "somehow-externally-managed",
384384
}
385385
ms.MachinePool.Spec.Replicas = pointer.Int32(0)
386386

@@ -908,7 +908,7 @@ func TestDiffASG(t *testing.T) {
908908
MachinePool: &expclusterv1.MachinePool{
909909
ObjectMeta: metav1.ObjectMeta{
910910
Annotations: map[string]string{
911-
scope.ReplicasManagedByAnnotation: scope.ExternalAutoscalerReplicasManagedByAnnotationValue,
911+
clusterv1.ReplicasManagedByAnnotation: "", // empty value counts as true (= externally managed)
912912
},
913913
},
914914
Spec: expclusterv1.MachinePoolSpec{

‎pkg/cloud/scope/machinepool.go

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -43,21 +43,6 @@ import (
4343
"sigs.k8s.io/cluster-api/util/patch"
4444
)
4545

46-
const (
47-
// ReplicasManagedByAnnotation is an annotation that indicates external (non-Cluster API) management of infra scaling.
48-
// The practical effect of this is that the capi "replica" count is derived from the number of observed infra machines,
49-
// instead of being a source of truth for eventual consistency.
50-
//
51-
// N.B. this is to be replaced by a direct reference to CAPI once https://github.com/kubernetes-sigs/cluster-api/pull/7107 is meged.
52-
ReplicasManagedByAnnotation = "cluster.x-k8s.io/replicas-managed-by"
53-
54-
// ExternalAutoscalerReplicasManagedByAnnotationValue is used with the "cluster.x-k8s.io/replicas-managed-by" annotation
55-
// to indicate an external autoscaler enforces replica count.
56-
//
57-
// N.B. this is to be replaced by a direct reference to CAPI once https://github.com/kubernetes-sigs/cluster-api/pull/7107 is meged.
58-
ExternalAutoscalerReplicasManagedByAnnotationValue = "external-autoscaler"
59-
)
60-
6146
// MachinePoolScope defines a scope defined around a machine and its cluster.
6247
type MachinePoolScope struct {
6348
logger.Logger
@@ -404,8 +389,3 @@ func (m *MachinePoolScope) LaunchTemplateName() string {
404389
func (m *MachinePoolScope) GetRuntimeObject() runtime.Object {
405390
return m.AWSMachinePool
406391
}
407-
408-
func ReplicasExternallyManaged(mp *expclusterv1.MachinePool) bool {
409-
val, ok := mp.Annotations[ReplicasManagedByAnnotation]
410-
return ok && val == ExternalAutoscalerReplicasManagedByAnnotationValue
411-
}

‎pkg/cloud/services/autoscaling/autoscalinggroup.go

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import (
3434
"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/converters"
3535
"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/scope"
3636
"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/record"
37+
"sigs.k8s.io/cluster-api/util/annotations"
3738
)
3839

3940
// SDKToAutoScalingGroup converts an AWS EC2 SDK AutoScalingGroup to the CAPA AutoScalingGroup type.
@@ -46,7 +47,7 @@ func (s *Service) SDKToAutoScalingGroup(v *autoscaling.Group) (*expinfrav1.AutoS
4647
MaxSize: int32(aws.Int64Value(v.MaxSize)),
4748
MinSize: int32(aws.Int64Value(v.MinSize)),
4849
CapacityRebalance: aws.BoolValue(v.CapacityRebalance),
49-
//TODO: determine what additional values go here and what else should be in the struct
50+
// TODO: determine what additional values go here and what else should be in the struct
5051
}
5152

5253
if v.VPCZoneIdentifier != nil {
@@ -177,7 +178,7 @@ func (s *Service) CreateASG(machinePoolScope *scope.MachinePoolScope) (*expinfra
177178
// Ignore the problem for externally managed clusters because MachinePool replicas will be updated to the right value automatically.
178179
if mpReplicas >= machinePoolScope.AWSMachinePool.Spec.MinSize && mpReplicas <= machinePoolScope.AWSMachinePool.Spec.MaxSize {
179180
input.DesiredCapacity = &mpReplicas
180-
} else if !scope.ReplicasExternallyManaged(machinePoolScope.MachinePool) {
181+
} else if !annotations.ReplicasManagedByExternalAutoscaler(machinePoolScope.MachinePool) {
181182
return nil, fmt.Errorf("incorrect number of replicas %d in MachinePool %v", mpReplicas, machinePoolScope.MachinePool.Name)
182183
}
183184

@@ -284,35 +285,35 @@ func (s *Service) DeleteASG(name string) error {
284285
}
285286

286287
// UpdateASG will update the ASG of a service.
287-
func (s *Service) UpdateASG(scope *scope.MachinePoolScope) error {
288-
subnetIDs, err := s.SubnetIDs(scope)
288+
func (s *Service) UpdateASG(machinePoolScope *scope.MachinePoolScope) error {
289+
subnetIDs, err := s.SubnetIDs(machinePoolScope)
289290
if err != nil {
290291
return fmt.Errorf("getting subnets for ASG: %w", err)
291292
}
292293

293294
input := &autoscaling.UpdateAutoScalingGroupInput{
294-
AutoScalingGroupName: aws.String(scope.Name()), //TODO: define dynamically - borrow logic from ec2
295-
MaxSize: aws.Int64(int64(scope.AWSMachinePool.Spec.MaxSize)),
296-
MinSize: aws.Int64(int64(scope.AWSMachinePool.Spec.MinSize)),
295+
AutoScalingGroupName: aws.String(machinePoolScope.Name()), // TODO: define dynamically - borrow logic from ec2
296+
MaxSize: aws.Int64(int64(machinePoolScope.AWSMachinePool.Spec.MaxSize)),
297+
MinSize: aws.Int64(int64(machinePoolScope.AWSMachinePool.Spec.MinSize)),
297298
VPCZoneIdentifier: aws.String(strings.Join(subnetIDs, ",")),
298-
CapacityRebalance: aws.Bool(scope.AWSMachinePool.Spec.CapacityRebalance),
299+
CapacityRebalance: aws.Bool(machinePoolScope.AWSMachinePool.Spec.CapacityRebalance),
299300
}
300301

301-
if scope.MachinePool.Spec.Replicas != nil {
302-
input.DesiredCapacity = aws.Int64(int64(*scope.MachinePool.Spec.Replicas))
302+
if machinePoolScope.MachinePool.Spec.Replicas != nil && !annotations.ReplicasManagedByExternalAutoscaler(machinePoolScope.MachinePool) {
303+
input.DesiredCapacity = aws.Int64(int64(*machinePoolScope.MachinePool.Spec.Replicas))
303304
}
304305

305-
if scope.AWSMachinePool.Spec.MixedInstancesPolicy != nil {
306-
input.MixedInstancesPolicy = createSDKMixedInstancesPolicy(scope.Name(), scope.AWSMachinePool.Spec.MixedInstancesPolicy)
306+
if machinePoolScope.AWSMachinePool.Spec.MixedInstancesPolicy != nil {
307+
input.MixedInstancesPolicy = createSDKMixedInstancesPolicy(machinePoolScope.Name(), machinePoolScope.AWSMachinePool.Spec.MixedInstancesPolicy)
307308
} else {
308309
input.LaunchTemplate = &autoscaling.LaunchTemplateSpecification{
309-
LaunchTemplateId: aws.String(scope.AWSMachinePool.Status.LaunchTemplateID),
310+
LaunchTemplateId: aws.String(machinePoolScope.AWSMachinePool.Status.LaunchTemplateID),
310311
Version: aws.String(expinfrav1.LaunchTemplateLatestVersion),
311312
}
312313
}
313314

314315
if _, err := s.ASGClient.UpdateAutoScalingGroupWithContext(context.TODO(), input); err != nil {
315-
return errors.Wrapf(err, "failed to update ASG %q", scope.Name())
316+
return errors.Wrapf(err, "failed to update ASG %q", machinePoolScope.Name())
316317
}
317318

318319
return nil

‎pkg/cloud/services/autoscaling/autoscalinggroup_test.go

Lines changed: 40 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
. "github.com/onsi/gomega"
3131
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3232
"k8s.io/apimachinery/pkg/runtime"
33+
"k8s.io/utils/pointer"
3334
"sigs.k8s.io/controller-runtime/pkg/client"
3435
"sigs.k8s.io/controller-runtime/pkg/client/fake"
3536

@@ -570,7 +571,7 @@ func TestServiceCreateASG(t *testing.T) {
570571
mps.AWSMachinePool.Spec.MaxSize = 5
571572
mps.MachinePool.Spec.Replicas = aws.Int32(1)
572573
mps.MachinePool.Annotations = map[string]string{
573-
scope.ReplicasManagedByAnnotation: scope.ExternalAutoscalerReplicasManagedByAnnotationValue,
574+
clusterv1.ReplicasManagedByAnnotation: "", // empty value counts as true (= externally managed)
574575
}
575576
},
576577
wantErr: false,
@@ -592,7 +593,7 @@ func TestServiceCreateASG(t *testing.T) {
592593
mps.AWSMachinePool.Spec.MaxSize = 5
593594
mps.MachinePool.Spec.Replicas = aws.Int32(6)
594595
mps.MachinePool.Annotations = map[string]string{
595-
scope.ReplicasManagedByAnnotation: scope.ExternalAutoscalerReplicasManagedByAnnotationValue,
596+
clusterv1.ReplicasManagedByAnnotation: "truthy",
596597
}
597598
},
598599
wantErr: false,
@@ -699,17 +700,26 @@ func TestServiceUpdateASG(t *testing.T) {
699700
machinePoolName string
700701
setupMachinePoolScope func(*scope.MachinePoolScope)
701702
wantErr bool
702-
expect func(e *mocks.MockEC2APIMockRecorder, m *mock_autoscalingiface.MockAutoScalingAPIMockRecorder)
703+
expect func(e *mocks.MockEC2APIMockRecorder, m *mock_autoscalingiface.MockAutoScalingAPIMockRecorder, g *WithT)
703704
}{
704705
{
705706
name: "should return without error if update ASG is successful",
706707
machinePoolName: "update-asg-success",
707708
wantErr: false,
708709
setupMachinePoolScope: func(mps *scope.MachinePoolScope) {
709-
mps.AWSMachinePool.Spec.Subnets = nil
710+
mps.MachinePool.Spec.Replicas = pointer.Int32(3)
711+
mps.AWSMachinePool.Spec.MinSize = 2
712+
mps.AWSMachinePool.Spec.MaxSize = 5
710713
},
711-
expect: func(e *mocks.MockEC2APIMockRecorder, m *mock_autoscalingiface.MockAutoScalingAPIMockRecorder) {
712-
m.UpdateAutoScalingGroupWithContext(context.TODO(), gomock.AssignableToTypeOf(&autoscaling.UpdateAutoScalingGroupInput{})).Return(&autoscaling.UpdateAutoScalingGroupOutput{}, nil)
714+
expect: func(e *mocks.MockEC2APIMockRecorder, m *mock_autoscalingiface.MockAutoScalingAPIMockRecorder, g *WithT) {
715+
m.UpdateAutoScalingGroupWithContext(context.TODO(), gomock.AssignableToTypeOf(&autoscaling.UpdateAutoScalingGroupInput{})).DoAndReturn(func(ctx context.Context, input *autoscaling.UpdateAutoScalingGroupInput, options ...request.Option) (*autoscaling.UpdateAutoScalingGroupOutput, error) {
716+
// CAPA should set min/max, and because there's no "externally managed" annotation, also the
717+
// "desired" number of instances
718+
g.Expect(input.MinSize).To(BeComparableTo(pointer.Int64(2)))
719+
g.Expect(input.MaxSize).To(BeComparableTo(pointer.Int64(5)))
720+
g.Expect(input.DesiredCapacity).To(BeComparableTo(pointer.Int64(3)))
721+
return &autoscaling.UpdateAutoScalingGroupOutput{}, nil
722+
})
713723
},
714724
},
715725
{
@@ -719,10 +729,31 @@ func TestServiceUpdateASG(t *testing.T) {
719729
setupMachinePoolScope: func(mps *scope.MachinePoolScope) {
720730
mps.AWSMachinePool.Spec.MixedInstancesPolicy = nil
721731
},
722-
expect: func(e *mocks.MockEC2APIMockRecorder, m *mock_autoscalingiface.MockAutoScalingAPIMockRecorder) {
732+
expect: func(e *mocks.MockEC2APIMockRecorder, m *mock_autoscalingiface.MockAutoScalingAPIMockRecorder, g *WithT) {
723733
m.UpdateAutoScalingGroupWithContext(context.TODO(), gomock.AssignableToTypeOf(&autoscaling.UpdateAutoScalingGroupInput{})).Return(nil, awserrors.NewFailedDependency("dependency failure"))
724734
},
725735
},
736+
{
737+
name: "externally managed replicas annotation",
738+
machinePoolName: "update-asg-externally-managed-replicas-annotation",
739+
wantErr: false,
740+
setupMachinePoolScope: func(mps *scope.MachinePoolScope) {
741+
mps.MachinePool.SetAnnotations(map[string]string{clusterv1.ReplicasManagedByAnnotation: "anything-that-is-not-false"})
742+
743+
mps.MachinePool.Spec.Replicas = pointer.Int32(40)
744+
mps.AWSMachinePool.Spec.MinSize = 20
745+
mps.AWSMachinePool.Spec.MaxSize = 50
746+
},
747+
expect: func(e *mocks.MockEC2APIMockRecorder, m *mock_autoscalingiface.MockAutoScalingAPIMockRecorder, g *WithT) {
748+
m.UpdateAutoScalingGroupWithContext(context.TODO(), gomock.AssignableToTypeOf(&autoscaling.UpdateAutoScalingGroupInput{})).DoAndReturn(func(ctx context.Context, input *autoscaling.UpdateAutoScalingGroupInput, options ...request.Option) (*autoscaling.UpdateAutoScalingGroupOutput, error) {
749+
// CAPA should set min/max, but not the externally managed "desired" number of instances
750+
g.Expect(input.MinSize).To(BeComparableTo(pointer.Int64(20)))
751+
g.Expect(input.MaxSize).To(BeComparableTo(pointer.Int64(50)))
752+
g.Expect(input.DesiredCapacity).To(BeNil())
753+
return &autoscaling.UpdateAutoScalingGroupOutput{}, nil
754+
})
755+
},
756+
},
726757
}
727758
for _, tt := range tests {
728759
t.Run(tt.name, func(t *testing.T) {
@@ -733,13 +764,14 @@ func TestServiceUpdateASG(t *testing.T) {
733764
g.Expect(err).ToNot(HaveOccurred())
734765
ec2Mock := mocks.NewMockEC2API(mockCtrl)
735766
asgMock := mock_autoscalingiface.NewMockAutoScalingAPI(mockCtrl)
736-
tt.expect(ec2Mock.EXPECT(), asgMock.EXPECT())
767+
tt.expect(ec2Mock.EXPECT(), asgMock.EXPECT(), g)
737768
s := NewService(clusterScope)
738769
s.ASGClient = asgMock
739770

740771
mps, err := getMachinePoolScope(fakeClient, clusterScope)
741772
g.Expect(err).ToNot(HaveOccurred())
742773
mps.AWSMachinePool.Name = tt.machinePoolName
774+
tt.setupMachinePoolScope(mps)
743775

744776
err = s.UpdateASG(mps)
745777
checkErr(tt.wantErr, err, g)

‎pkg/cloud/services/eks/nodegroup.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,10 @@ import (
3434
expinfrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/exp/api/v1beta2"
3535
"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/awserrors"
3636
"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/converters"
37-
"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/scope"
3837
"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/wait"
3938
"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/record"
4039
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
40+
"sigs.k8s.io/cluster-api/util/annotations"
4141
)
4242

4343
func (s *NodegroupService) describeNodegroup() (*eks.Nodegroup, error) {
@@ -533,7 +533,7 @@ func (s *NodegroupService) reconcileNodegroup(ctx context.Context) error {
533533
break
534534
}
535535

536-
if scope.ReplicasExternallyManaged(s.scope.MachinePool) {
536+
if annotations.ReplicasManagedByExternalAutoscaler(s.scope.MachinePool) {
537537
// Set MachinePool replicas to the node group DesiredCapacity
538538
ngDesiredCapacity := int32(aws.Int64Value(ng.ScalingConfig.DesiredSize))
539539
if *s.scope.MachinePool.Spec.Replicas != ngDesiredCapacity {

0 commit comments

Comments
 (0)