Skip to content

Commit 49a2804

Browse files
awesomenixawesomenix
committed
Add support for TerminateNotificationTimeout in machinepools
- VMSS supports terminate notifications through scheduled events - We can use these signals to be able to cordon and drain VMs during upgrade - Add new option to specify configurable timeout per machinepool - Webhook validation - Unittests
1 parent 92164bb commit 49a2804

File tree

7 files changed

+266
-16
lines changed

7 files changed

+266
-16
lines changed

‎cloud/services/scalesets/vmss.go‎

Lines changed: 34 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -38,22 +38,23 @@ import (
3838
// Spec input specification for Get/CreateOrUpdate/Delete calls
3939
type (
4040
Spec struct {
41-
Name string
42-
ResourceGroup string
43-
Location string
44-
ClusterName string
45-
MachinePoolName string
46-
Sku string
47-
Capacity int64
48-
SSHKeyData string
49-
Image *infrav1.Image
50-
OSDisk infrav1.OSDisk
51-
DataDisks []infrav1.DataDisk
52-
CustomData string
53-
SubnetID string
54-
PublicLoadBalancerName string
55-
AdditionalTags infrav1.Tags
56-
AcceleratedNetworking *bool
41+
Name string
42+
ResourceGroup string
43+
Location string
44+
ClusterName string
45+
MachinePoolName string
46+
Sku string
47+
Capacity int64
48+
SSHKeyData string
49+
Image *infrav1.Image
50+
OSDisk infrav1.OSDisk
51+
DataDisks []infrav1.DataDisk
52+
CustomData string
53+
SubnetID string
54+
PublicLoadBalancerName string
55+
AdditionalTags infrav1.Tags
56+
AcceleratedNetworking *bool
57+
TerminateNotificationTimeout *int
5758
}
5859
)
5960

@@ -195,6 +196,23 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error {
195196
},
196197
}
197198

199+
if vmssSpec.TerminateNotificationTimeout != nil {
200+
vmss.VirtualMachineProfile.ScheduledEventsProfile = &compute.ScheduledEventsProfile{
201+
TerminateNotificationProfile: &compute.TerminateNotificationProfile{
202+
Enable: to.BoolPtr(true),
203+
NotBeforeTimeout: to.StringPtr(fmt.Sprintf("PT%dM", *vmssSpec.TerminateNotificationTimeout)),
204+
},
205+
}
206+
// Once we have scheduled events termination notification we can switch upgrade policy to be rolling
207+
vmss.VirtualMachineScaleSetProperties.UpgradePolicy = &compute.UpgradePolicy{
208+
// Prefer rolling upgrade compared to Automatic (which updates all instances at same time)
209+
Mode: compute.UpgradeModeRolling,
210+
// We need to set the rolling upgrade policy based on user defined values
211+
// for now lets stick to defaults, future PR will inlcude the configurability
212+
// RollingUpgradePolicy: &compute.RollingUpgradePolicy{},
213+
}
214+
}
215+
198216
_, err = s.Client.Get(ctx, vmssSpec.ResourceGroup, vmssSpec.Name)
199217
if !azure.ResourceNotFound(err) {
200218
if err != nil {

‎cloud/services/scalesets/vmss_test.go‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -895,6 +895,156 @@ func TestService_Reconcile(t *testing.T) {
895895
g.Expect(err).To(gomega.MatchError("vm memory should be bigger or equal to at least 2Gi"))
896896
},
897897
},
898+
{
899+
Name: "WithTerminateNotificationTimeout",
900+
SpecFactory: func(g *gomega.GomegaWithT, scope *scope.ClusterScope, mpScope *scope.MachinePoolScope) interface{} {
901+
return &Spec{
902+
Name: mpScope.Name(),
903+
ResourceGroup: scope.AzureCluster.Spec.ResourceGroup,
904+
Location: scope.AzureCluster.Spec.Location,
905+
ClusterName: scope.Cluster.Name,
906+
SubnetID: scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID,
907+
PublicLoadBalancerName: scope.Cluster.Name,
908+
MachinePoolName: mpScope.Name(),
909+
Sku: "skuName",
910+
Capacity: 2,
911+
SSHKeyData: "sshKeyData",
912+
OSDisk: infrav1.OSDisk{
913+
OSType: "Linux",
914+
DiskSizeGB: 120,
915+
ManagedDisk: infrav1.ManagedDisk{
916+
StorageAccountType: "accountType",
917+
},
918+
},
919+
Image: &infrav1.Image{
920+
ID: to.StringPtr("image"),
921+
},
922+
CustomData: "customData",
923+
TerminateNotificationTimeout: to.IntPtr(7),
924+
}
925+
},
926+
Setup: func(ctx context.Context, g *gomega.GomegaWithT, svc *Service, scope *scope.ClusterScope, mpScope *scope.MachinePoolScope, spec *Spec) *gomock.Controller {
927+
mockCtrl := gomock.NewController(t)
928+
vmssMock := mock_scalesets.NewMockClient(mockCtrl)
929+
svc.Client = vmssMock
930+
skus := []compute.ResourceSku{
931+
{
932+
Name: to.StringPtr("skuName"),
933+
Kind: to.StringPtr(string(resourceskus.VirtualMachines)),
934+
Locations: &[]string{
935+
"fake-location",
936+
},
937+
LocationInfo: &[]compute.ResourceSkuLocationInfo{
938+
{
939+
Location: to.StringPtr("fake-location"),
940+
Zones: &[]string{"1"},
941+
},
942+
},
943+
Capabilities: &[]compute.ResourceSkuCapabilities{
944+
{
945+
Name: to.StringPtr(resourceskus.VCPUs),
946+
Value: to.StringPtr("4"),
947+
},
948+
{
949+
Name: to.StringPtr(resourceskus.MemoryGB),
950+
Value: to.StringPtr("8"),
951+
},
952+
{
953+
Name: to.StringPtr(resourceskus.AcceleratedNetworking),
954+
Value: to.StringPtr(string(resourceskus.CapabilitySupported)),
955+
},
956+
},
957+
},
958+
}
959+
resourceSkusCache := resourceskus.NewStaticCache(skus)
960+
961+
svc.ResourceSKUCache = resourceSkusCache
962+
lbMock := mock_loadbalancers.NewMockClient(mockCtrl)
963+
svc.LoadBalancersClient = lbMock
964+
965+
storageProfile, err := svc.generateStorageProfile(ctx, *spec, resourceskus.SKU(skus[0]))
966+
g.Expect(err).ToNot(gomega.HaveOccurred())
967+
968+
vmss := compute.VirtualMachineScaleSet{
969+
Location: to.StringPtr(scope.Location()),
970+
Tags: map[string]*string{
971+
"Name": to.StringPtr("capz-mp-0"),
972+
"kubernetes.io_cluster_capz-mp-0": to.StringPtr("owned"),
973+
"sigs.k8s.io_cluster-api-provider-azure_cluster_test-cluster": to.StringPtr("owned"),
974+
"sigs.k8s.io_cluster-api-provider-azure_role": to.StringPtr("node"),
975+
},
976+
Sku: &compute.Sku{
977+
Name: to.StringPtr(spec.Sku),
978+
Tier: to.StringPtr("Standard"),
979+
Capacity: to.Int64Ptr(spec.Capacity),
980+
},
981+
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
982+
UpgradePolicy: &compute.UpgradePolicy{
983+
Mode: compute.UpgradeModeRolling,
984+
},
985+
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{
986+
OsProfile: &compute.VirtualMachineScaleSetOSProfile{
987+
ComputerNamePrefix: to.StringPtr(spec.Name),
988+
AdminUsername: to.StringPtr(azure.DefaultUserName),
989+
CustomData: to.StringPtr(spec.CustomData),
990+
LinuxConfiguration: &compute.LinuxConfiguration{
991+
SSH: &compute.SSHConfiguration{
992+
PublicKeys: &[]compute.SSHPublicKey{
993+
{
994+
Path: to.StringPtr(fmt.Sprintf("/home/%s/.ssh/authorized_keys", azure.DefaultUserName)),
995+
KeyData: to.StringPtr(spec.SSHKeyData),
996+
},
997+
},
998+
},
999+
DisablePasswordAuthentication: to.BoolPtr(true),
1000+
},
1001+
},
1002+
StorageProfile: storageProfile,
1003+
NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{
1004+
NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{
1005+
{
1006+
Name: to.StringPtr(spec.Name + "-netconfig"),
1007+
VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{
1008+
Primary: to.BoolPtr(true),
1009+
EnableAcceleratedNetworking: to.BoolPtr(true),
1010+
EnableIPForwarding: to.BoolPtr(true),
1011+
IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{
1012+
{
1013+
Name: to.StringPtr(spec.Name + "-ipconfig"),
1014+
VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{
1015+
Subnet: &compute.APIEntityReference{
1016+
ID: to.StringPtr(scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID),
1017+
},
1018+
Primary: to.BoolPtr(true),
1019+
PrivateIPAddressVersion: compute.IPv4,
1020+
LoadBalancerBackendAddressPools: &[]compute.SubResource{{ID: to.StringPtr("cluster-name-outboundBackendPool")}},
1021+
},
1022+
},
1023+
},
1024+
},
1025+
},
1026+
},
1027+
},
1028+
ScheduledEventsProfile: &compute.ScheduledEventsProfile{
1029+
TerminateNotificationProfile: &compute.TerminateNotificationProfile{
1030+
Enable: to.BoolPtr(true),
1031+
NotBeforeTimeout: to.StringPtr("PT7M"),
1032+
},
1033+
},
1034+
},
1035+
},
1036+
}
1037+
1038+
lbMock.EXPECT().Get(gomock.Any(), scope.AzureCluster.Spec.ResourceGroup, spec.ClusterName).Return(getFakeNodeOutboundLoadBalancer(), nil)
1039+
vmssMock.EXPECT().Get(gomock.Any(), scope.AzureCluster.Spec.ResourceGroup, spec.Name).Return(compute.VirtualMachineScaleSet{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 404}, "Not found"))
1040+
vmssMock.EXPECT().CreateOrUpdate(gomock.Any(), scope.AzureCluster.Spec.ResourceGroup, spec.Name, gomockinternal.DiffEq(vmss)).Return(nil)
1041+
1042+
return mockCtrl
1043+
},
1044+
Expect: func(ctx context.Context, g *gomega.GomegaWithT, err error) {
1045+
g.Expect(err).ToNot(gomega.HaveOccurred())
1046+
},
1047+
},
8981048
}
8991049

9001050
for _, c := range cases {

‎config/crd/bases/exp.infrastructure.cluster.x-k8s.io_azuremachinepools.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,11 @@ spec:
269269
description: SSHPublicKey is the SSH public key string base64
270270
encoded to add to a Virtual Machine
271271
type: string
272+
terminateNotificationTimeout:
273+
description: TerminateNotificationTimeout enables or disables
274+
VMSS scheduled events termination notification with specified
275+
timeout allowed values are between 5 and 15 (mins)
276+
type: integer
272277
vmSize:
273278
description: VMSize is the size of the Virtual Machine to build.
274279
See https://docs.microsoft.com/en-us/rest/api/compute/virtualmachines/createorupdate#virtualmachinesizetypes

‎exp/api/v1alpha3/azuremachinepool_test.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package v1alpha3_test
1919
import (
2020
"testing"
2121

22+
"github.com/Azure/go-autorest/autorest/to"
2223
"github.com/onsi/gomega"
2324

2425
infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3"
@@ -79,6 +80,53 @@ func TestAzureMachinePool_Validate(t *testing.T) {
7980
g.Expect(actual.Error()).To(gomega.ContainSubstring("You must supply a ID, Marketplace or SharedGallery image details"))
8081
},
8182
},
83+
{
84+
Name: "HasValidTerminateNotificationTimeout",
85+
Factory: func(_ *gomega.GomegaWithT) *exp.AzureMachinePool {
86+
return &exp.AzureMachinePool{
87+
Spec: exp.AzureMachinePoolSpec{
88+
Template: exp.AzureMachineTemplate{
89+
TerminateNotificationTimeout: to.IntPtr(7),
90+
},
91+
},
92+
}
93+
},
94+
Expect: func(g *gomega.GomegaWithT, actual error) {
95+
g.Expect(actual).ToNot(gomega.HaveOccurred())
96+
},
97+
},
98+
{
99+
Name: "HasInvalidMaximumTerminateNotificationTimeout",
100+
Factory: func(_ *gomega.GomegaWithT) *exp.AzureMachinePool {
101+
return &exp.AzureMachinePool{
102+
Spec: exp.AzureMachinePoolSpec{
103+
Template: exp.AzureMachineTemplate{
104+
TerminateNotificationTimeout: to.IntPtr(20),
105+
},
106+
},
107+
}
108+
},
109+
Expect: func(g *gomega.GomegaWithT, actual error) {
110+
g.Expect(actual).To(gomega.HaveOccurred())
111+
g.Expect(actual.Error()).To(gomega.ContainSubstring("Maximum timeout 15 is allowed for TerminateNotificationTimeout"))
112+
},
113+
},
114+
{
115+
Name: "HasInvalidMinimumTerminateNotificationTimeout",
116+
Factory: func(_ *gomega.GomegaWithT) *exp.AzureMachinePool {
117+
return &exp.AzureMachinePool{
118+
Spec: exp.AzureMachinePoolSpec{
119+
Template: exp.AzureMachineTemplate{
120+
TerminateNotificationTimeout: to.IntPtr(3),
121+
},
122+
},
123+
}
124+
},
125+
Expect: func(g *gomega.GomegaWithT, actual error) {
126+
g.Expect(actual).To(gomega.HaveOccurred())
127+
g.Expect(actual.Error()).To(gomega.ContainSubstring("Minimum timeout 5 is allowed for TerminateNotificationTimeout"))
128+
},
129+
},
82130
}
83131

84132
for _, c := range cases {

‎exp/api/v1alpha3/azuremachinepool_types.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,11 @@ type (
5151
// If AcceleratedNetworking is set to true with a VMSize that does not support it, Azure will return an error.
5252
// +optional
5353
AcceleratedNetworking *bool `json:"acceleratedNetworking,omitempty"`
54+
55+
// TerminateNotificationTimeout enables or disables VMSS scheduled events termination notification with specified timeout
56+
// allowed values are between 5 and 15 (mins)
57+
// +optional
58+
TerminateNotificationTimeout *int `json:"terminateNotificationTimeout,omitempty"`
5459
}
5560

5661
// AzureMachinePoolSpec defines the desired state of AzureMachinePool

‎exp/api/v1alpha3/azuremachinepool_webhook.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ limitations under the License.
1717
package v1alpha3
1818

1919
import (
20+
"errors"
21+
2022
"k8s.io/apimachinery/pkg/runtime"
2123
kerrors "k8s.io/apimachinery/pkg/util/errors"
2224
"k8s.io/apimachinery/pkg/util/validation/field"
@@ -71,6 +73,7 @@ func (amp *AzureMachinePool) ValidateDelete() error {
7173
func (amp *AzureMachinePool) Validate() error {
7274
validators := []func() error{
7375
amp.ValidateImage,
76+
amp.ValidateTerminateNotificationTimeout,
7477
}
7578

7679
var errs []error
@@ -99,3 +102,19 @@ func (amp *AzureMachinePool) ValidateImage() error {
99102
}
100103
return nil
101104
}
105+
106+
// ValidateTerminateNotificationTimeout termination notification timeout to be between 5 and 15
107+
func (amp *AzureMachinePool) ValidateTerminateNotificationTimeout() error {
108+
if amp.Spec.Template.TerminateNotificationTimeout == nil {
109+
return nil
110+
}
111+
if *amp.Spec.Template.TerminateNotificationTimeout < 5 {
112+
return errors.New("Minimum timeout 5 is allowed for TerminateNotificationTimeout")
113+
}
114+
115+
if *amp.Spec.Template.TerminateNotificationTimeout > 15 {
116+
return errors.New("Maximum timeout 15 is allowed for TerminateNotificationTimeout")
117+
}
118+
119+
return nil
120+
}

‎exp/api/v1alpha3/zz_generated.deepcopy.go‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)