diff --git a/api/v1beta1/azuremachine_types.go b/api/v1beta1/azuremachine_types.go index 04a8a70c7c6..090f24edace 100644 --- a/api/v1beta1/azuremachine_types.go +++ b/api/v1beta1/azuremachine_types.go @@ -137,6 +137,11 @@ type AzureMachineSpec struct { // +optional DisableExtensionOperations *bool `json:"disableExtensionOperations,omitempty"` + // DisableVMBootstrapExtension specifies whether the VM bootstrap extension should be disabled on the virtual machine. + // Use this setting if you want to disable only the bootstrapping extension and not all extensions. + // +optional + DisableVMBootstrapExtension *bool `json:"disableVMBootstrapExtension,omitempty"` + // VMExtensions specifies a list of extensions to be added to the virtual machine. // +optional VMExtensions []VMExtension `json:"vmExtensions,omitempty"` diff --git a/api/v1beta1/azuremachine_webhook.go b/api/v1beta1/azuremachine_webhook.go index d3f892da778..246d160da24 100644 --- a/api/v1beta1/azuremachine_webhook.go +++ b/api/v1beta1/azuremachine_webhook.go @@ -221,6 +221,13 @@ func (mw *azureMachineWebhook) ValidateUpdate(_ context.Context, oldObj, newObj allErrs = append(allErrs, err) } + if err := webhookutils.ValidateImmutable( + field.NewPath("spec", "disableVMBootstrapExtension"), + old.Spec.DisableVMBootstrapExtension, + m.Spec.DisableVMBootstrapExtension); err != nil { + allErrs = append(allErrs, err) + } + if len(allErrs) == 0 { return nil, nil } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index b523662a8d0..f27a8bf70aa 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -938,6 +938,11 @@ func (in *AzureMachineSpec) DeepCopyInto(out *AzureMachineSpec) { *out = new(bool) **out = **in } + if in.DisableVMBootstrapExtension != nil { + in, out := &in.DisableVMBootstrapExtension, &out.DisableVMBootstrapExtension + *out = new(bool) + **out = **in + } if in.VMExtensions != nil { in, out := &in.VMExtensions, &out.VMExtensions *out = make([]VMExtension, len(*in)) diff --git a/azure/defaults.go b/azure/defaults.go index 0f86619c7d1..02e5508fa5c 100644 --- a/azure/defaults.go +++ b/azure/defaults.go @@ -111,7 +111,7 @@ const ( var ( // LinuxBootstrapExtensionCommand is the command the VM bootstrap extension will execute to verify Linux nodes bootstrap completes successfully. - LinuxBootstrapExtensionCommand = fmt.Sprintf("for i in $(seq 1 %d); do test -f %s && break; if [ $i -eq %d ]; then exit 1; else sleep %d; fi; done", bootstrapExtensionRetries, bootstrapSentinelFile, bootstrapExtensionRetries, bootstrapExtensionSleep) + LinuxBootstrapExtensionCommand = fmt.Sprintf("for i in $(seq 1 %d); do test -f %s && break; if [ $i -eq %d ]; then echo 'Error joining node to cluster: kubeadm init or join failed. To debug, check the cloud-init, kubelet, or other bootstrap logs: https://capz.sigs.k8s.io/self-managed/troubleshooting.html#checking-cloud-init-logs-ubuntu'; exit 1; else sleep %d; fi; done", bootstrapExtensionRetries, bootstrapSentinelFile, bootstrapExtensionRetries, bootstrapExtensionSleep) // WindowsBootstrapExtensionCommand is the command the VM bootstrap extension will execute to verify Windows nodes bootstrap completes successfully. WindowsBootstrapExtensionCommand = fmt.Sprintf("powershell.exe -Command \"for ($i = 0; $i -lt %d; $i++) {if (Test-Path '%s') {exit 0} else {Start-Sleep -Seconds %d}} exit -2\"", bootstrapExtensionRetries, bootstrapSentinelFile, bootstrapExtensionSleep) diff --git a/azure/scope/machine.go b/azure/scope/machine.go index d6a96964f36..67635826d8b 100644 --- a/azure/scope/machine.go +++ b/azure/scope/machine.go @@ -161,29 +161,30 @@ func (m *MachineScope) InitMachineCache(ctx context.Context) error { // VMSpec returns the VM spec. func (m *MachineScope) VMSpec() azure.ResourceSpecGetter { spec := &virtualmachines.VMSpec{ - Name: m.Name(), - Location: m.Location(), - ExtendedLocation: m.ExtendedLocation(), - ResourceGroup: m.NodeResourceGroup(), - ClusterName: m.ClusterName(), - Role: m.Role(), - NICIDs: m.NICIDs(), - SSHKeyData: m.AzureMachine.Spec.SSHPublicKey, - Size: m.AzureMachine.Spec.VMSize, - OSDisk: m.AzureMachine.Spec.OSDisk, - DataDisks: m.AzureMachine.Spec.DataDisks, - AvailabilitySetID: m.AvailabilitySetID(), - Zone: m.AvailabilityZone(), - Identity: m.AzureMachine.Spec.Identity, - UserAssignedIdentities: m.AzureMachine.Spec.UserAssignedIdentities, - SpotVMOptions: m.AzureMachine.Spec.SpotVMOptions, - SecurityProfile: m.AzureMachine.Spec.SecurityProfile, - DiagnosticsProfile: m.AzureMachine.Spec.Diagnostics, - DisableExtensionOperations: ptr.Deref(m.AzureMachine.Spec.DisableExtensionOperations, false), - AdditionalTags: m.AdditionalTags(), - AdditionalCapabilities: m.AzureMachine.Spec.AdditionalCapabilities, - CapacityReservationGroupID: m.GetCapacityReservationGroupID(), - ProviderID: m.ProviderID(), + Name: m.Name(), + Location: m.Location(), + ExtendedLocation: m.ExtendedLocation(), + ResourceGroup: m.NodeResourceGroup(), + ClusterName: m.ClusterName(), + Role: m.Role(), + NICIDs: m.NICIDs(), + SSHKeyData: m.AzureMachine.Spec.SSHPublicKey, + Size: m.AzureMachine.Spec.VMSize, + OSDisk: m.AzureMachine.Spec.OSDisk, + DataDisks: m.AzureMachine.Spec.DataDisks, + AvailabilitySetID: m.AvailabilitySetID(), + Zone: m.AvailabilityZone(), + Identity: m.AzureMachine.Spec.Identity, + UserAssignedIdentities: m.AzureMachine.Spec.UserAssignedIdentities, + SpotVMOptions: m.AzureMachine.Spec.SpotVMOptions, + SecurityProfile: m.AzureMachine.Spec.SecurityProfile, + DiagnosticsProfile: m.AzureMachine.Spec.Diagnostics, + DisableExtensionOperations: ptr.Deref(m.AzureMachine.Spec.DisableExtensionOperations, false), + DisableVMBootstrapExtension: ptr.Deref(m.AzureMachine.Spec.DisableVMBootstrapExtension, false), + AdditionalTags: m.AdditionalTags(), + AdditionalCapabilities: m.AzureMachine.Spec.AdditionalCapabilities, + CapacityReservationGroupID: m.GetCapacityReservationGroupID(), + ProviderID: m.ProviderID(), } if m.cache != nil { spec.SKU = m.cache.VMSKU @@ -400,15 +401,17 @@ func (m *MachineScope) VMExtensionSpecs() []azure.ResourceSpecGetter { }) } - cpuArchitectureType, _ := m.cache.VMSKU.GetCapability(resourceskus.CPUArchitectureType) - bootstrapExtensionSpec := azure.GetBootstrappingVMExtension(m.AzureMachine.Spec.OSDisk.OSType, m.CloudEnvironment(), m.Name(), cpuArchitectureType) + if !ptr.Deref(m.AzureMachine.Spec.DisableVMBootstrapExtension, false) { + cpuArchitectureType, _ := m.cache.VMSKU.GetCapability(resourceskus.CPUArchitectureType) + bootstrapExtensionSpec := azure.GetBootstrappingVMExtension(m.AzureMachine.Spec.OSDisk.OSType, m.CloudEnvironment(), m.Name(), cpuArchitectureType) - if bootstrapExtensionSpec != nil { - extensionSpecs = append(extensionSpecs, &vmextensions.VMExtensionSpec{ - ExtensionSpec: *bootstrapExtensionSpec, - ResourceGroup: m.NodeResourceGroup(), - Location: m.Location(), - }) + if bootstrapExtensionSpec != nil { + extensionSpecs = append(extensionSpecs, &vmextensions.VMExtensionSpec{ + ExtensionSpec: *bootstrapExtensionSpec, + ResourceGroup: m.NodeResourceGroup(), + Location: m.Location(), + }) + } } return extensionSpecs diff --git a/azure/scope/machine_test.go b/azure/scope/machine_test.go index cfbb44b3588..30c4127ca1b 100644 --- a/azure/scope/machine_test.go +++ b/azure/scope/machine_test.go @@ -917,6 +917,74 @@ func TestMachineScope_VMExtensionSpecs(t *testing.T) { }, }, }, + { + name: "If a custom VM extension is specified and bootstrap extension is disabled, it returns only the custom VM extension", + machineScope: MachineScope{ + Machine: &clusterv1.Machine{}, + AzureMachine: &infrav1.AzureMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machine-name", + }, + Spec: infrav1.AzureMachineSpec{ + OSDisk: infrav1.OSDisk{ + OSType: "Linux", + }, + DisableVMBootstrapExtension: ptr.To(true), + VMExtensions: []infrav1.VMExtension{ + { + Name: "custom-vm-extension", + Publisher: "Microsoft.Azure.Extensions", + Version: "2.0", + Settings: map[string]string{ + "timestamp": "1234567890", + }, + ProtectedSettings: map[string]string{ + "commandToExecute": "echo hello world", + }, + }, + }, + }, + }, + ClusterScoper: &ClusterScope{ + AzureClients: AzureClients{ + EnvironmentSettings: auth.EnvironmentSettings{ + Environment: azureautorest.Environment{ + Name: azureautorest.PublicCloud.Name, + }, + }, + }, + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + ResourceGroup: "my-rg", + AzureClusterClassSpec: infrav1.AzureClusterClassSpec{ + Location: "westus", + }, + }, + }, + }, + cache: &MachineCache{ + VMSKU: resourceskus.SKU{}, + }, + }, + want: []azure.ResourceSpecGetter{ + &vmextensions.VMExtensionSpec{ + ExtensionSpec: azure.ExtensionSpec{ + Name: "custom-vm-extension", + VMName: "machine-name", + Publisher: "Microsoft.Azure.Extensions", + Version: "2.0", + Settings: map[string]string{ + "timestamp": "1234567890", + }, + ProtectedSettings: map[string]string{ + "commandToExecute": "echo hello world", + }, + }, + ResourceGroup: "my-rg", + Location: "westus", + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/azure/services/virtualmachines/spec.go b/azure/services/virtualmachines/spec.go index 7f626366edc..70ccced92f8 100644 --- a/azure/services/virtualmachines/spec.go +++ b/azure/services/virtualmachines/spec.go @@ -34,32 +34,33 @@ import ( // VMSpec defines the specification for a Virtual Machine. type VMSpec struct { - Name string - ResourceGroup string - Location string - ExtendedLocation *infrav1.ExtendedLocationSpec - ClusterName string - Role string - NICIDs []string - SSHKeyData string - Size string - AvailabilitySetID string - Zone string - Identity infrav1.VMIdentity - OSDisk infrav1.OSDisk - DataDisks []infrav1.DataDisk - UserAssignedIdentities []infrav1.UserAssignedIdentity - SpotVMOptions *infrav1.SpotVMOptions - SecurityProfile *infrav1.SecurityProfile - AdditionalTags infrav1.Tags - AdditionalCapabilities *infrav1.AdditionalCapabilities - DiagnosticsProfile *infrav1.Diagnostics - DisableExtensionOperations bool - CapacityReservationGroupID string - SKU resourceskus.SKU - Image *infrav1.Image - BootstrapData string - ProviderID string + Name string + ResourceGroup string + Location string + ExtendedLocation *infrav1.ExtendedLocationSpec + ClusterName string + Role string + NICIDs []string + SSHKeyData string + Size string + AvailabilitySetID string + Zone string + Identity infrav1.VMIdentity + OSDisk infrav1.OSDisk + DataDisks []infrav1.DataDisk + UserAssignedIdentities []infrav1.UserAssignedIdentity + SpotVMOptions *infrav1.SpotVMOptions + SecurityProfile *infrav1.SecurityProfile + AdditionalTags infrav1.Tags + AdditionalCapabilities *infrav1.AdditionalCapabilities + DiagnosticsProfile *infrav1.Diagnostics + DisableExtensionOperations bool + DisableVMBootstrapExtension bool + CapacityReservationGroupID string + SKU resourceskus.SKU + Image *infrav1.Image + BootstrapData string + ProviderID string } // ResourceName returns the name of the virtual machine. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml index 088d7358d5c..df1d1983160 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml @@ -248,6 +248,11 @@ spec: Use this setting only if VMExtensions are not supported by your image, as it disables CAPZ bootstrapping extension used for detecting Kubernetes bootstrap failure. This may only be set to True when no extensions are configured on the virtual machine. type: boolean + disableVMBootstrapExtension: + description: |- + DisableVMBootstrapExtension specifies whether the VM bootstrap extension should be disabled on the virtual machine. + Use this setting if you want to disable only the bootstrapping extension and not all extensions. + type: boolean dnsServers: description: DNSServers adds a list of DNS Server IP addresses to the VM NICs. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml index 54d3ef25afc..1f8e95256b6 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml @@ -259,6 +259,11 @@ spec: Use this setting only if VMExtensions are not supported by your image, as it disables CAPZ bootstrapping extension used for detecting Kubernetes bootstrap failure. This may only be set to True when no extensions are configured on the virtual machine. type: boolean + disableVMBootstrapExtension: + description: |- + DisableVMBootstrapExtension specifies whether the VM bootstrap extension should be disabled on the virtual machine. + Use this setting if you want to disable only the bootstrapping extension and not all extensions. + type: boolean dnsServers: description: DNSServers adds a list of DNS Server IP addresses to the VM NICs.