Skip to content

Add flag to disable bootstrap extension and improve error message #5509

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 22, 2025
Merged

Add flag to disable bootstrap extension and improve error message #5509

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions api/v1beta1/azuremachine_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,11 @@ type AzureMachineSpec struct {
// +optional
DisableExtensionOperations *bool `json:"disableExtensionOperations,omitempty"`

// DisableVMBootstrapExtension specifies whether the VM bootstrap extension should be disabled on the virtual machine.
// Use this setting if you want to disable only the bootstrapping extension and not all extensions.
// +optional
DisableVMBootstrapExtension *bool `json:"disableVMBootstrapExtension,omitempty"`

// VMExtensions specifies a list of extensions to be added to the virtual machine.
// +optional
VMExtensions []VMExtension `json:"vmExtensions,omitempty"`
Expand Down
7 changes: 7 additions & 0 deletions api/v1beta1/azuremachine_webhook.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,13 @@
allErrs = append(allErrs, err)
}

if err := webhookutils.ValidateImmutable(
field.NewPath("spec", "disableVMBootstrapExtension"),
old.Spec.DisableVMBootstrapExtension,
m.Spec.DisableVMBootstrapExtension); err != nil {
allErrs = append(allErrs, err)
}

Check warning on line 229 in api/v1beta1/azuremachine_webhook.go

View check run for this annotation

Codecov / codecov/patch

api/v1beta1/azuremachine_webhook.go#L228-L229 

Added lines #L228 - L229 were not covered by tests

if len(allErrs) == 0 {
return nil, nil
}
Expand Down
5 changes: 5 additions & 0 deletions api/v1beta1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion azure/defaults.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ const (

var (
// LinuxBootstrapExtensionCommand is the command the VM bootstrap extension will execute to verify Linux nodes bootstrap completes successfully.
LinuxBootstrapExtensionCommand = fmt.Sprintf("for i in $(seq 1 %d); do test -f %s && break; if [ $i -eq %d ]; then exit 1; else sleep %d; fi; done", bootstrapExtensionRetries, bootstrapSentinelFile, bootstrapExtensionRetries, bootstrapExtensionSleep)
LinuxBootstrapExtensionCommand = fmt.Sprintf("for i in $(seq 1 %d); do test -f %s && break; if [ $i -eq %d ]; then echo 'Error joining node to cluster: kubeadm init or join failed. To debug, check the cloud-init, kubelet, or other bootstrap logs: https://capz.sigs.k8s.io/self-managed/troubleshooting.html#checking-cloud-init-logs-ubuntu'; exit 1; else sleep %d; fi; done", bootstrapExtensionRetries, bootstrapSentinelFile, bootstrapExtensionRetries, bootstrapExtensionSleep)
// WindowsBootstrapExtensionCommand is the command the VM bootstrap extension will execute to verify Windows nodes bootstrap completes successfully.
WindowsBootstrapExtensionCommand = fmt.Sprintf("powershell.exe -Command \"for ($i = 0; $i -lt %d; $i++) {if (Test-Path '%s') {exit 0} else {Start-Sleep -Seconds %d}} exit -2\"",
bootstrapExtensionRetries, bootstrapSentinelFile, bootstrapExtensionSleep)
Expand Down
65 changes: 34 additions & 31 deletions azure/scope/machine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,29 +161,30 @@
// VMSpec returns the VM spec.
func (m *MachineScope) VMSpec() azure.ResourceSpecGetter {
spec := &virtualmachines.VMSpec{
Name: m.Name(),
Location: m.Location(),
ExtendedLocation: m.ExtendedLocation(),
ResourceGroup: m.NodeResourceGroup(),
ClusterName: m.ClusterName(),
Role: m.Role(),
NICIDs: m.NICIDs(),
SSHKeyData: m.AzureMachine.Spec.SSHPublicKey,
Size: m.AzureMachine.Spec.VMSize,
OSDisk: m.AzureMachine.Spec.OSDisk,
DataDisks: m.AzureMachine.Spec.DataDisks,
AvailabilitySetID: m.AvailabilitySetID(),
Zone: m.AvailabilityZone(),
Identity: m.AzureMachine.Spec.Identity,
UserAssignedIdentities: m.AzureMachine.Spec.UserAssignedIdentities,
SpotVMOptions: m.AzureMachine.Spec.SpotVMOptions,
SecurityProfile: m.AzureMachine.Spec.SecurityProfile,
DiagnosticsProfile: m.AzureMachine.Spec.Diagnostics,
DisableExtensionOperations: ptr.Deref(m.AzureMachine.Spec.DisableExtensionOperations, false),
AdditionalTags: m.AdditionalTags(),
AdditionalCapabilities: m.AzureMachine.Spec.AdditionalCapabilities,
CapacityReservationGroupID: m.GetCapacityReservationGroupID(),
ProviderID: m.ProviderID(),
Name: m.Name(),
Location: m.Location(),
ExtendedLocation: m.ExtendedLocation(),
ResourceGroup: m.NodeResourceGroup(),
ClusterName: m.ClusterName(),
Role: m.Role(),
NICIDs: m.NICIDs(),
SSHKeyData: m.AzureMachine.Spec.SSHPublicKey,
Size: m.AzureMachine.Spec.VMSize,
OSDisk: m.AzureMachine.Spec.OSDisk,
DataDisks: m.AzureMachine.Spec.DataDisks,
AvailabilitySetID: m.AvailabilitySetID(),
Zone: m.AvailabilityZone(),
Identity: m.AzureMachine.Spec.Identity,
UserAssignedIdentities: m.AzureMachine.Spec.UserAssignedIdentities,
SpotVMOptions: m.AzureMachine.Spec.SpotVMOptions,
SecurityProfile: m.AzureMachine.Spec.SecurityProfile,
DiagnosticsProfile: m.AzureMachine.Spec.Diagnostics,
DisableExtensionOperations: ptr.Deref(m.AzureMachine.Spec.DisableExtensionOperations, false),
DisableVMBootstrapExtension: ptr.Deref(m.AzureMachine.Spec.DisableVMBootstrapExtension, false),
AdditionalTags: m.AdditionalTags(),
AdditionalCapabilities: m.AzureMachine.Spec.AdditionalCapabilities,
CapacityReservationGroupID: m.GetCapacityReservationGroupID(),
ProviderID: m.ProviderID(),

Check warning on line 187 in azure/scope/machine.go

View check run for this annotation

Codecov / codecov/patch

azure/scope/machine.go#L164-L187 

Added lines #L164 - L187 were not covered by tests
}
if m.cache != nil {
spec.SKU = m.cache.VMSKU
Expand Down Expand Up @@ -400,15 +401,17 @@
})
}

cpuArchitectureType, _ := m.cache.VMSKU.GetCapability(resourceskus.CPUArchitectureType)
bootstrapExtensionSpec := azure.GetBootstrappingVMExtension(m.AzureMachine.Spec.OSDisk.OSType, m.CloudEnvironment(), m.Name(), cpuArchitectureType)
if !ptr.Deref(m.AzureMachine.Spec.DisableVMBootstrapExtension, false) {
cpuArchitectureType, _ := m.cache.VMSKU.GetCapability(resourceskus.CPUArchitectureType)
bootstrapExtensionSpec := azure.GetBootstrappingVMExtension(m.AzureMachine.Spec.OSDisk.OSType, m.CloudEnvironment(), m.Name(), cpuArchitectureType)

if bootstrapExtensionSpec != nil {
extensionSpecs = append(extensionSpecs, &vmextensions.VMExtensionSpec{
ExtensionSpec: *bootstrapExtensionSpec,
ResourceGroup: m.NodeResourceGroup(),
Location: m.Location(),
})
if bootstrapExtensionSpec != nil {
extensionSpecs = append(extensionSpecs, &vmextensions.VMExtensionSpec{
ExtensionSpec: *bootstrapExtensionSpec,
ResourceGroup: m.NodeResourceGroup(),
Location: m.Location(),
})
}
}

return extensionSpecs
Expand Down
68 changes: 68 additions & 0 deletions azure/scope/machine_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -917,6 +917,74 @@ func TestMachineScope_VMExtensionSpecs(t *testing.T) {
},
},
},
{
name: "If a custom VM extension is specified and bootstrap extension is disabled, it returns only the custom VM extension",
machineScope: MachineScope{
Machine: &clusterv1.Machine{},
AzureMachine: &infrav1.AzureMachine{
ObjectMeta: metav1.ObjectMeta{
Name: "machine-name",
},
Spec: infrav1.AzureMachineSpec{
OSDisk: infrav1.OSDisk{
OSType: "Linux",
},
DisableVMBootstrapExtension: ptr.To(true),
VMExtensions: []infrav1.VMExtension{
{
Name: "custom-vm-extension",
Publisher: "Microsoft.Azure.Extensions",
Version: "2.0",
Settings: map[string]string{
"timestamp": "1234567890",
},
ProtectedSettings: map[string]string{
"commandToExecute": "echo hello world",
},
},
},
},
},
ClusterScoper: &ClusterScope{
AzureClients: AzureClients{
EnvironmentSettings: auth.EnvironmentSettings{
Environment: azureautorest.Environment{
Name: azureautorest.PublicCloud.Name,
},
},
},
AzureCluster: &infrav1.AzureCluster{
Spec: infrav1.AzureClusterSpec{
ResourceGroup: "my-rg",
AzureClusterClassSpec: infrav1.AzureClusterClassSpec{
Location: "westus",
},
},
},
},
cache: &MachineCache{
VMSKU: resourceskus.SKU{},
},
},
want: []azure.ResourceSpecGetter{
&vmextensions.VMExtensionSpec{
ExtensionSpec: azure.ExtensionSpec{
Name: "custom-vm-extension",
VMName: "machine-name",
Publisher: "Microsoft.Azure.Extensions",
Version: "2.0",
Settings: map[string]string{
"timestamp": "1234567890",
},
ProtectedSettings: map[string]string{
"commandToExecute": "echo hello world",
},
},
ResourceGroup: "my-rg",
Location: "westus",
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
53 changes: 27 additions & 26 deletions azure/services/virtualmachines/spec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,32 +34,33 @@ import (

// VMSpec defines the specification for a Virtual Machine.
type VMSpec struct {
Name string
ResourceGroup string
Location string
ExtendedLocation *infrav1.ExtendedLocationSpec
ClusterName string
Role string
NICIDs []string
SSHKeyData string
Size string
AvailabilitySetID string
Zone string
Identity infrav1.VMIdentity
OSDisk infrav1.OSDisk
DataDisks []infrav1.DataDisk
UserAssignedIdentities []infrav1.UserAssignedIdentity
SpotVMOptions *infrav1.SpotVMOptions
SecurityProfile *infrav1.SecurityProfile
AdditionalTags infrav1.Tags
AdditionalCapabilities *infrav1.AdditionalCapabilities
DiagnosticsProfile *infrav1.Diagnostics
DisableExtensionOperations bool
CapacityReservationGroupID string
SKU resourceskus.SKU
Image *infrav1.Image
BootstrapData string
ProviderID string
Name string
ResourceGroup string
Location string
ExtendedLocation *infrav1.ExtendedLocationSpec
ClusterName string
Role string
NICIDs []string
SSHKeyData string
Size string
AvailabilitySetID string
Zone string
Identity infrav1.VMIdentity
OSDisk infrav1.OSDisk
DataDisks []infrav1.DataDisk
UserAssignedIdentities []infrav1.UserAssignedIdentity
SpotVMOptions *infrav1.SpotVMOptions
SecurityProfile *infrav1.SecurityProfile
AdditionalTags infrav1.Tags
AdditionalCapabilities *infrav1.AdditionalCapabilities
DiagnosticsProfile *infrav1.Diagnostics
DisableExtensionOperations bool
DisableVMBootstrapExtension bool
CapacityReservationGroupID string
SKU resourceskus.SKU
Image *infrav1.Image
BootstrapData string
ProviderID string
}

// ResourceName returns the name of the virtual machine.
Expand Down
5 changes: 5 additions & 0 deletions config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,11 @@ spec:
Use this setting only if VMExtensions are not supported by your image, as it disables CAPZ bootstrapping extension used for detecting Kubernetes bootstrap failure.
This may only be set to True when no extensions are configured on the virtual machine.
type: boolean
disableVMBootstrapExtension:
description: |-
DisableVMBootstrapExtension specifies whether the VM bootstrap extension should be disabled on the virtual machine.
Use this setting if you want to disable only the bootstrapping extension and not all extensions.
type: boolean
dnsServers:
description: DNSServers adds a list of DNS Server IP addresses to
the VM NICs.
Expand Down
5 changes: 5 additions & 0 deletions config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,11 @@ spec:
Use this setting only if VMExtensions are not supported by your image, as it disables CAPZ bootstrapping extension used for detecting Kubernetes bootstrap failure.
This may only be set to True when no extensions are configured on the virtual machine.
type: boolean
disableVMBootstrapExtension:
description: |-
DisableVMBootstrapExtension specifies whether the VM bootstrap extension should be disabled on the virtual machine.
Use this setting if you want to disable only the bootstrapping extension and not all extensions.
type: boolean
dnsServers:
description: DNSServers adds a list of DNS Server IP addresses
to the VM NICs.
Expand Down
Loading