From e86570cffa88cbff5e6db7e56e2aa0c51fc5e49a Mon Sep 17 00:00:00 2001 From: Patrick Dillon Date: Mon, 17 Feb 2025 13:14:59 -0500 Subject: [PATCH 01/10] UPSTREAM: 5532: Add Azure Stack as a valid environment Adds AzureStack as a valid cloud environment. The value "HybridEnvironment" is the value provided by the Azure autorest package. It would be possible to have a different user-facing value, such as AzureStackCloud, but internally within the code it is necessary to check for the value "HybridEnvironment" returned by autorest. This commit opts to use a single value, rather than separate user-facing and internal values. --- api/v1beta1/types_class.go | 2 ++ azure/defaults.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/api/v1beta1/types_class.go b/api/v1beta1/types_class.go index 452eab517f8..2646b12cb8d 100644 --- a/api/v1beta1/types_class.go +++ b/api/v1beta1/types_class.go @@ -48,6 +48,7 @@ type AzureClusterClassSpec struct { // - GermanCloud: "AzureGermanCloud" // - PublicCloud: "AzurePublicCloud" // - USGovernmentCloud: "AzureUSGovernmentCloud" + // - StackCloud: "HybridEnvironment" // // Note that values other than the default must also be accompanied by corresponding changes to the // aso-controller-settings Secret to configure ASO to refer to the non-Public cloud. ASO currently does @@ -186,6 +187,7 @@ type AzureManagedControlPlaneClassSpec struct { // - PublicCloud: "AzurePublicCloud" // - USGovernmentCloud: "AzureUSGovernmentCloud" // + // // Note that values other than the default must also be accompanied by corresponding changes to the // aso-controller-settings Secret to configure ASO to refer to the non-Public cloud. ASO currently does // not support referring to multiple different clouds in a single installation. The following fields must diff --git a/azure/defaults.go b/azure/defaults.go index 0f86619c7d1..3d2c66a8fb6 100644 --- a/azure/defaults.go +++ b/azure/defaults.go @@ -44,6 +44,8 @@ const ( ChinaCloudName = "AzureChinaCloud" // USGovernmentCloudName is the name of the Azure US Government cloud. USGovernmentCloudName = "AzureUSGovernmentCloud" + // StackCloudName is the name for Azure Stack hybrid cloud environments. + StackCloudName = "HybridEnvironment" ) const ( From 31bc41f9d814364b34092803248ff3d84464ad61 Mon Sep 17 00:00:00 2001 From: Patrick Dillon Date: Mon, 17 Feb 2025 13:27:02 -0500 Subject: [PATCH 02/10] UPSTREAM: 5532: Add ARMEndpoint to Cluster Class Adds the ARMEndpoint field for specifying the ARM Resource Manager Endpoint for use with Azure Stack deployments. The endpoint is used to configure the environment as well as manage resources. --- api/v1beta1/types_class.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/v1beta1/types_class.go b/api/v1beta1/types_class.go index 2646b12cb8d..b9690694a91 100644 --- a/api/v1beta1/types_class.go +++ b/api/v1beta1/types_class.go @@ -78,6 +78,12 @@ type AzureClusterClassSpec struct { // See: https://learn.microsoft.com/azure/reliability/availability-zones-overview // +optional FailureDomains clusterv1.FailureDomains `json:"failureDomains,omitempty"` + + // ARMEndpoint specifies a URL for the ARM Resource Manager endpoint. + // It may only be specified when the AzureEnvironment is set to AzureStackCloud, + // in which case it is required. + // +optional + ARMEndpoint string `json:"armEndpoint,omitempty"` } // AzureManagedControlPlaneClassSpec defines the AzureManagedControlPlane properties that may be shared across several azure managed control planes. From 37a57be2b5ad5b7ddc9ae54992f30b3e80a17e38 Mon Sep 17 00:00:00 2001 From: Patrick Dillon Date: Mon, 17 Feb 2025 13:46:05 -0500 Subject: [PATCH 03/10] UPSTREAM: 5532: Populate AzureClients & Authorizer for Azure Stack Uses ARMEndpoint from Cluster scope to configure Azure Stack settings for Azure Client and Authorizer, which will be used to configure ARM options for the V2 SDK. --- azure/scope/clients.go | 8 +++++--- azure/scope/cluster.go | 3 ++- azure/scope/managedcontrolplane.go | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/azure/scope/clients.go b/azure/scope/clients.go index b766b066747..e472cc8925c 100644 --- a/azure/scope/clients.go +++ b/azure/scope/clients.go @@ -81,12 +81,12 @@ func (c *AzureClients) HashKey() string { return base64.URLEncoding.EncodeToString(hasher.Sum(nil)) } -func (c *AzureClients) setCredentialsWithProvider(ctx context.Context, subscriptionID, environmentName string, credentialsProvider CredentialsProvider) error { +func (c *AzureClients) setCredentialsWithProvider(ctx context.Context, subscriptionID, environmentName, armEndpoint string, credentialsProvider CredentialsProvider) error { if credentialsProvider == nil { return fmt.Errorf("credentials provider cannot have an empty value") } - settings, err := c.getSettingsFromEnvironment(environmentName) + settings, err := c.getSettingsFromEnvironment(environmentName, armEndpoint) if err != nil { return err } @@ -121,7 +121,7 @@ func (c *AzureClients) setCredentialsWithProvider(ctx context.Context, subscript return err } -func (c *AzureClients) getSettingsFromEnvironment(environmentName string) (s auth.EnvironmentSettings, err error) { +func (c *AzureClients) getSettingsFromEnvironment(environmentName, armEndpoint string) (s auth.EnvironmentSettings, err error) { s = auth.EnvironmentSettings{ Values: map[string]string{}, } @@ -138,6 +138,8 @@ func (c *AzureClients) getSettingsFromEnvironment(environmentName string) (s aut setValue(s, "AZURE_AD_RESOURCE") if v := s.Values["AZURE_ENVIRONMENT"]; v == "" { s.Environment = azureautorest.PublicCloud + } else if len(armEndpoint) > 0 { + s.Environment, err = azureautorest.EnvironmentFromURL(armEndpoint) } else { s.Environment, err = azureautorest.EnvironmentFromName(v) } diff --git a/azure/scope/cluster.go b/azure/scope/cluster.go index bff9416387c..0aad1092ca7 100644 --- a/azure/scope/cluster.go +++ b/azure/scope/cluster.go @@ -84,7 +84,8 @@ func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterSc if err != nil { return nil, errors.Wrap(err, "failed to init credentials provider") } - err = params.AzureClients.setCredentialsWithProvider(ctx, params.AzureCluster.Spec.SubscriptionID, params.AzureCluster.Spec.AzureEnvironment, credentialsProvider) + spec := params.AzureCluster.Spec + err = params.AzureClients.setCredentialsWithProvider(ctx, spec.SubscriptionID, spec.AzureEnvironment, spec.ARMEndpoint, credentialsProvider) if err != nil { return nil, errors.Wrap(err, "failed to configure azure settings and credentials for Identity") } diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index 0956cf6478a..53e4e4c934f 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -97,7 +97,7 @@ func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlane return nil, errors.Wrap(err, "failed to init credentials provider") } - if err := params.AzureClients.setCredentialsWithProvider(ctx, params.ControlPlane.Spec.SubscriptionID, params.ControlPlane.Spec.AzureEnvironment, credentialsProvider); err != nil { + if err := params.AzureClients.setCredentialsWithProvider(ctx, params.ControlPlane.Spec.SubscriptionID, params.ControlPlane.Spec.AzureEnvironment, "", credentialsProvider); err != nil { return nil, errors.Wrap(err, "failed to configure azure settings and credentials for Identity") } From 3f54f57ad6df8bae580b29d9b91a814919ee2378 Mon Sep 17 00:00:00 2001 From: Patrick Dillon Date: Mon, 17 Feb 2025 14:16:56 -0500 Subject: [PATCH 04/10] UPSTREAM: 5532: Set ARMClientOptions for Azure Stack Sets ARM Client Options when using the Azure Stack environment. Extends ARMClientOptions to accept an ARMEndpoint, which can be obtained from the authorizer interface, the same source the cloud environment. Sets the APIVersion to a hybrid cloud profile to ensure compatibility with hybrid environments. --- azure/defaults.go | 24 +++++++++++++++++++++- azure/defaults_test.go | 7 ++++--- azure/services/availabilitysets/client.go | 2 +- azure/services/disks/client.go | 2 +- azure/services/identities/client.go | 4 ++-- azure/services/inboundnatrules/client.go | 2 +- azure/services/loadbalancers/client.go | 4 ++-- azure/services/networkinterfaces/client.go | 2 +- azure/services/privatedns/link_client.go | 2 +- azure/services/privatedns/record_client.go | 2 +- azure/services/privatedns/zone_client.go | 2 +- azure/services/publicips/client.go | 2 +- azure/services/resourcehealth/client.go | 2 +- azure/services/resourceskus/client.go | 2 +- azure/services/roleassignments/client.go | 2 +- azure/services/routetables/client.go | 2 +- azure/services/scalesets/client.go | 4 ++-- azure/services/scalesetvms/client.go | 2 +- azure/services/securitygroups/client.go | 4 ++-- azure/services/tags/client.go | 2 +- azure/services/virtualmachines/client.go | 2 +- azure/services/vmextensions/client.go | 2 +- azure/services/vnetpeerings/client.go | 2 +- 23 files changed, 52 insertions(+), 29 deletions(-) diff --git a/azure/defaults.go b/azure/defaults.go index 3d2c66a8fb6..2c27dedc67c 100644 --- a/azure/defaults.go +++ b/azure/defaults.go @@ -27,6 +27,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" "github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel" + "github.com/Azure/go-autorest/autorest/azure" "go.opentelemetry.io/otel" "sigs.k8s.io/cluster-api-provider-azure/util/tele" @@ -111,6 +112,12 @@ const ( CustomHeaderPrefix = "infrastructure.cluster.x-k8s.io/custom-header-" ) +const ( + // StackAPIVersion is the API version profile to set for ARM clients. See: + // https://learn.microsoft.com/en-us/azure-stack/user/azure-stack-profiles-azure-resource-manager-versions?view=azs-2408#overview-of-the-2020-09-01-hybrid-profile + StackAPIVersionProfile = "2020-06-01" +) + var ( // LinuxBootstrapExtensionCommand is the command the VM bootstrap extension will execute to verify Linux nodes bootstrap completes successfully. LinuxBootstrapExtensionCommand = fmt.Sprintf("for i in $(seq 1 %d); do test -f %s && break; if [ $i -eq %d ]; then exit 1; else sleep %d; fi; done", bootstrapExtensionRetries, bootstrapSentinelFile, bootstrapExtensionRetries, bootstrapExtensionSleep) @@ -359,7 +366,7 @@ func UserAgent() string { } // ARMClientOptions returns default ARM client options for CAPZ SDK v2 requests. -func ARMClientOptions(azureEnvironment string, extraPolicies ...policy.Policy) (*arm.ClientOptions, error) { +func ARMClientOptions(azureEnvironment, armEndpoint string, extraPolicies ...policy.Policy) (*arm.ClientOptions, error) { opts := &arm.ClientOptions{} switch azureEnvironment { @@ -369,6 +376,21 @@ func ARMClientOptions(azureEnvironment string, extraPolicies ...policy.Policy) ( opts.Cloud = cloud.AzureChina case USGovernmentCloudName: opts.Cloud = cloud.AzureGovernment + case StackCloudName: + cloudEnv, err := azure.EnvironmentFromURL(armEndpoint) + if err != nil { + return nil, fmt.Errorf("unable to get Azure Stack cloud environment: %w", err) + } + opts.APIVersion = StackAPIVersionProfile + opts.Cloud = cloud.Configuration{ + ActiveDirectoryAuthorityHost: cloudEnv.ActiveDirectoryEndpoint, + Services: map[cloud.ServiceName]cloud.ServiceConfiguration{ + cloud.ResourceManager: { + Audience: cloudEnv.TokenAudience, + Endpoint: cloudEnv.ResourceManagerEndpoint, + }, + }, + } case "": // No cloud name provided, so leave at defaults. default: diff --git a/azure/defaults_test.go b/azure/defaults_test.go index 88708df72cf..2a627dcdc80 100644 --- a/azure/defaults_test.go +++ b/azure/defaults_test.go @@ -38,6 +38,7 @@ func TestARMClientOptions(t *testing.T) { tests := []struct { name string cloudName string + armEndpoint string expectedCloud cloud.Configuration expectError bool }{ @@ -72,7 +73,7 @@ func TestARMClientOptions(t *testing.T) { t.Parallel() g := NewWithT(t) - opts, err := ARMClientOptions(tc.cloudName) + opts, err := ARMClientOptions(tc.cloudName, tc.armEndpoint) if tc.expectError { g.Expect(err).To(HaveOccurred()) return @@ -99,7 +100,7 @@ func TestPerCallPolicies(t *testing.T) { defer server.Close() // Call the factory function and ensure it has both PerCallPolicies. - opts, err := ARMClientOptions("") + opts, err := ARMClientOptions("", "") g.Expect(err).NotTo(HaveOccurred()) g.Expect(opts.PerCallPolicies).To(HaveLen(2)) g.Expect(opts.PerCallPolicies).To(ContainElement(BeAssignableToTypeOf(correlationIDPolicy{}))) @@ -184,7 +185,7 @@ func TestCustomPutPatchHeaderPolicy(t *testing.T) { // Create options with a custom PUT/PATCH header per-call policy getterMock := mock_azure.NewMockResourceSpecGetterWithHeaders(mockCtrl) getterMock.EXPECT().CustomHeaders().Return(tc.headers).AnyTimes() - opts, err := ARMClientOptions("", CustomPutPatchHeaderPolicy{Headers: tc.headers}) + opts, err := ARMClientOptions("", "", CustomPutPatchHeaderPolicy{Headers: tc.headers}) g.Expect(err).NotTo(HaveOccurred()) // Create a request diff --git a/azure/services/availabilitysets/client.go b/azure/services/availabilitysets/client.go index 859789d3f37..800d5a4560e 100644 --- a/azure/services/availabilitysets/client.go +++ b/azure/services/availabilitysets/client.go @@ -34,7 +34,7 @@ type AzureClient struct { // NewClient creates a new availability sets client from an authorizer. func NewClient(auth azure.Authorizer) (*AzureClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create availabilitysets client options") } diff --git a/azure/services/disks/client.go b/azure/services/disks/client.go index 58cdb4345fc..698f572e9bc 100644 --- a/azure/services/disks/client.go +++ b/azure/services/disks/client.go @@ -37,7 +37,7 @@ type azureClient struct { // newClient creates a new disks client from an authorizer. func newClient(auth azure.Authorizer, apiCallTimeout time.Duration) (*azureClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create disks client options") } diff --git a/azure/services/identities/client.go b/azure/services/identities/client.go index e12a25a10ab..c1cca1bd747 100644 --- a/azure/services/identities/client.go +++ b/azure/services/identities/client.go @@ -41,7 +41,7 @@ type AzureClient struct { // NewClient creates a new MSI client from an authorizer. func NewClient(auth azure.Authorizer) (Client, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create identities client options") } @@ -54,7 +54,7 @@ func NewClient(auth azure.Authorizer) (Client, error) { // NewClientBySub creates a new MSI client with a given subscriptionID. func NewClientBySub(auth azure.Authorizer, subscriptionID string) (Client, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create identities client options") } diff --git a/azure/services/inboundnatrules/client.go b/azure/services/inboundnatrules/client.go index 0b8f5f6e3a9..8500cd5d3aa 100644 --- a/azure/services/inboundnatrules/client.go +++ b/azure/services/inboundnatrules/client.go @@ -44,7 +44,7 @@ var _ client = (*azureClient)(nil) // newClient creates a new inbound NAT rules client from an authorizer. func newClient(auth azure.Authorizer, apiCallTimeout time.Duration) (*azureClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create inboundnatrules client options") } diff --git a/azure/services/loadbalancers/client.go b/azure/services/loadbalancers/client.go index 0981f4855a4..972875c398c 100644 --- a/azure/services/loadbalancers/client.go +++ b/azure/services/loadbalancers/client.go @@ -39,7 +39,7 @@ type azureClient struct { // newClient creates a new load balancer client from an authorizer. func newClient(auth azure.Authorizer, apiCallTimeout time.Duration) (*azureClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to get load balancer client options") } @@ -86,7 +86,7 @@ func (ac *azureClient) CreateOrUpdateAsync(ctx context.Context, spec azure.Resou } // Create a new client that knows how to add etag headers to the request. - clientOpts, err := azure.ARMClientOptions(ac.auth.CloudEnvironment(), extraPolicies...) + clientOpts, err := azure.ARMClientOptions(ac.auth.CloudEnvironment(), ac.auth.BaseURI(), extraPolicies...) if err != nil { return nil, nil, errors.Wrap(err, "failed to create loadbalancer client options") } diff --git a/azure/services/networkinterfaces/client.go b/azure/services/networkinterfaces/client.go index 62fefd8d304..114f827cd05 100644 --- a/azure/services/networkinterfaces/client.go +++ b/azure/services/networkinterfaces/client.go @@ -37,7 +37,7 @@ type azureClient struct { // NewClient creates a new network interfaces client from an authorizer. func NewClient(auth azure.Authorizer, apiCallTimeout time.Duration) (*azureClient, error) { //nolint:revive // leave it as is - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create networkinterfaces client options") } diff --git a/azure/services/privatedns/link_client.go b/azure/services/privatedns/link_client.go index 0c2104d32ae..7c913633be1 100644 --- a/azure/services/privatedns/link_client.go +++ b/azure/services/privatedns/link_client.go @@ -37,7 +37,7 @@ type azureVirtualNetworkLinksClient struct { // newVirtualNetworkLinksClient creates a virtual network links client from an authorizer. func newVirtualNetworkLinksClient(auth azure.Authorizer, apiCallTimeout time.Duration) (*azureVirtualNetworkLinksClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create virtualnetworkslink client options") } diff --git a/azure/services/privatedns/record_client.go b/azure/services/privatedns/record_client.go index 01e6ce7ff8c..1483bd77cf2 100644 --- a/azure/services/privatedns/record_client.go +++ b/azure/services/privatedns/record_client.go @@ -34,7 +34,7 @@ type azureRecordsClient struct { // newRecordSetsClient creates a record sets client from an authorizer. func newRecordSetsClient(auth azure.Authorizer) (*azureRecordsClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create recordsets client options") } diff --git a/azure/services/privatedns/zone_client.go b/azure/services/privatedns/zone_client.go index 86e137251c5..193dbd5f102 100644 --- a/azure/services/privatedns/zone_client.go +++ b/azure/services/privatedns/zone_client.go @@ -37,7 +37,7 @@ type azureZonesClient struct { // newPrivateZonesClient creates a private zones client from an authorizer. func newPrivateZonesClient(auth azure.Authorizer, apiCallTimeout time.Duration) (*azureZonesClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create privatezones client options") } diff --git a/azure/services/publicips/client.go b/azure/services/publicips/client.go index b37d05529f2..396c66f92b3 100644 --- a/azure/services/publicips/client.go +++ b/azure/services/publicips/client.go @@ -37,7 +37,7 @@ type AzureClient struct { // NewClient creates a new public IP client from an authorizer. func NewClient(auth azure.Authorizer, apiCallTimeout time.Duration) (*AzureClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create publicips client options") } diff --git a/azure/services/resourcehealth/client.go b/azure/services/resourcehealth/client.go index a597dbfae97..189b507873c 100644 --- a/azure/services/resourcehealth/client.go +++ b/azure/services/resourcehealth/client.go @@ -38,7 +38,7 @@ type azureClient struct { // newClient creates a new resource health client from an authorizer. func newClient(auth azure.Authorizer) (*azureClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create resourcehealth client options") } diff --git a/azure/services/resourceskus/client.go b/azure/services/resourceskus/client.go index ed84116192f..c442f8c7ed5 100644 --- a/azure/services/resourceskus/client.go +++ b/azure/services/resourceskus/client.go @@ -40,7 +40,7 @@ var _ Client = &AzureClient{} // NewClient creates a new Resource SKUs client from an authorizer. func NewClient(auth azure.Authorizer) (*AzureClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create resourceskus client options") } diff --git a/azure/services/roleassignments/client.go b/azure/services/roleassignments/client.go index dd7643ec8d3..a35f3897966 100644 --- a/azure/services/roleassignments/client.go +++ b/azure/services/roleassignments/client.go @@ -34,7 +34,7 @@ type azureClient struct { // newClient creates a new role assignments client from an authorizer. func newClient(auth azure.Authorizer) (*azureClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create roleassignments client options") } diff --git a/azure/services/routetables/client.go b/azure/services/routetables/client.go index b535d6d0c66..b3bac74da59 100644 --- a/azure/services/routetables/client.go +++ b/azure/services/routetables/client.go @@ -37,7 +37,7 @@ type azureClient struct { // newClient creates a new route tables client from an authorizer. func newClient(auth azure.Authorizer, apiCallTimeout time.Duration) (*azureClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create routetables client options") } diff --git a/azure/services/scalesets/client.go b/azure/services/scalesets/client.go index d498a37240d..2b906f8d791 100644 --- a/azure/services/scalesets/client.go +++ b/azure/services/scalesets/client.go @@ -67,7 +67,7 @@ func NewClient(auth azure.Authorizer, apiCallTimeout time.Duration) (*AzureClien // newVirtualMachineScaleSetVMsClient creates a vmss VM client from an authorizer. func newVirtualMachineScaleSetVMsClient(auth azure.Authorizer) (*armcompute.VirtualMachineScaleSetVMsClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create scalesetvms client options") } @@ -80,7 +80,7 @@ func newVirtualMachineScaleSetVMsClient(auth azure.Authorizer) (*armcompute.Virt // newVirtualMachineScaleSetsClient creates a vmss client from an authorizer. func newVirtualMachineScaleSetsClient(auth azure.Authorizer) (*armcompute.VirtualMachineScaleSetsClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create scalesets client options") } diff --git a/azure/services/scalesetvms/client.go b/azure/services/scalesetvms/client.go index b4aab4471e5..11ce72b63b7 100644 --- a/azure/services/scalesetvms/client.go +++ b/azure/services/scalesetvms/client.go @@ -46,7 +46,7 @@ var _ client = &azureClient{} // newClient creates a VMSS client from an authorizer. func newClient(auth azure.Authorizer, apiCallTimeout time.Duration) (*azureClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create scalesetvms client options") } diff --git a/azure/services/securitygroups/client.go b/azure/services/securitygroups/client.go index 607048860d0..c09d00e05be 100644 --- a/azure/services/securitygroups/client.go +++ b/azure/services/securitygroups/client.go @@ -39,7 +39,7 @@ type azureClient struct { // newClient creates a new security groups client from an authorizer. func newClient(auth azure.Authorizer, apiCallTimeout time.Duration) (*azureClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create securitygroups client options") } @@ -84,7 +84,7 @@ func (ac *azureClient) CreateOrUpdateAsync(ctx context.Context, spec azure.Resou } // Create a new client that knows how to add the etag header. - clientOpts, err := azure.ARMClientOptions(ac.auth.CloudEnvironment(), extraPolicies...) + clientOpts, err := azure.ARMClientOptions(ac.auth.CloudEnvironment(), ac.auth.BaseURI(), extraPolicies...) if err != nil { return nil, nil, errors.Wrap(err, "failed to create securitygroups client options") } diff --git a/azure/services/tags/client.go b/azure/services/tags/client.go index ebde41d8e79..c63b7937f13 100644 --- a/azure/services/tags/client.go +++ b/azure/services/tags/client.go @@ -41,7 +41,7 @@ var _ client = (*AzureClient)(nil) // NewClient creates a tags client from an authorizer. func NewClient(auth azure.Authorizer) (*AzureClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create tags client options") } diff --git a/azure/services/virtualmachines/client.go b/azure/services/virtualmachines/client.go index 1e2bbea08d4..5f312f0facc 100644 --- a/azure/services/virtualmachines/client.go +++ b/azure/services/virtualmachines/client.go @@ -49,7 +49,7 @@ var _ Client = &AzureClient{} // NewClient creates a VMs client from an authorizer. func NewClient(auth azure.Authorizer, apiCallTimeout time.Duration) (*AzureClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create virtualmachines client options") } diff --git a/azure/services/vmextensions/client.go b/azure/services/vmextensions/client.go index 187007ff784..09a58540811 100644 --- a/azure/services/vmextensions/client.go +++ b/azure/services/vmextensions/client.go @@ -37,7 +37,7 @@ type azureClient struct { // newClient creates a new vm extensions client from an authorizer. func newClient(auth azure.Authorizer, apiCallTimeout time.Duration) (*azureClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create virtualmachineextensions client options") } diff --git a/azure/services/vnetpeerings/client.go b/azure/services/vnetpeerings/client.go index 893c545d487..c556dbb4f23 100644 --- a/azure/services/vnetpeerings/client.go +++ b/azure/services/vnetpeerings/client.go @@ -37,7 +37,7 @@ type AzureClient struct { // NewClient creates a new virtual network peerings client from an authorizer. func NewClient(auth azure.Authorizer, apiCallTimeout time.Duration) (*AzureClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) + opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) if err != nil { return nil, errors.Wrap(err, "failed to create vnetpeerings client options") } From 6f2b8074ace381ff634a0acde9efb82826cdce4f Mon Sep 17 00:00:00 2001 From: Patrick Dillon Date: Mon, 3 Mar 2025 11:05:48 -0500 Subject: [PATCH 05/10] UPSTREAM: 5532: Generate CRDs for AzureStack --- .../infrastructure.cluster.x-k8s.io_azureclusters.yaml | 7 +++++++ ...rastructure.cluster.x-k8s.io_azureclustertemplates.yaml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml index 5692ad1ea9c..1293762205f 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml @@ -84,6 +84,12 @@ spec: AdditionalTags is an optional set of tags to add to Azure resources managed by the Azure provider, in addition to the ones added by default. type: object + armEndpoint: + description: |- + ARMEndpoint specifies a URL for the ARM Resource Manager endpoint. + It may only be specified when the AzureEnvironment is set to AzureStackCloud, + in which case it is required. + type: string azureEnvironment: description: |- AzureEnvironment is the name of the AzureCloud to be used. @@ -92,6 +98,7 @@ spec: - GermanCloud: "AzureGermanCloud" - PublicCloud: "AzurePublicCloud" - USGovernmentCloud: "AzureUSGovernmentCloud" + - StackCloud: "HybridEnvironment" Note that values other than the default must also be accompanied by corresponding changes to the aso-controller-settings Secret to configure ASO to refer to the non-Public cloud. ASO currently does diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml index b1cffc10c83..b0caaada440 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml @@ -57,6 +57,12 @@ spec: AdditionalTags is an optional set of tags to add to Azure resources managed by the Azure provider, in addition to the ones added by default. type: object + armEndpoint: + description: |- + ARMEndpoint specifies a URL for the ARM Resource Manager endpoint. + It may only be specified when the AzureEnvironment is set to AzureStackCloud, + in which case it is required. + type: string azureEnvironment: description: |- AzureEnvironment is the name of the AzureCloud to be used. @@ -65,6 +71,7 @@ spec: - GermanCloud: "AzureGermanCloud" - PublicCloud: "AzurePublicCloud" - USGovernmentCloud: "AzureUSGovernmentCloud" + - StackCloud: "HybridEnvironment" Note that values other than the default must also be accompanied by corresponding changes to the aso-controller-settings Secret to configure ASO to refer to the non-Public cloud. ASO currently does From 6d31469e2fdaa3ca834510f2c74c38c387bb78d4 Mon Sep 17 00:00:00 2001 From: Patrick Dillon Date: Mon, 3 Mar 2025 14:55:23 -0500 Subject: [PATCH 06/10] UPSTREAM: 5532: Skip privatedns zones on Azure Stack Azure Stack Hub does not support private dns zones, so skip them. --- azure/scope/cluster.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/azure/scope/cluster.go b/azure/scope/cluster.go index 0aad1092ca7..0c2d55f45ae 100644 --- a/azure/scope/cluster.go +++ b/azure/scope/cluster.go @@ -558,7 +558,7 @@ func (s *ClusterScope) VNetSpec() azure.ASOResourceSpecGetter[*asonetworkv1api20 // PrivateDNSSpec returns the private dns zone spec. func (s *ClusterScope) PrivateDNSSpec() (zoneSpec azure.ResourceSpecGetter, linkSpec, recordSpec []azure.ResourceSpecGetter) { - if s.IsAPIServerPrivate() { + if s.IsAPIServerPrivate() && !s.IsHybridEnvironment() { resourceGroup := s.ResourceGroup() if s.AzureCluster.Spec.NetworkSpec.PrivateDNSZoneResourceGroup != "" { resourceGroup = s.AzureCluster.Spec.NetworkSpec.PrivateDNSZoneResourceGroup @@ -1234,3 +1234,8 @@ func (s *ClusterScope) getLastAppliedSecurityRules(nsgName string) map[string]in } return lastAppliedSecurityRules } + +// IsHybridEnvironment returns true if the cluster is running on Azure Stack. +func (s *ClusterScope) IsHybridEnvironment() bool { + return strings.EqualFold(s.Environment.Name, azure.StackCloudName) +} From 0ea7c646ae09a2e4c0143bf1f880daa33a7f45dd Mon Sep 17 00:00:00 2001 From: Patrick Dillon Date: Mon, 24 Mar 2025 12:18:50 -0400 Subject: [PATCH 07/10] UPSTREAM: 5532: AzureStack: handle missing avail set sku cache The Resource SKU API for availability sets may not be available in an Azure Stack environment. The cache is used to determine the fault domain count. For Azure Stack, we can default to 2. Future work could potentially set this programatically or expose the fault domain count in the API. --- azure/scope/machine.go | 16 ++++---- azure/services/availabilitysets/spec.go | 53 ++++++++++++++++--------- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/azure/scope/machine.go b/azure/scope/machine.go index d6a96964f36..1f91de2eb59 100644 --- a/azure/scope/machine.go +++ b/azure/scope/machine.go @@ -150,7 +150,8 @@ func (m *MachineScope) InitMachineCache(ctx context.Context) error { } m.cache.availabilitySetSKU, err = skuCache.Get(ctx, string(armcompute.AvailabilitySetSKUTypesAligned), resourceskus.AvailabilitySets) - if err != nil { + // Resource SKU API for availability sets may not be available in Azure Stack environments. + if err != nil && !strings.EqualFold(m.CloudEnvironment(), "HybridEnvironment") { return errors.Wrapf(err, "failed to get availability set SKU %s in compute api", string(armcompute.AvailabilitySetSKUTypesAligned)) } } @@ -494,12 +495,13 @@ func (m *MachineScope) AvailabilitySetSpec() azure.ResourceSpecGetter { } spec := &availabilitysets.AvailabilitySetSpec{ - Name: availabilitySetName, - ResourceGroup: m.NodeResourceGroup(), - ClusterName: m.ClusterName(), - Location: m.Location(), - SKU: nil, - AdditionalTags: m.AdditionalTags(), + Name: availabilitySetName, + ResourceGroup: m.NodeResourceGroup(), + ClusterName: m.ClusterName(), + Location: m.Location(), + CloudEnvironment: m.CloudEnvironment(), + SKU: nil, + AdditionalTags: m.AdditionalTags(), } if m.cache != nil { diff --git a/azure/services/availabilitysets/spec.go b/azure/services/availabilitysets/spec.go index ea522da07ee..809e9fa0ef9 100644 --- a/azure/services/availabilitysets/spec.go +++ b/azure/services/availabilitysets/spec.go @@ -19,24 +19,27 @@ package availabilitysets import ( "context" "strconv" + "strings" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" "github.com/pkg/errors" "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/converters" "sigs.k8s.io/cluster-api-provider-azure/azure/services/resourceskus" ) // AvailabilitySetSpec defines the specification for an availability set. type AvailabilitySetSpec struct { - Name string - ResourceGroup string - ClusterName string - Location string - SKU *resourceskus.SKU - AdditionalTags infrav1.Tags + Name string + ResourceGroup string + ClusterName string + Location string + CloudEnvironment string + SKU *resourceskus.SKU + AdditionalTags infrav1.Tags } // ResourceName returns the name of the availability set. @@ -64,20 +67,10 @@ func (s *AvailabilitySetSpec) Parameters(_ context.Context, existing interface{} return nil, nil } - if s.SKU == nil { - return nil, errors.New("unable to get required availability set SKU from machine cache") - } - - var faultDomainCount *int32 - faultDomainCountStr, ok := s.SKU.GetCapability(resourceskus.MaximumPlatformFaultDomainCount) - if !ok { - return nil, errors.Errorf("unable to get required availability set SKU capability %s", resourceskus.MaximumPlatformFaultDomainCount) - } - count, err := strconv.ParseInt(faultDomainCountStr, 10, 32) + faultDomainCount, err := getFaultDomainCount(s.SKU, s.CloudEnvironment) if err != nil { - return nil, errors.Wrapf(err, "unable to parse availability set fault domain count") + return nil, err } - faultDomainCount = ptr.To[int32](int32(count)) asParams := armcompute.AvailabilitySet{ SKU: &armcompute.SKU{ @@ -98,3 +91,27 @@ func (s *AvailabilitySetSpec) Parameters(_ context.Context, existing interface{} return asParams, nil } + +func getFaultDomainCount(SKU *resourceskus.SKU, cloudEnvironment string) (*int32, error) { + // Azure Stack environments may not implement the resource SKU API + // for availability sets. Use a default value instead. + if strings.EqualFold(cloudEnvironment, azure.StackCloudName) { + return ptr.To(int32(2)), nil + } + + if SKU == nil { + return nil, errors.New("unable to get required availability set SKU from machine cache") + } + + var faultDomainCount *int32 + faultDomainCountStr, ok := SKU.GetCapability(resourceskus.MaximumPlatformFaultDomainCount) + if !ok { + return nil, errors.Errorf("unable to get required availability set SKU capability %s", resourceskus.MaximumPlatformFaultDomainCount) + } + count, err := strconv.ParseInt(faultDomainCountStr, 10, 32) + if err != nil { + return nil, errors.Wrapf(err, "unable to parse availability set fault domain count") + } + faultDomainCount = ptr.To[int32](int32(count)) + return faultDomainCount, nil +} From eb7c2316a5a5aa98c91ecb0ffd64c400189e4986 Mon Sep 17 00:00:00 2001 From: Patrick Dillon Date: Thu, 27 Mar 2025 14:14:59 -0400 Subject: [PATCH 08/10] UPSTREAM: 5532: Skip tag reconciliation in Azure Stack The tag service using the V2 SDK is not available in azure stack. Skip tag reconciliation in Azure Stack environments. --- controllers/azuremachine_reconciler.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/controllers/azuremachine_reconciler.go b/controllers/azuremachine_reconciler.go index 544ccc02694..d0bb9ef8d7d 100644 --- a/controllers/azuremachine_reconciler.go +++ b/controllers/azuremachine_reconciler.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "strings" "github.com/pkg/errors" @@ -101,10 +102,13 @@ func newAzureMachineService(machineScope *scope.MachineScope) (*azureMachineServ virtualmachinesSvc, roleAssignmentsSvc, vmextensionsSvc, - tagsSvc, }, skuCache: cache, } + if !strings.EqualFold(machineScope.CloudEnvironment(), azure.StackCloudName) { + ams.services = append(ams.services, tagsSvc) + } + ams.Reconcile = ams.reconcile ams.Pause = ams.pause ams.Delete = ams.delete From 838a0ce60d2ea82f169db5af00ab976b286081b9 Mon Sep 17 00:00:00 2001 From: Patrick Dillon Date: Thu, 27 Mar 2025 14:35:13 -0400 Subject: [PATCH 09/10] UPSTREAM: 5532: Change Disk Client API Version for Azure Stack The standard 2020-06-01 API Version is not supported for disk operations in Azure Stack, so change to the compatible 2018-06-01 profile. --- azure/defaults.go | 4 ++++ azure/services/disks/client.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/azure/defaults.go b/azure/defaults.go index 2c27dedc67c..28f7aca1d34 100644 --- a/azure/defaults.go +++ b/azure/defaults.go @@ -116,6 +116,10 @@ const ( // StackAPIVersion is the API version profile to set for ARM clients. See: // https://learn.microsoft.com/en-us/azure-stack/user/azure-stack-profiles-azure-resource-manager-versions?view=azs-2408#overview-of-the-2020-09-01-hybrid-profile StackAPIVersionProfile = "2020-06-01" + + // StackDiskAPIVersionProfile is the API Version to set for the disk client. + // API Version Profile "2020-06-01" is not supported for disks. + StackDiskAPIVersionProfile = "2018-06-01" ) var ( diff --git a/azure/services/disks/client.go b/azure/services/disks/client.go index 698f572e9bc..af4ebe27aab 100644 --- a/azure/services/disks/client.go +++ b/azure/services/disks/client.go @@ -18,6 +18,7 @@ package disks import ( "context" + "strings" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" @@ -38,6 +39,9 @@ type azureClient struct { // newClient creates a new disks client from an authorizer. func newClient(auth azure.Authorizer, apiCallTimeout time.Duration) (*azureClient, error) { opts, err := azure.ARMClientOptions(auth.CloudEnvironment(), auth.BaseURI()) + if strings.EqualFold(auth.CloudEnvironment(), azure.StackCloudName) { + opts.APIVersion = azure.StackDiskAPIVersionProfile + } if err != nil { return nil, errors.Wrap(err, "failed to create disks client options") } From 02b025995d93c1f07746b074cddc1c99a2317e0c Mon Sep 17 00:00:00 2001 From: Patrick Dillon Date: Thu, 27 Mar 2025 17:17:42 -0400 Subject: [PATCH 10/10] UPSTREAM: 5532: Retry VM Delete without Force if Bad Request Azure Stack returns a 400 error when trying to delete a VM with the force flag and the error message suggests retrying without the flag. --- azure/errors.go | 6 ++++++ azure/services/virtualmachines/client.go | 11 +++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/azure/errors.go b/azure/errors.go index 0d719e80037..8e77269acd4 100644 --- a/azure/errors.go +++ b/azure/errors.go @@ -34,6 +34,12 @@ func ResourceNotFound(err error) bool { return errors.As(err, &rerr) && rerr.StatusCode == http.StatusNotFound } +// BadRequest parses an error to check if it its status code is Bad Request (400). +func BadRequest(err error) bool { + var rerr *azcore.ResponseError + return errors.As(err, &rerr) && rerr.StatusCode == http.StatusBadRequest +} + // VMDeletedError is returned when a virtual machine is deleted outside of capz. type VMDeletedError struct { ProviderID string diff --git a/azure/services/virtualmachines/client.go b/azure/services/virtualmachines/client.go index 5f312f0facc..e7b63c12404 100644 --- a/azure/services/virtualmachines/client.go +++ b/azure/services/virtualmachines/client.go @@ -109,14 +109,21 @@ func (ac *AzureClient) CreateOrUpdateAsync(ctx context.Context, spec azure.Resou // request to Azure and if accepted without error, the func will return a Poller which can be used to track the ongoing // progress of the operation. func (ac *AzureClient) DeleteAsync(ctx context.Context, spec azure.ResourceSpecGetter, resumeToken string) (poller *runtime.Poller[armcompute.VirtualMachinesClientDeleteResponse], err error) { - ctx, _, done := tele.StartSpanWithLogger(ctx, "virtualmachines.AzureClient.Delete") + ctx, log, done := tele.StartSpanWithLogger(ctx, "virtualmachines.AzureClient.Delete") defer done() forceDelete := ptr.To(true) opts := &armcompute.VirtualMachinesClientBeginDeleteOptions{ResumeToken: resumeToken, ForceDeletion: forceDelete} poller, err = ac.virtualmachines.BeginDelete(ctx, spec.ResourceGroupName(), spec.ResourceName(), opts) if err != nil { - return nil, err + if azure.BadRequest(err) { + log.Info("Failed to Begin VM Delete with Force Deletion, retrying without the force flag") + opts.ForceDeletion = ptr.To(false) + poller, err = ac.virtualmachines.BeginDelete(ctx, spec.ResourceGroupName(), spec.ResourceName(), opts) + } + if err != nil { + return nil, err + } } ctx, cancel := context.WithTimeout(ctx, ac.apiCallTimeout)