Skip to content

Commit b37fc68

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #2020 from michalno1/main
Support passing custom headers to AKS Managed Cluster and Node Pool create/update requests
2 parents 4b70fae + c0416bc commit b37fc68

File tree

20 files changed

+544
-26
lines changed

20 files changed

+544
-26
lines changed

‎azure/defaults.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,14 @@ const (
8484
ProviderIDPrefix = "azure://"
8585
)
8686

87+
const (
88+
// CustomHeaderPrefix is the prefix of annotations that enable additional cluster / node pool features.
89+
// Whatever follows the prefix will be passed as a header to cluster/node pool creation/update requests.
90+
// E.g. add `"infrastructure.cluster.x-k8s.io/custom-header-UseGPUDedicatedVHD": "true"` annotation to
91+
// AzureManagedMachinePool CR to enable creating GPU nodes by the node pool.
92+
CustomHeaderPrefix = "infrastructure.cluster.x-k8s.io/custom-header-"
93+
)
94+
8795
var (
8896
// LinuxBootstrapExtensionCommand is the command the VM bootstrap extension will execute to verify Linux nodes bootstrap completes successfully.
8997
LinuxBootstrapExtensionCommand = fmt.Sprintf("for i in $(seq 1 %d); do test -f %s && break; if [ $i -eq %d ]; then return 1; else sleep %d; fi; done", bootstrapExtensionRetries, bootstrapSentinelFile, bootstrapExtensionRetries, bootstrapExtensionSleep)

‎azure/scope/managedcontrolplane.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -391,6 +391,10 @@ func (s *ManagedControlPlaneScope) FailureDomains() []string {
391391
return []string{}
392392
}
393393

394+
func (s *ManagedControlPlaneScope) ManagedClusterAnnotations() map[string]string {
395+
return s.ControlPlane.Annotations
396+
}
397+
394398
// ManagedClusterSpec returns the managed cluster spec.
395399
func (s *ManagedControlPlaneScope) ManagedClusterSpec() (azure.ManagedClusterSpec, error) {
396400
decodedSSHPublicKey, err := base64.StdEncoding.DecodeString(s.ControlPlane.Spec.SSHPublicKey)
@@ -557,6 +561,10 @@ func (s *ManagedControlPlaneScope) GetAllAgentPoolSpecs(ctx context.Context) ([]
557561
return ammps, nil
558562
}
559563

564+
func (s *ManagedControlPlaneScope) AgentPoolAnnotations() map[string]string {
565+
return s.InfraMachinePool.Annotations
566+
}
567+
560568
// AgentPoolSpec returns an azure.AgentPoolSpec for currently reconciled AzureManagedMachinePool.
561569
func (s *ManagedControlPlaneScope) AgentPoolSpec() azure.AgentPoolSpec {
562570
return buildAgentPoolSpec(s.ControlPlane, s.MachinePool, s.InfraMachinePool)

‎azure/services/agentpools/agentpools.go‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"github.com/pkg/errors"
2828
infrav1alpha4 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
2929
"sigs.k8s.io/cluster-api-provider-azure/azure"
30+
"sigs.k8s.io/cluster-api-provider-azure/util/maps"
3031
"sigs.k8s.io/cluster-api-provider-azure/util/tele"
3132
)
3233

@@ -35,6 +36,7 @@ type ManagedMachinePoolScope interface {
3536
azure.ClusterDescriber
3637

3738
NodeResourceGroup() string
39+
AgentPoolAnnotations() map[string]string
3840
AgentPoolSpec() azure.AgentPoolSpec
3941
SetAgentPoolProviderIDList([]string)
4042
SetAgentPoolReplicas(int32)
@@ -64,7 +66,6 @@ func (s *Service) Reconcile(ctx context.Context) error {
6466
defer done()
6567

6668
agentPoolSpec := s.scope.AgentPoolSpec()
67-
6869
profile := containerservice.AgentPool{
6970
ManagedClusterAgentPoolProfileProperties: &containerservice.ManagedClusterAgentPoolProfileProperties{
7071
VMSize: &agentPoolSpec.SKU,
@@ -96,8 +97,10 @@ func (s *Service) Reconcile(ctx context.Context) error {
9697
// AKS will populate defaults and read-only values, which we want
9798
// to strip/clean to match what we expect.
9899

100+
customHeaders := maps.FilterByKeyPrefix(s.scope.AgentPoolAnnotations(), azure.CustomHeaderPrefix)
99101
if isCreate := azure.ResourceNotFound(err); isCreate {
100-
err = s.Client.CreateOrUpdate(ctx, agentPoolSpec.ResourceGroup, agentPoolSpec.Cluster, agentPoolSpec.Name, profile)
102+
err = s.Client.CreateOrUpdate(ctx, agentPoolSpec.ResourceGroup, agentPoolSpec.Cluster, agentPoolSpec.Name,
103+
profile, customHeaders)
101104
if err != nil && azure.ResourceNotFound(err) {
102105
return azure.WithTransientError(errors.Wrap(err, "agent pool dependent resource does not exist yet"), 20*time.Second)
103106
} else if err != nil {
@@ -138,7 +141,8 @@ func (s *Service) Reconcile(ctx context.Context) error {
138141
diff := cmp.Diff(normalizedProfile, existingProfile)
139142
if diff != "" {
140143
log.V(2).Info(fmt.Sprintf("Update required (+new -old):\n%s", diff))
141-
err = s.Client.CreateOrUpdate(ctx, agentPoolSpec.ResourceGroup, agentPoolSpec.Cluster, agentPoolSpec.Name, profile)
144+
err = s.Client.CreateOrUpdate(ctx, agentPoolSpec.ResourceGroup, agentPoolSpec.Cluster, agentPoolSpec.Name,
145+
profile, customHeaders)
142146
if err != nil {
143147
return errors.Wrap(err, "failed to create or update agent pool")
144148
}

‎azure/services/agentpools/agentpools_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ func TestReconcile(t *testing.T) {
5555
expectedError: "",
5656
expect: func(m *mock_agentpools.MockClientMockRecorder, provisioningstate string) {
5757
pv := provisioningstate
58-
m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-cluster", "my-agentpool", gomock.Any()).Return(nil)
58+
m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-cluster", "my-agentpool", gomock.Any(), gomock.Any()).Return(nil)
5959
m.Get(gomockinternal.AContext(), "my-rg", "my-cluster", "my-agentpool").Return(containerservice.AgentPool{ManagedClusterAgentPoolProfileProperties: &containerservice.ManagedClusterAgentPoolProfileProperties{
6060
ProvisioningState: &pv,
6161
}}, nil)
@@ -144,7 +144,7 @@ func TestReconcile(t *testing.T) {
144144
},
145145
expectedError: "",
146146
expect: func(m *mock_agentpools.MockClientMockRecorder) {
147-
m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-cluster", "my-agentpool", gomock.Any()).Return(nil)
147+
m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-cluster", "my-agentpool", gomock.Any(), gomock.Any()).Return(nil)
148148
m.Get(gomockinternal.AContext(), "my-rg", "my-cluster", "my-agentpool").Return(containerservice.AgentPool{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 404}, "Not Found"))
149149
},
150150
},
@@ -182,7 +182,7 @@ func TestReconcile(t *testing.T) {
182182
expectedError: "",
183183
expect: func(m *mock_agentpools.MockClientMockRecorder) {
184184
m.Get(gomockinternal.AContext(), "my-rg", "my-cluster", "my-agent-pool").Return(containerservice.AgentPool{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 404}, "Not found"))
185-
m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-cluster", "my-agent-pool", gomock.AssignableToTypeOf(containerservice.AgentPool{})).Return(nil)
185+
m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-cluster", "my-agent-pool", gomock.AssignableToTypeOf(containerservice.AgentPool{}), gomock.Any()).Return(nil)
186186
},
187187
},
188188
{
@@ -201,7 +201,7 @@ func TestReconcile(t *testing.T) {
201201
expectedError: "failed to create or update agent pool: #: Internal Server Error: StatusCode=500",
202202
expect: func(m *mock_agentpools.MockClientMockRecorder) {
203203
m.Get(gomockinternal.AContext(), "my-rg", "my-cluster", "my-agent-pool").Return(containerservice.AgentPool{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 404}, "Not found"))
204-
m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-cluster", "my-agent-pool", gomock.AssignableToTypeOf(containerservice.AgentPool{})).Return(autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error"))
204+
m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-cluster", "my-agent-pool", gomock.AssignableToTypeOf(containerservice.AgentPool{}), gomock.Any()).Return(autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error"))
205205
},
206206
},
207207
{
@@ -228,7 +228,7 @@ func TestReconcile(t *testing.T) {
228228
ProvisioningState: to.StringPtr("Failed"),
229229
},
230230
}, nil)
231-
m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-cluster", "my-agent-pool", gomock.AssignableToTypeOf(containerservice.AgentPool{})).Return(autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error"))
231+
m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-cluster", "my-agent-pool", gomock.AssignableToTypeOf(containerservice.AgentPool{}), gomock.Any()).Return(autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error"))
232232
},
233233
},
234234
{

‎azure/services/agentpools/client.go‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import (
2929
// Client wraps go-sdk.
3030
type Client interface {
3131
Get(context.Context, string, string, string) (containerservice.AgentPool, error)
32-
CreateOrUpdate(context.Context, string, string, string, containerservice.AgentPool) error
32+
CreateOrUpdate(context.Context, string, string, string, containerservice.AgentPool, map[string]string) error
3333
Delete(context.Context, string, string, string) error
3434
}
3535

@@ -62,11 +62,20 @@ func (ac *AzureClient) Get(ctx context.Context, resourceGroupName, cluster, name
6262
}
6363

6464
// CreateOrUpdate creates or updates an agent pool.
65-
func (ac *AzureClient) CreateOrUpdate(ctx context.Context, resourceGroupName, cluster, name string, properties containerservice.AgentPool) error {
65+
func (ac *AzureClient) CreateOrUpdate(ctx context.Context, resourceGroupName, cluster, name string,
66+
properties containerservice.AgentPool, customHeaders map[string]string) error {
6667
ctx, _, done := tele.StartSpanWithLogger(ctx, "agentpools.AzureClient.CreateOrUpdate")
6768
defer done()
6869

69-
future, err := ac.agentpools.CreateOrUpdate(ctx, resourceGroupName, cluster, name, properties)
70+
preparer, err := ac.agentpools.CreateOrUpdatePreparer(ctx, resourceGroupName, cluster, name, properties)
71+
if err != nil {
72+
return errors.Wrap(err, "failed to prepare operation")
73+
}
74+
for key, element := range customHeaders {
75+
preparer.Header.Add(key, element)
76+
}
77+
78+
future, err := ac.agentpools.CreateOrUpdateSender(preparer)
7079
if err != nil {
7180
return errors.Wrap(err, "failed to begin operation")
7281
}

‎azure/services/agentpools/mock_agentpools/agentpools_mock.go‎

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎azure/services/managedclusters/client.go‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import (
3030
type Client interface {
3131
Get(context.Context, string, string) (containerservice.ManagedCluster, error)
3232
GetCredentials(context.Context, string, string) ([]byte, error)
33-
CreateOrUpdate(context.Context, string, string, containerservice.ManagedCluster) (containerservice.ManagedCluster, error)
33+
CreateOrUpdate(context.Context, string, string, containerservice.ManagedCluster, map[string]string) (containerservice.ManagedCluster, error)
3434
Delete(context.Context, string, string) error
3535
}
3636

@@ -78,11 +78,19 @@ func (ac *AzureClient) GetCredentials(ctx context.Context, resourceGroupName, na
7878
}
7979

8080
// CreateOrUpdate creates or updates a managed cluster.
81-
func (ac *AzureClient) CreateOrUpdate(ctx context.Context, resourceGroupName, name string, cluster containerservice.ManagedCluster) (containerservice.ManagedCluster, error) {
81+
func (ac *AzureClient) CreateOrUpdate(ctx context.Context, resourceGroupName, name string, cluster containerservice.ManagedCluster, headers map[string]string) (containerservice.ManagedCluster, error) {
8282
ctx, _, done := tele.StartSpanWithLogger(ctx, "managedclusters.AzureClient.CreateOrUpdate")
8383
defer done()
8484

85-
future, err := ac.managedclusters.CreateOrUpdate(ctx, resourceGroupName, name, cluster)
85+
preparer, err := ac.managedclusters.CreateOrUpdatePreparer(ctx, resourceGroupName, name, cluster)
86+
if err != nil {
87+
return containerservice.ManagedCluster{}, errors.Wrap(err, "failed to prepare operation")
88+
}
89+
for key, value := range headers {
90+
preparer.Header.Add(key, value)
91+
}
92+
93+
future, err := ac.managedclusters.CreateOrUpdateSender(preparer)
8694
if err != nil {
8795
return containerservice.ManagedCluster{}, errors.Wrap(err, "failed to begin operation")
8896
}

‎azure/services/managedclusters/managedclusters.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"k8s.io/klog/v2"
3131
infrav1alpha4 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
3232
"sigs.k8s.io/cluster-api-provider-azure/azure"
33+
"sigs.k8s.io/cluster-api-provider-azure/util/maps"
3334
"sigs.k8s.io/cluster-api-provider-azure/util/tele"
3435
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
3536
)
@@ -42,6 +43,7 @@ var (
4243
// ManagedClusterScope defines the scope interface for a managed cluster.
4344
type ManagedClusterScope interface {
4445
azure.ClusterDescriber
46+
ManagedClusterAnnotations() map[string]string
4547
ManagedClusterSpec() (azure.ManagedClusterSpec, error)
4648
GetAllAgentPoolSpecs(ctx context.Context) ([]azure.AgentPoolSpec, error)
4749
SetControlPlaneEndpoint(clusterv1.APIEndpoint)
@@ -292,8 +294,9 @@ func (s *Service) Reconcile(ctx context.Context) error {
292294
}
293295
}
294296

297+
customHeaders := maps.FilterByKeyPrefix(s.Scope.ManagedClusterAnnotations(), azure.CustomHeaderPrefix)
295298
if isCreate {
296-
managedCluster, err = s.Client.CreateOrUpdate(ctx, managedClusterSpec.ResourceGroupName, managedClusterSpec.Name, managedCluster)
299+
managedCluster, err = s.Client.CreateOrUpdate(ctx, managedClusterSpec.ResourceGroupName, managedClusterSpec.Name, managedCluster, customHeaders)
297300
if err != nil {
298301
return fmt.Errorf("failed to create managed cluster, %w", err)
299302
}
@@ -322,7 +325,7 @@ func (s *Service) Reconcile(ctx context.Context) error {
322325
diff := computeDiffOfNormalizedClusters(managedCluster, existingMC)
323326
if diff != "" {
324327
klog.V(2).Infof("Update required (+new -old):\n%s", diff)
325-
managedCluster, err = s.Client.CreateOrUpdate(ctx, managedClusterSpec.ResourceGroupName, managedClusterSpec.Name, managedCluster)
328+
managedCluster, err = s.Client.CreateOrUpdate(ctx, managedClusterSpec.ResourceGroupName, managedClusterSpec.Name, managedCluster, customHeaders)
326329
if err != nil {
327330
return fmt.Errorf("failed to update managed cluster, %w", err)
328331
}

‎azure/services/managedclusters/managedclusters_test.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ func TestReconcile(t *testing.T) {
4343
provisioningStatesToTest: []string{"Canceled", "Succeeded", "Failed"},
4444
expectedError: "",
4545
expect: func(m *mock_managedclusters.MockClientMockRecorder, provisioningstate string, s *mock_managedclusters.MockManagedClusterScopeMockRecorder) {
46-
m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-managedcluster", gomock.Any()).Return(containerservice.ManagedCluster{ManagedClusterProperties: &containerservice.ManagedClusterProperties{
46+
m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-managedcluster", gomock.Any(), map[string]string{"myFeature": "true"}).Return(containerservice.ManagedCluster{ManagedClusterProperties: &containerservice.ManagedClusterProperties{
4747
Fqdn: pointer.String("my-managedcluster-fqdn"),
4848
ProvisioningState: &provisioningstate,
4949
}}, nil)
@@ -55,6 +55,9 @@ func TestReconcile(t *testing.T) {
5555
m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Times(1)
5656
s.ClusterName().AnyTimes().Return("my-managedcluster")
5757
s.ResourceGroup().AnyTimes().Return("my-rg")
58+
s.ManagedClusterAnnotations().Times(1).Return(map[string]string{
59+
"infrastructure.cluster.x-k8s.io/custom-header-myFeature": "true",
60+
})
5861
s.ManagedClusterSpec().AnyTimes().Return(azure.ManagedClusterSpec{
5962
Name: "my-managedcluster",
6063
ResourceGroupName: "my-rg",
@@ -73,6 +76,7 @@ func TestReconcile(t *testing.T) {
7376
}}, nil)
7477
s.ClusterName().AnyTimes().Return("my-managedcluster")
7578
s.ResourceGroup().AnyTimes().Return("my-rg")
79+
s.ManagedClusterAnnotations().Times(1).Return(map[string]string{})
7680
s.ManagedClusterSpec().AnyTimes().Return(azure.ManagedClusterSpec{
7781
Name: "my-managedcluster",
7882
ResourceGroupName: "my-rg",
@@ -120,11 +124,12 @@ func TestReconcile(t *testing.T) {
120124
name: "no managedcluster exists",
121125
expectedError: "",
122126
expect: func(m *mock_managedclusters.MockClientMockRecorder, s *mock_managedclusters.MockManagedClusterScopeMockRecorder) {
123-
m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-managedcluster", gomock.Any()).Return(containerservice.ManagedCluster{ManagedClusterProperties: &containerservice.ManagedClusterProperties{}}, nil)
127+
m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-managedcluster", gomock.Any(), gomock.Any()).Return(containerservice.ManagedCluster{ManagedClusterProperties: &containerservice.ManagedClusterProperties{}}, nil)
124128
m.Get(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return(containerservice.ManagedCluster{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 404}, "Not Found"))
125129
m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Times(1)
126130
s.ClusterName().AnyTimes().Return("my-managedcluster")
127131
s.ResourceGroup().AnyTimes().Return("my-rg")
132+
s.ManagedClusterAnnotations().Times(1).Return(map[string]string{})
128133
s.ManagedClusterSpec().AnyTimes().Return(azure.ManagedClusterSpec{
129134
Name: "my-managedcluster",
130135
ResourceGroupName: "my-rg",

‎azure/services/managedclusters/mock_managedclusters/client_mock.go‎

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)