Skip to content

Commit bff377c

Browse files
cameronmeissnercameronmeissner
authored
test(e2e): assign kubelet and abe2e identities to E2E VMs, use kubelet identity to authenticate with private ACR (#7352)
1 parent 0913cbc commit bff377c

File tree

6 files changed

+46
-52
lines changed

6 files changed

+46
-52
lines changed

‎e2e/cluster.go‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,11 +100,11 @@ func prepareCluster(ctx context.Context, cluster *armcontainerservice.ManagedClu
100100
}
101101

102102
if isNonAnonymousPull {
103-
identity, err := config.Azure.UserAssignedIdentities.Get(ctx, resourceGroupName, config.VMIdentityName, nil)
103+
kubeletIdentity, err := getClusterKubeletIdentity(cluster)
104104
if err != nil {
105-
return nil, fmt.Errorf("failed to get VM identity: %w", err)
105+
return nil, fmt.Errorf("getting kubelet identity to assign ACR pull permissions: %w", err)
106106
}
107-
if err := assignACRPullToIdentity(ctx, config.GetPrivateACRName(isNonAnonymousPull, *cluster.Location), *identity.Properties.PrincipalID, *cluster.Location); err != nil {
107+
if err := assignACRPullToIdentity(ctx, config.GetPrivateACRName(isNonAnonymousPull, *cluster.Location), *kubeletIdentity.ObjectID, *cluster.Location); err != nil {
108108
return nil, fmt.Errorf("assign acr pull to the managed identity: %w", err)
109109
}
110110
}
@@ -139,6 +139,17 @@ func prepareCluster(ctx context.Context, cluster *armcontainerservice.ManagedClu
139139
}, nil
140140
}
141141

142+
func getClusterKubeletIdentity(cluster *armcontainerservice.ManagedCluster) (*armcontainerservice.UserAssignedIdentity, error) {
143+
if cluster == nil || cluster.Properties == nil || cluster.Properties.IdentityProfile == nil {
144+
return nil, fmt.Errorf("cannot dereference cluster identity profile to extract kubelet identity ID")
145+
}
146+
kubeletIdentity := cluster.Properties.IdentityProfile["kubeletidentity"]
147+
if kubeletIdentity == nil {
148+
return nil, fmt.Errorf("kubelet identity is missing from cluster properties")
149+
}
150+
return kubeletIdentity, nil
151+
}
152+
142153
func extractClusterParameters(ctx context.Context, kube *Kubeclient, cluster *armcontainerservice.ManagedCluster) (*ClusterParams, error) {
143154
kubeconfig, err := clientcmd.Load(kube.KubeConfig)
144155
if err != nil {

‎e2e/node_config.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ var baseKubeletConfig = &aksnodeconfigv1.KubeletConfig{
9797
},
9898
}
9999

100-
func getBaseNBC(t testing.TB, cluster *Cluster, vhd *config.Image) *datamodel.NodeBootstrappingConfiguration {
100+
func getBaseNBC(t testing.TB, cluster *Cluster, vhd *config.Image) (*datamodel.NodeBootstrappingConfiguration, error) {
101101
var nbc *datamodel.NodeBootstrappingConfiguration
102102

103103
if vhd.Distro.IsWindowsDistro() {
@@ -110,17 +110,23 @@ func getBaseNBC(t testing.TB, cluster *Cluster, vhd *config.Image) *datamodel.No
110110
nbc.ContainerService.Properties.ClusterID = *cluster.Model.ID
111111
nbc.SubscriptionID = config.Config.SubscriptionID
112112
nbc.ResourceGroupName = *cluster.Model.Properties.NodeResourceGroup
113-
nbc.TenantID = *cluster.Model.Identity.TenantID
114113
} else {
115114
nbc = baseTemplateLinux(t, *cluster.Model.Location, *cluster.Model.Properties.CurrentKubernetesVersion, vhd.Arch)
116115
}
117116

117+
kubeletIdentity, err := getClusterKubeletIdentity(cluster.Model)
118+
if err != nil {
119+
return nil, fmt.Errorf("getting cluster kubelet identity to create base NodeBootstrappingConfiguration: %w", err)
120+
}
121+
nbc.UserAssignedIdentityClientID = *kubeletIdentity.ClientID
122+
123+
nbc.TenantID = *cluster.Model.Identity.TenantID
118124
nbc.ContainerService.Properties.CertificateProfile.CaCertificate = string(cluster.ClusterParams.CACert)
119125
nbc.KubeletClientTLSBootstrapToken = &cluster.ClusterParams.BootstrapToken
120126
nbc.ContainerService.Properties.HostedMasterProfile.FQDN = cluster.ClusterParams.FQDN
121127
nbc.ContainerService.Properties.AgentPoolProfiles[0].Distro = vhd.Distro
122128
nbc.AgentPoolProfile.Distro = vhd.Distro
123-
return nbc
129+
return nbc, nil
124130
}
125131

126132
// is a temporary workaround

‎e2e/scenario_test.go‎

Lines changed: 10 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -195,8 +195,9 @@ func Test_AzureLinuxV2_SecureTLSBootstrapping_BootstrapToken_Fallback(t *testing
195195
VHD: config.VHDAzureLinuxV2Gen2,
196196
BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
197197
nbc.SecureTLSBootstrappingConfig = &datamodel.SecureTLSBootstrappingConfig{
198-
Enabled: true,
199-
Deadline: (30 * time.Second).String(),
198+
Enabled: true,
199+
Deadline: (30 * time.Second).String(),
200+
AADResource: "https://management.azure.com/", // use an unexpected AAD resource to force a secure TLS bootstrapping failure
200201
}
201202
},
202203
},
@@ -899,12 +900,6 @@ func Test_Ubuntu2204_AirGap(t *testing.T) {
899900
// TODO: refactor NonAnonymous tests to use the same cluster as Anonymous airgap
900901
// or deprecate anonymous ACR airgap tests once it is unsupported
901902
func Test_Ubuntu2204_AirGap_NonAnonymousACR(t *testing.T) {
902-
location := config.Config.DefaultLocation
903-
904-
ctx := newTestCtx(t)
905-
identity, err := config.Azure.UserAssignedIdentities.Get(ctx, config.ResourceGroupName(location), config.VMIdentityName, nil)
906-
require.NoError(t, err)
907-
908903
RunScenario(t, &Scenario{
909904
Description: "Tests that a node using the Ubuntu 2204 VHD and is airgap can be properly bootstrapped",
910905
Tags: Tags{
@@ -915,9 +910,6 @@ func Test_Ubuntu2204_AirGap_NonAnonymousACR(t *testing.T) {
915910
Cluster: ClusterKubenetAirgapNonAnon,
916911
VHD: config.VHDUbuntu2204Gen2Containerd,
917912
BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
918-
nbc.TenantID = *identity.Properties.TenantID
919-
nbc.UserAssignedIdentityClientID = *identity.Properties.ClientID
920-
921913
nbc.OutboundType = datamodel.OutboundTypeBlock
922914
nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
923915
PrivateEgress: &datamodel.PrivateEgress{
@@ -971,10 +963,6 @@ func Test_Ubuntu2204Gen2_ContainerdAirgappedK8sNotCached(t *testing.T) {
971963
}
972964

973965
func Test_Ubuntu2204Gen2_ContainerdAirgappedNonAnonymousK8sNotCached(t *testing.T) {
974-
location := config.Config.DefaultLocation
975-
ctx := newTestCtx(t)
976-
identity, err := config.Azure.UserAssignedIdentities.Get(ctx, config.ResourceGroupName(location), config.VMIdentityName, nil)
977-
require.NoError(t, err)
978966
RunScenario(t, &Scenario{
979967
Description: "Tests that a node using the Ubuntu 2204 VHD without k8s binary and is airgap can be properly bootstrapped",
980968
Tags: Tags{
@@ -985,13 +973,11 @@ func Test_Ubuntu2204Gen2_ContainerdAirgappedNonAnonymousK8sNotCached(t *testing.
985973
Cluster: ClusterKubenetAirgapNonAnon,
986974
VHD: config.VHDUbuntu2204Gen2ContainerdAirgappedK8sNotCached,
987975
BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
988-
nbc.TenantID = *identity.Properties.TenantID
989-
nbc.UserAssignedIdentityClientID = *identity.Properties.ClientID
990976
nbc.OutboundType = datamodel.OutboundTypeBlock
991977
nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
992978
PrivateEgress: &datamodel.PrivateEgress{
993979
Enabled: true,
994-
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io", config.PrivateACRNameNotAnon(location)),
980+
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io", config.PrivateACRNameNotAnon(config.Config.DefaultLocation)),
995981
},
996982
}
997983
nbc.AgentPoolProfile.LocalDNSProfile = nil
@@ -1000,7 +986,7 @@ func Test_Ubuntu2204Gen2_ContainerdAirgappedNonAnonymousK8sNotCached(t *testing.
1000986
// intentionally using private acr url to get kube binaries
1001987
nbc.AgentPoolProfile.KubernetesConfig.CustomKubeBinaryURL = fmt.Sprintf(
1002988
"%s.azurecr.io/oss/binaries/kubernetes/kubernetes-node:v%s-linux-amd64",
1003-
config.PrivateACRNameNotAnon(location),
989+
config.PrivateACRNameNotAnon(config.Config.DefaultLocation),
1004990
nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion)
1005991
nbc.K8sComponents.LinuxCredentialProviderURL = fmt.Sprintf(
1006992
"https://packages.aks.azure.com/cloud-provider-azure/v%s/binaries/azure-acr-credential-provider-linux-amd64-v%s.tar.gz",
@@ -1910,8 +1896,9 @@ func Test_Ubuntu2404Gen2_SecureTLSBootstrapping_BootstrapToken_Fallback(t *testi
19101896
VHD: config.VHDUbuntu2404Gen2Containerd,
19111897
BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
19121898
nbc.SecureTLSBootstrappingConfig = &datamodel.SecureTLSBootstrappingConfig{
1913-
Enabled: true,
1914-
Deadline: (30 * time.Second).String(),
1899+
Enabled: true,
1900+
Deadline: (30 * time.Second).String(),
1901+
AADResource: "https://management.azure.com/", // use an unexpected AAD resource to force a secure TLS bootstrapping failure
19151902
}
19161903
},
19171904
},
@@ -2118,10 +2105,6 @@ func Test_Ubuntu2204_PMC_Install(t *testing.T) {
21182105
}
21192106

21202107
func Test_Ubuntu2204Gen2_ContainerdAirgappedNonAnonymousK8sNotCached_InstallPackage(t *testing.T) {
2121-
location := config.Config.DefaultLocation
2122-
ctx := newTestCtx(t)
2123-
identity, err := config.Azure.UserAssignedIdentities.Get(ctx, config.ResourceGroupName(location), config.VMIdentityName, nil)
2124-
require.NoError(t, err)
21252108
RunScenario(t, &Scenario{
21262109
Description: "Tests that a node using the Ubuntu 2204 VHD without k8s binary and is airgap can be properly bootstrapped",
21272110
Tags: Tags{
@@ -2132,13 +2115,11 @@ func Test_Ubuntu2204Gen2_ContainerdAirgappedNonAnonymousK8sNotCached_InstallPack
21322115
Cluster: ClusterKubenetAirgapNonAnon,
21332116
VHD: config.VHDUbuntu2204Gen2ContainerdAirgappedK8sNotCached,
21342117
BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
2135-
nbc.TenantID = *identity.Properties.TenantID
2136-
nbc.UserAssignedIdentityClientID = *identity.Properties.ClientID
21372118
nbc.OutboundType = datamodel.OutboundTypeBlock
21382119
nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
21392120
PrivateEgress: &datamodel.PrivateEgress{
21402121
Enabled: true,
2141-
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io", config.PrivateACRNameNotAnon(location)),
2122+
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io", config.PrivateACRNameNotAnon(config.Config.DefaultLocation)),
21422123
},
21432124
}
21442125
nbc.AgentPoolProfile.LocalDNSProfile = nil
@@ -2147,7 +2128,7 @@ func Test_Ubuntu2204Gen2_ContainerdAirgappedNonAnonymousK8sNotCached_InstallPack
21472128
// intentionally using private acr url to get kube binaries
21482129
nbc.AgentPoolProfile.KubernetesConfig.CustomKubeBinaryURL = fmt.Sprintf(
21492130
"%s.azurecr.io/oss/binaries/kubernetes/kubernetes-node:v%s-linux-amd64",
2150-
config.PrivateACRNameNotAnon(location),
2131+
config.PrivateACRNameNotAnon(config.Config.DefaultLocation),
21512132
nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion)
21522133
nbc.K8sComponents.LinuxCredentialProviderURL = fmt.Sprintf(
21532134
"https://packages.aks.azure.com/cloud-provider-azure/v%s/binaries/azure-acr-credential-provider-linux-amd64-v%s.tar.gz",

‎e2e/test_helpers.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,9 @@ func prepareAKSNode(ctx context.Context, s *Scenario) *ScenarioVM {
229229
s.T.Fatalf("exactly one of BootstrapConfigMutator or AKSNodeConfigMutator must be set")
230230
}
231231

232-
nbc := getBaseNBC(s.T, s.Runtime.Cluster, s.VHD)
232+
var err error
233+
nbc, err := getBaseNBC(s.T, s.Runtime.Cluster, s.VHD)
234+
require.NoError(s.T, err)
233235

234236
if s.IsWindows() {
235237
nbc.ContainerService.Properties.WindowsProfile.CseScriptsPackageURL = "https://packages.aks.azure.com/aks/windows/cse/"
@@ -244,7 +246,6 @@ func prepareAKSNode(ctx context.Context, s *Scenario) *ScenarioVM {
244246
s.AKSNodeConfigMutator(nodeconfig)
245247
s.Runtime.AKSNodeConfig = nodeconfig
246248
}
247-
var err error
248249
publicKeyData := datamodel.PublicKey{KeyData: string(SSHKeyPublic)}
249250

250251
// check it all.

‎e2e/types.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,6 @@ func (s *Scenario) updateTags(ctx context.Context, vmss *armcompute.VirtualMachi
238238
}
239239
}
240240
vmss.Tags["owner"] = to.Ptr(owner)
241-
242241
}
243242

244243
func getLoggedInAzUser() (string, error) {

‎e2e/vmss.go‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -194,20 +194,16 @@ func createVMSSModel(ctx context.Context, s *Scenario) armcompute.VirtualMachine
194194
}
195195

196196
model := getBaseVMSSModel(s, customData, cse)
197-
if s.Tags.NonAnonymousACR {
198-
// add acr pull identity
199-
userAssignedIdentity := fmt.Sprintf(
200-
"/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ManagedIdentity/userAssignedIdentities/%s",
201-
config.Config.SubscriptionID,
202-
config.ResourceGroupName(s.Location),
203-
config.VMIdentityName,
204-
)
205-
model.Identity = &armcompute.VirtualMachineScaleSetIdentity{
206-
Type: to.Ptr(armcompute.ResourceIdentityTypeSystemAssignedUserAssigned),
207-
UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{
208-
userAssignedIdentity: {},
209-
},
210-
}
197+
198+
// always assign the abe2e + kubelet identities to the VMSS
199+
kubeletIdentity, err := getClusterKubeletIdentity(s.Runtime.Cluster.Model)
200+
require.NoError(s.T, err)
201+
model.Identity = &armcompute.VirtualMachineScaleSetIdentity{
202+
Type: to.Ptr(armcompute.ResourceIdentityTypeSystemAssignedUserAssigned),
203+
UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{
204+
*kubeletIdentity.ResourceID: {},
205+
config.Config.VMIdentityResourceID(s.Location): {},
206+
},
211207
}
212208

213209
isAzureCNI, err := cluster.IsAzureCNI()

0 commit comments

Comments
 (0)