Skip to content

Commit 1528bdd

Browse files
fseldowCopilot
fseldow
and
Copilot
authored
test: refactor network isolated cluster e2e (#7885)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent f30e520 commit 1528bdd

File tree

9 files changed

+208
-205
lines changed

9 files changed

+208
-205
lines changed

‎e2e/aks_model.go‎

Lines changed: 40 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -471,21 +471,49 @@ func addFirewallRules(
471471
return nil
472472
}
473473

474-
func addAirgapNetworkSettings(ctx context.Context, clusterModel *armcontainerservice.ManagedCluster, privateACRName, location string) error {
475-
toolkit.Logf(ctx, "Adding network settings for airgap cluster %s in rg %s", *clusterModel.Name, *clusterModel.Properties.NodeResourceGroup)
474+
func addPrivateAzureContainerRegistry(ctx context.Context, cluster *armcontainerservice.ManagedCluster, kube *Kubeclient, resourceGroupName string, kubeletIdentity *armcontainerservice.UserAssignedIdentity, isNonAnonymousPull bool) error {
475+
if cluster == nil || kube == nil || kubeletIdentity == nil {
476+
return errors.New("cluster, kubeclient, and kubeletIdentity cannot be nil when adding Private Azure Container Registry")
477+
}
478+
if err := createPrivateAzureContainerRegistry(ctx, cluster, resourceGroupName, isNonAnonymousPull); err != nil {
479+
return fmt.Errorf("failed to create private acr: %w", err)
480+
}
481+
482+
if err := createPrivateAzureContainerRegistryPullSecret(ctx, cluster, kube, resourceGroupName, isNonAnonymousPull); err != nil {
483+
return fmt.Errorf("create private acr pull secret: %w", err)
484+
}
485+
vnet, err := getClusterVNet(ctx, *cluster.Properties.NodeResourceGroup)
486+
if err != nil {
487+
return err
488+
}
489+
490+
err = addPrivateEndpointForACR(ctx, *cluster.Properties.NodeResourceGroup, config.GetPrivateACRName(isNonAnonymousPull, *cluster.Location), vnet, *cluster.Location)
491+
if err != nil {
492+
return err
493+
}
494+
495+
if err := assignACRPullToIdentity(ctx, config.GetPrivateACRName(isNonAnonymousPull, *cluster.Location), *kubeletIdentity.ObjectID, *cluster.Location); err != nil {
496+
return fmt.Errorf("assigning acr pull permissions to kubelet identity: %w", err)
497+
}
498+
499+
return nil
500+
}
501+
502+
func addNetworkIsolatedSettings(ctx context.Context, clusterModel *armcontainerservice.ManagedCluster, location string) error {
503+
defer toolkit.LogStepCtx(ctx, fmt.Sprintf("Adding network settings for network isolated cluster %s in rg %s", *clusterModel.Name, *clusterModel.Properties.NodeResourceGroup))
476504

477505
vnet, err := getClusterVNet(ctx, *clusterModel.Properties.NodeResourceGroup)
478506
if err != nil {
479507
return err
480508
}
481509
subnetId := vnet.subnetId
482510

483-
nsgParams, err := airGapSecurityGroup(location, *clusterModel.Properties.Fqdn)
511+
nsgParams, err := networkIsolatedSecurityGroup(location, *clusterModel.Properties.Fqdn)
484512
if err != nil {
485513
return err
486514
}
487515

488-
nsg, err := createAirgapSecurityGroup(ctx, clusterModel, nsgParams, nil)
516+
nsg, err := createNetworkIsolatedSecurityGroup(ctx, clusterModel, nsgParams, nil)
489517
if err != nil {
490518
return err
491519
}
@@ -503,19 +531,14 @@ func addAirgapNetworkSettings(ctx context.Context, clusterModel *armcontainerser
503531
return err
504532
}
505533

506-
err = addPrivateEndpointForACR(ctx, *clusterModel.Properties.NodeResourceGroup, privateACRName, vnet, location)
507-
if err != nil {
508-
return err
509-
}
510-
511-
toolkit.Logf(ctx, "updated cluster %s subnet with airgap settings", *clusterModel.Name)
534+
toolkit.Logf(ctx, "updated cluster %s subnet with network isolated cluster settings", *clusterModel.Name)
512535
return nil
513536
}
514537

515-
func airGapSecurityGroup(location, clusterFQDN string) (armnetwork.SecurityGroup, error) {
538+
func networkIsolatedSecurityGroup(location, clusterFQDN string) (armnetwork.SecurityGroup, error) {
516539
requiredRules, err := getRequiredSecurityRules(clusterFQDN)
517540
if err != nil {
518-
return armnetwork.SecurityGroup{}, fmt.Errorf("failed to get required security rules for airgap resource group: %w", err)
541+
return armnetwork.SecurityGroup{}, fmt.Errorf("failed to get required security rules for network isolated resource group: %w", err)
519542
}
520543

521544
allowVnet := &armnetwork.SecurityRule{
@@ -550,7 +573,7 @@ func airGapSecurityGroup(location, clusterFQDN string) (armnetwork.SecurityGroup
550573

551574
return armnetwork.SecurityGroup{
552575
Location: &location,
553-
Name: &config.Config.AirgapNSGName,
576+
Name: &config.Config.NetworkIsolatedNSGName,
554577
Properties: &armnetwork.SecurityGroupPropertiesFormat{SecurityRules: rules},
555578
}, nil
556579
}
@@ -559,7 +582,7 @@ func addPrivateEndpointForACR(ctx context.Context, nodeResourceGroup, privateACR
559582
toolkit.Logf(ctx, "Checking if private endpoint for private container registry is in rg %s", nodeResourceGroup)
560583
var err error
561584
var privateEndpoint *armnetwork.PrivateEndpoint
562-
privateEndpointName := "PE-for-ABE2ETests"
585+
privateEndpointName := fmt.Sprintf("PE-for-%s", privateACRName)
563586
if privateEndpoint, err = createPrivateEndpoint(ctx, nodeResourceGroup, privateEndpointName, privateACRName, vnet, location); err != nil {
564587
return err
565588
}
@@ -718,7 +741,7 @@ func deletePrivateAzureContainerRegistry(ctx context.Context, resourceGroup, pri
718741
return nil
719742
}
720743

721-
// if the ACR needs to be recreated so does the airgap k8s cluster
744+
// if the ACR needs to be recreated so does the network isolated k8s cluster
722745
func shouldRecreateACR(ctx context.Context, resourceGroup, privateACRName string) (error, bool) {
723746
toolkit.Logf(ctx, "Checking if private Azure Container Registry cache rules are correct in rg %s", resourceGroup)
724747

@@ -1010,8 +1033,8 @@ func getSecurityRule(name, destinationAddressPrefix string, priority int32) *arm
10101033
}
10111034
}
10121035

1013-
func createAirgapSecurityGroup(ctx context.Context, cluster *armcontainerservice.ManagedCluster, nsgParams armnetwork.SecurityGroup, options *armnetwork.SecurityGroupsClientBeginCreateOrUpdateOptions) (*armnetwork.SecurityGroupsClientCreateOrUpdateResponse, error) {
1014-
poller, err := config.Azure.SecurityGroup.BeginCreateOrUpdate(ctx, *cluster.Properties.NodeResourceGroup, config.Config.AirgapNSGName, nsgParams, options)
1036+
func createNetworkIsolatedSecurityGroup(ctx context.Context, cluster *armcontainerservice.ManagedCluster, nsgParams armnetwork.SecurityGroup, options *armnetwork.SecurityGroupsClientBeginCreateOrUpdateOptions) (*armnetwork.SecurityGroupsClientCreateOrUpdateResponse, error) {
1037+
poller, err := config.Azure.SecurityGroup.BeginCreateOrUpdate(ctx, *cluster.Properties.NodeResourceGroup, config.Config.NetworkIsolatedNSGName, nsgParams, options)
10151038
if err != nil {
10161039
return nil, err
10171040
}

‎e2e/cache.go‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -160,25 +160,25 @@ func clusterKubenet(ctx context.Context, request ClusterRequest) (*Cluster, erro
160160
return prepareCluster(ctx, getKubenetClusterModel("abe2e-kubenet-v4", request.Location, request.K8sSystemPoolSKU), false, false)
161161
}
162162

163-
var ClusterKubenetAirgap = cachedFunc(clusterKubenetAirgap)
163+
var ClusterAzureNetwork = cachedFunc(clusterAzureNetwork)
164164

165-
// clusterKubenetAirgap creates an airgapped kubenet cluster (no internet access)
166-
func clusterKubenetAirgap(ctx context.Context, request ClusterRequest) (*Cluster, error) {
167-
return prepareCluster(ctx, getKubenetClusterModel("abe2e-kubenet-airgap-v3", request.Location, request.K8sSystemPoolSKU), true, false)
165+
// clusterAzureNetwork creates a cluster with Azure CNI networking
166+
func clusterAzureNetwork(ctx context.Context, request ClusterRequest) (*Cluster, error) {
167+
return prepareCluster(ctx, getAzureNetworkClusterModel("abe2e-azure-network-v3", request.Location, request.K8sSystemPoolSKU), false, false)
168168
}
169169

170-
var ClusterKubenetAirgapNonAnon = cachedFunc(clusterKubenetAirgapNonAnon)
170+
var ClusterAzureBootstrapProfileCache = cachedFunc(clusterAzureBootstrapProfileCache)
171171

172-
// clusterKubenetAirgapNonAnon creates an airgapped kubenet cluster with non-anonymous image pulls
173-
func clusterKubenetAirgapNonAnon(ctx context.Context, request ClusterRequest) (*Cluster, error) {
174-
return prepareCluster(ctx, getKubenetClusterModel("abe2e-kubenet-nonanonpull-airgap-v3", request.Location, request.K8sSystemPoolSKU), true, true)
172+
// clusterAzureBootstrapProfileCache creates a cluster with bootstrap profile cache but without network isolation
173+
func clusterAzureBootstrapProfileCache(ctx context.Context, request ClusterRequest) (*Cluster, error) {
174+
return prepareCluster(ctx, getAzureNetworkClusterModel("abe2e-azure-bootstrapprofile-cache-v1", request.Location, request.K8sSystemPoolSKU), false, true)
175175
}
176176

177-
var ClusterAzureNetwork = cachedFunc(clusterAzureNetwork)
177+
var ClusterAzureNetworkIsolated = cachedFunc(clusterAzureNetworkIsolated)
178178

179-
// clusterAzureNetwork creates a cluster with Azure CNI networking
180-
func clusterAzureNetwork(ctx context.Context, request ClusterRequest) (*Cluster, error) {
181-
return prepareCluster(ctx, getAzureNetworkClusterModel("abe2e-azure-network-v3", request.Location, request.K8sSystemPoolSKU), false, false)
179+
// clusterAzureNetworkIsolated creates a networkisolated Azure network cluster (no internet access)
180+
func clusterAzureNetworkIsolated(ctx context.Context, request ClusterRequest) (*Cluster, error) {
181+
return prepareCluster(ctx, getAzureNetworkClusterModel("abe2e-azure-networkisolated-v1", request.Location, request.K8sSystemPoolSKU), true, false)
182182
}
183183

184184
var ClusterAzureOverlayNetwork = cachedFunc(clusterAzureOverlayNetwork)

‎e2e/cluster.go‎

Lines changed: 19 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ func (c *Cluster) MaxPodsPerNode() (int, error) {
5959
return 0, fmt.Errorf("cluster agentpool profiles were nil or empty: %+v", c.Model)
6060
}
6161

62-
func prepareCluster(ctx context.Context, cluster *armcontainerservice.ManagedCluster, isAirgap, isNonAnonymousPull bool) (*Cluster, error) {
62+
func prepareCluster(ctx context.Context, cluster *armcontainerservice.ManagedCluster, isNetworkIsolated, attachPrivateAcr bool) (*Cluster, error) {
6363
defer toolkit.LogStepCtx(ctx, "preparing cluster")()
6464
ctx, cancel := context.WithTimeout(ctx, config.Config.TestTimeoutCluster)
6565
defer cancel()
@@ -91,38 +91,32 @@ func prepareCluster(ctx context.Context, cluster *armcontainerservice.ManagedClu
9191
return nil, fmt.Errorf("get kube client using cluster %q: %w", *cluster.Name, err)
9292
}
9393

94-
toolkit.Logf(ctx, "using private acr %q isNonAnonymousPull %v", config.GetPrivateACRName(isNonAnonymousPull, *cluster.Location), isNonAnonymousPull)
95-
if isAirgap {
96-
// private acr must be created before we add the debug daemonsets
97-
if err := createPrivateAzureContainerRegistry(ctx, cluster, resourceGroupName, isNonAnonymousPull); err != nil {
98-
return nil, fmt.Errorf("failed to create private acr: %w", err)
99-
}
100-
101-
if err := createPrivateAzureContainerRegistryPullSecret(ctx, cluster, kube, resourceGroupName, isNonAnonymousPull); err != nil {
102-
return nil, fmt.Errorf("create private acr pull secret: %w", err)
103-
}
104-
105-
if err := addAirgapNetworkSettings(ctx, cluster, config.GetPrivateACRName(isNonAnonymousPull, *cluster.Location), *cluster.Location); err != nil {
106-
return nil, fmt.Errorf("add airgap network settings: %w", err)
107-
}
108-
}
109-
110-
if err := addFirewallRules(ctx, cluster, *cluster.Location); err != nil {
111-
return nil, fmt.Errorf("add firewall rules: %w", err)
112-
}
113-
11494
kubeletIdentity, err := getClusterKubeletIdentity(cluster)
11595
if err != nil {
11696
return nil, fmt.Errorf("getting cluster kubelet identity: %w", err)
11797
}
11898

119-
if isNonAnonymousPull {
120-
if err := assignACRPullToIdentity(ctx, config.GetPrivateACRName(isNonAnonymousPull, *cluster.Location), *kubeletIdentity.ObjectID, *cluster.Location); err != nil {
121-
return nil, fmt.Errorf("assigning acr pull permissions to kubelet identity: %w", err)
99+
if isNetworkIsolated || attachPrivateAcr {
100+
// private acr must be created before we add the debug daemonsets
101+
if err := addPrivateAzureContainerRegistry(ctx, cluster, kube, resourceGroupName, kubeletIdentity, true); err != nil {
102+
return nil, fmt.Errorf("add private azure container registry (true): %w", err)
103+
}
104+
if err := addPrivateAzureContainerRegistry(ctx, cluster, kube, resourceGroupName, kubeletIdentity, false); err != nil {
105+
return nil, fmt.Errorf("add private azure container registry (false): %w", err)
106+
}
107+
}
108+
if isNetworkIsolated {
109+
if err := addNetworkIsolatedSettings(ctx, cluster, *cluster.Location); err != nil {
110+
return nil, fmt.Errorf("add network isolated settings: %w", err)
111+
}
112+
}
113+
if !isNetworkIsolated { // network isolated cluster blocks all egress via NSG
114+
if err := addFirewallRules(ctx, cluster, *cluster.Location); err != nil {
115+
return nil, fmt.Errorf("add firewall rules: %w", err)
122116
}
123117
}
124118

125-
if err := kube.EnsureDebugDaemonsets(ctx, isAirgap, config.GetPrivateACRName(isNonAnonymousPull, *cluster.Location)); err != nil {
119+
if err := kube.EnsureDebugDaemonsets(ctx, isNetworkIsolated, config.GetPrivateACRName(true, *cluster.Location)); err != nil {
126120
return nil, fmt.Errorf("ensure debug daemonsets for %q: %w", *cluster.Name, err)
127121
}
128122

‎e2e/config/config.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,19 +37,18 @@ func ResourceGroupName(location string) string {
3737
}
3838

3939
func PrivateACRNameNotAnon(location string) string {
40-
return "e2eprivateacrnonanon" + location // will have anonymous pull enabled
40+
return "abe2eprivatenonanon" + location // will have anonymous pull enabled
4141
}
4242

4343
func PrivateACRName(location string) string {
44-
return "e2eprivateacr" + location // will not have anonymous pull enabled
44+
return "abe2eprivate" + location // will not have anonymous pull enabled
4545
}
4646

4747
type Configuration struct {
4848
// The defaults should only be used when running tests locally, as the CI will set these env vars.
4949
// We have separate Linux and Windows consts to have different defaults - they use the same env vars.
5050
ACRSecretName string `env:"ACR_SECRET_NAME" envDefault:"acr-secret-code2"`
51-
AirgapNSGName string `env:"AIRGAP_NSG_NAME" envDefault:"abe2e-airgap-securityGroup"`
52-
AzureContainerRegistrytargetRepository string `env:"ACR_TARGET_REPOSITORY" envDefault:"*"`
51+
AzureContainerRegistrytargetRepository string `env:"ACR_TARGET_REPOSITORY" envDefault:"aks-managed-repository/*"`
5352
BlobContainer string `env:"BLOB_CONTAINER" envDefault:"abe2e"`
5453
BlobStorageAccountPrefix string `env:"BLOB_STORAGE_ACCOUNT_PREFIX" envDefault:"abe2e"`
5554
BuildID string `env:"BUILD_ID" envDefault:"local"`
@@ -67,6 +66,7 @@ type Configuration struct {
6766
GallerySubscriptionIDWindows string `env:"GALLERY_SUBSCRIPTION_ID" envDefault:"c4c3550e-a965-4993-a50c-628fd38cd3e1"`
6867
IgnoreScenariosWithMissingVHD bool `env:"IGNORE_SCENARIOS_WITH_MISSING_VHD"`
6968
KeepVMSS bool `env:"KEEP_VMSS"`
69+
NetworkIsolatedNSGName string `env:"NETWORK_ISOLATED_NSG_NAME" envDefault:"abe2e-networkisolated-securityGroup"`
7070
SIGVersionTagName string `env:"SIG_VERSION_TAG_NAME" envDefault:"branch"`
7171
SIGVersionTagValue string `env:"SIG_VERSION_TAG_VALUE" envDefault:"refs/heads/main"`
7272
SkipTestsWithSKUCapacityIssue bool `env:"SKIP_TESTS_WITH_SKU_CAPACITY_ISSUE"`

‎e2e/config/vhd.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ var (
154154
}
155155

156156
// without kubelet, kubectl, credential-provider and wasm
157-
VHDUbuntu2204Gen2ContainerdAirgappedK8sNotCached = &Image{
157+
VHDUbuntu2204Gen2ContainerdNetworkIsolatedK8sNotCached = &Image{
158158
Name: "2204Gen2",
159159
OS: OSUbuntu,
160160
Arch: "amd64",

‎e2e/kube.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -293,14 +293,14 @@ func getClusterKubeconfigBytes(ctx context.Context, resourceGroupName, clusterNa
293293
}
294294

295295
// this is a bit ugly, but we don't want to execute this piece concurrently with other tests
296-
func (k *Kubeclient) EnsureDebugDaemonsets(ctx context.Context, isAirgap bool, privateACRName string) error {
297-
ds := daemonsetDebug(ctx, hostNetworkDebugAppLabel, "nodepool1", privateACRName, true, isAirgap)
296+
func (k *Kubeclient) EnsureDebugDaemonsets(ctx context.Context, isNetworkIsolated bool, privateACRName string) error {
297+
ds := daemonsetDebug(ctx, hostNetworkDebugAppLabel, "nodepool1", privateACRName, true, isNetworkIsolated)
298298
err := k.CreateDaemonset(ctx, ds)
299299
if err != nil {
300300
return err
301301
}
302302

303-
nonHostDS := daemonsetDebug(ctx, podNetworkDebugAppLabel, "nodepool2", privateACRName, false, isAirgap)
303+
nonHostDS := daemonsetDebug(ctx, podNetworkDebugAppLabel, "nodepool2", privateACRName, false, isNetworkIsolated)
304304
err = k.CreateDaemonset(ctx, nonHostDS)
305305
if err != nil {
306306
return err
@@ -358,11 +358,11 @@ func (k *Kubeclient) createKubernetesSecret(ctx context.Context, namespace, secr
358358
return nil
359359
}
360360

361-
func daemonsetDebug(ctx context.Context, deploymentName, targetNodeLabel, privateACRName string, isHostNetwork, isAirgap bool) *appsv1.DaemonSet {
361+
func daemonsetDebug(ctx context.Context, deploymentName, targetNodeLabel, privateACRName string, isHostNetwork, isNetworkIsolated bool) *appsv1.DaemonSet {
362362
image := "mcr.microsoft.com/cbl-mariner/base/core:2.0"
363363
secretName := ""
364-
if isAirgap {
365-
image = fmt.Sprintf("%s.azurecr.io/cbl-mariner/base/core:2.0", privateACRName)
364+
if isNetworkIsolated {
365+
image = fmt.Sprintf("%s.azurecr.io/aks-managed-repository/cbl-mariner/base/core:2.0", privateACRName)
366366
secretName = config.Config.ACRSecretName
367367
}
368368
toolkit.Logf(ctx, "Creating daemonset %s with image %s", deploymentName, image)

0 commit comments

Comments
 (0)