feat: add mooncake mcr containerd config host - linux (#7342)

Author: fseldow

e2e/aks_model.go

@@ -219,14 +219,29 @@ func getFirewall(ctx context.Context, location, firewallSubnetID, publicIPID str
 		TargetFqdns: []*string{to.Ptr(blobStorageFqdn)},
 	}
 
+	// needed for Mock Azure China Cloud tests
+	mooncakeMAR := "mcr.azure.cn"
+	mooncakeMARData := "*.data.mcr.azure.cn"
+	mooncakeMARRule := armnetwork.AzureFirewallApplicationRule{
+		Name:            to.Ptr("mooncake-mar-fqdn"),
+		SourceAddresses: []*string{to.Ptr("*")},
+		Protocols: []*armnetwork.AzureFirewallApplicationRuleProtocol{
+			{
+				ProtocolType: to.Ptr(armnetwork.AzureFirewallApplicationRuleProtocolTypeHTTPS),
+				Port:         to.Ptr[int32](443),
+			},
+		},
+		TargetFqdns: []*string{to.Ptr(mooncakeMAR), to.Ptr(mooncakeMARData)},
+	}
+
 	appRuleCollection := armnetwork.AzureFirewallApplicationRuleCollection{
 		Name: to.Ptr("aksfwar"),
 		Properties: &armnetwork.AzureFirewallApplicationRuleCollectionPropertiesFormat{
 			Priority: to.Ptr[int32](100),
 			Action: &armnetwork.AzureFirewallRCAction{
 				Type: to.Ptr(armnetwork.AzureFirewallRCActionTypeAllow),
 			},
-			Rules: []*armnetwork.AzureFirewallApplicationRule{&aksAppRule, &blobStorageAppRule},
+			Rules: []*armnetwork.AzureFirewallApplicationRule{&aksAppRule, &blobStorageAppRule, &mooncakeMARRule},
 		},
 	}
 

e2e/kube.go

@@ -471,6 +471,9 @@ func getClusterSubnetID(ctx context.Context, mcResourceGroupName string) (string
 
 func podHTTPServerLinux(s *Scenario) *corev1.Pod {
 	image := "mcr.microsoft.com/cbl-mariner/busybox:2.0"
+	if s.Tags.MockAzureChinaCloud {
+		image = "mcr.azk8s.cn/cbl-mariner/busybox:2.0"
+	}
 	return &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      fmt.Sprintf("%s-test-pod", s.Runtime.VM.KubeName),

e2e/scenario_test.go

@@ -1846,6 +1846,61 @@ func Test_Ubuntu2404Gen2(t *testing.T) {
 	})
 }
 
+func Test_Ubuntu2404Gen2_McrChinaCloud_Scriptless(t *testing.T) {
+	RunScenario(t, &Scenario{
+		Tags: Tags{
+			MockAzureChinaCloud: true,
+			Scriptless:          true,
+		},
+		Description: "Tests that a node using the Ubuntu 2404 VHD can be properly bootstrapped with containerd v2",
+		Config: Config{
+			Cluster: ClusterKubenet,
+			VHD:     config.VHDUbuntu2404Gen2Containerd,
+			AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) {
+			},
+			VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {
+				if vmss.Tags == nil {
+					vmss.Tags = map[string]*string{}
+				}
+				vmss.Tags["E2EMockAzureChinaCloud"] = to.Ptr("true")
+			},
+			Validator: func(ctx context.Context, s *Scenario) {
+				ValidateDirectoryContent(ctx, s, "/etc/containerd/certs.d/mcr.azk8s.cn", []string{"hosts.toml"})
+			},
+		},
+	})
+}
+
+func Test_Ubuntu2404Gen2_McrChinaCloud(t *testing.T) {
+	RunScenario(t, &Scenario{
+		Tags: Tags{
+			MockAzureChinaCloud: true,
+		},
+		Description: "Tests that a node using the Ubuntu 2404 VHD can be properly bootstrapped with containerd v2",
+		Config: Config{
+			Cluster: ClusterKubenet,
+			VHD:     config.VHDUbuntu2404Gen2Containerd,
+			BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
+			},
+			VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {
+				if vmss.Tags == nil {
+					vmss.Tags = map[string]*string{}
+				}
+				vmss.Tags["E2EMockAzureChinaCloud"] = to.Ptr("true")
+			},
+			Validator: func(ctx context.Context, s *Scenario) {
+				containerdVersions := components.GetExpectedPackageVersions("containerd", "ubuntu", "r2404")
+				runcVersions := components.GetExpectedPackageVersions("runc", "ubuntu", "r2404")
+				ValidateContainerd2Properties(ctx, s, containerdVersions)
+				ValidateRuncVersion(ctx, s, runcVersions)
+				ValidateContainerRuntimePlugins(ctx, s)
+				ValidateSSHServiceEnabled(ctx, s)
+				ValidateDirectoryContent(ctx, s, "/etc/containerd/certs.d/mcr.azk8s.cn", []string{"hosts.toml"})
+			},
+		},
+	})
+}
+
 func Test_Ubuntu2204_SecureTLSBootstrapping_BootstrapToken_Fallback(t *testing.T) {
 	RunScenario(t, &Scenario{
 		Description: "Tests that a node using an Ubuntu 2204 Gen2 VHD can be properly bootstrapped even if secure TLS bootstrapping fails",

e2e/types.go

@@ -33,6 +33,7 @@ type Tags struct {
 	KubeletCustomConfig    bool
 	Scriptless             bool
 	VHDCaching             bool
+	MockAzureChinaCloud    bool
 }
 
 // MatchesFilters checks if the Tags struct matches all given filters.

parts/linux/cloud-init/artifacts/cse_config.sh

@@ -355,8 +355,12 @@ EOF
     echo "${CONTAINERD_CONFIG_CONTENT}" | base64 -d > /etc/containerd/config.toml || exit $ERR_FILE_WATCH_TIMEOUT
   fi
 
+  export -f e2e_mock_azure_china_cloud
+  E2EMockAzureChinaCloud=$(retrycmd_silent 10 1 10 bash -cx e2e_mock_azure_china_cloud)
   if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then
     logs_to_events "AKS.CSE.ensureContainerd.configureContainerdRegistryHost" configureContainerdRegistryHost
+  elif [ "${TARGET_CLOUD}" = "AzureChinaCloud" ] || [ "${E2EMockAzureChinaCloud}" = "true" ]; then
+    logs_to_events "AKS.CSE.ensureContainerd.configureContainerdLegacyMooncakeMcrHost" configureContainerdLegacyMooncakeMcrHost
   fi
 
   tee "/etc/sysctl.d/99-force-bridge-forward.conf" > /dev/null <<EOF
@@ -384,6 +388,24 @@ configureContainerdRegistryHost() {
 EOF
 }
 
+# this function craetes containerd host config to map mcr.azk8s.cn host to mcr.azure.cn
+# containerd will resolve mcr.azk8s.cn as mcr.azure.cn and pull the image. If failed, it will fallback to mcr.azk8s.cn
+# https://github.com/containerd/containerd/blob/main/docs/hosts.md#registry-configuration---examples
+# TODO(xinhl): remove when aks rp fully deprecates mcr.azk8s.cn
+configureContainerdLegacyMooncakeMcrHost() {
+    LEGACY_MCR_REPOSITORY_BASE="mcr.azk8s.cn"
+    CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/${LEGACY_MCR_REPOSITORY_BASE}/hosts.toml"
+    mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
+    touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
+    chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
+
+    TARGET_MCR_REPOSITORY_BASE="mcr.azure.cn"
+    tee "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}" > /dev/null <<EOF
+[host."https://${TARGET_MCR_REPOSITORY_BASE}"]
+  capabilities = ["pull", "resolve"]
+EOF
+}
+
 ensureNoDupOnPromiscuBridge() {
     systemctlEnableAndStart ensure-no-dup 30 || exit $ERR_SYSTEMCTL_START_FAIL
 }

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -687,6 +687,17 @@ should_enforce_kube_pmc_install() {
     echo "${should_enforce,,}"
 }
 
+e2e_mock_azure_china_cloud() {
+    set -x
+    body=$(curl -fsSL -H "Metadata: true" --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2021-02-01")
+    ret=$?
+    if [ "$ret" -ne 0 ]; then
+      return $ret
+    fi
+    should_enforce=$(echo "$body" | jq -r '.compute.tagsList[] | select(.name == "E2EMockAzureChinaCloud") | .value')
+    echo "${should_enforce,,}"
+}
+
 enableManagedGPUExperience() {
     set -x
     body=$(curl -fsSL -H "Metadata: true" --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2021-02-01")