[Containerapp]: az containerapp arc: Enable setup custom core dns for Aks AzureCore on Arc. (#8428)

Author: LaylaLiu-gmail

src/containerapp/HISTORY.rst

@@ -16,6 +16,7 @@ upcoming
 * 'az containerapp env http-route-config': Add commands for the http-route-config feature area.
 * 'az containerapp env java-component': Support more flexible configuration updates with new parameters `--set-configurations`, `--replace-configurations`, `--remove-configurations` and `--remove-all-configurations`.
 * 'az containerapp env java-component gateway-for-spring create/update': Support `--bind` and `--unbind`
+* 'az containerapp arc': Enable setup custom core dns for Aks AzureCore on Arc.
 
 1.1.0b1
 ++++++

src/containerapp/azext_containerapp/_arc_utils.py

@@ -140,3 +140,23 @@
 RUNTIME_GENERIC = "generic"
 RUNTIME_JAVA = "java"
 SUPPORTED_RUNTIME_LIST = [RUNTIME_GENERIC, RUNTIME_JAVA]
+
+AKS_AZURE_LOCAL_DISTRO = "AksAzureLocal"
+SETUP_CORE_DNS_SUPPORTED_DISTRO = [AKS_AZURE_LOCAL_DISTRO]
+CUSTOM_CORE_DNS_VOLUME_NAME = 'custom-config-volume'
+CUSTOM_CORE_DNS_VOLUME_MOUNT_PATH = '/etc/coredns/custom'
+CUSTOM_CORE_DNS = 'coredns-custom'
+CORE_DNS = 'coredns'
+KUBE_SYSTEM = 'kube-system'
+EMPTY_CUSTOM_CORE_DNS = """
+apiVersion: v1
+data:
+kind: ConfigMap
+metadata:
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+  name: coredns-custom
+  namespace: kube-system
+"""

src/containerapp/azext_containerapp/_constants.py

@@ -1015,6 +1015,23 @@
           az containerapp connected-env list -g MyResourceGroup
 """
 
+helps['containerapp arc'] = """
+    type: group
+    short-summary: Install prerequisites for Kubernetes cluster on Arc
+"""
+
+helps['containerapp arc setup-core-dns'] = """
+    type: command
+    short-summary: Setup CoreDNS for Kubernetes cluster on Arc
+    examples:
+    - name: Setup CoreDNS for Aks on Azure Local on Arc
+      text: |
+          az containerapp arc setup-core-dns --distro AksAzureLocal
+    - name: Setup CoreDNS for Aks on Azure Local on Arc by specifying the kubeconfig and kubecontext.
+      text: |
+          az containerapp arc setup-core-dns --distro AksAzureLocal --kube-config /path/to/kubeconfig --kube-context kubeContextName
+"""
+
 helps['containerapp connected-env dapr-component'] = """
     type: group
     short-summary: Commands to manage Dapr components for Container Apps connected environments.

src/containerapp/azext_containerapp/_help.py

@@ -19,7 +19,8 @@
 from ._validators import (validate_env_name_or_id, validate_build_env_vars,
                           validate_custom_location_name_or_id, validate_env_name_or_id_for_up,
                           validate_otlp_headers, validate_target_port_range, validate_timeout_in_seconds)
-from ._constants import MAXIMUM_CONTAINER_APP_NAME_LENGTH, MAXIMUM_APP_RESILIENCY_NAME_LENGTH, MAXIMUM_COMPONENT_RESILIENCY_NAME_LENGTH
+from ._constants import (MAXIMUM_CONTAINER_APP_NAME_LENGTH, MAXIMUM_APP_RESILIENCY_NAME_LENGTH, MAXIMUM_COMPONENT_RESILIENCY_NAME_LENGTH,
+                         AKS_AZURE_LOCAL_DISTRO)
 
 
 def load_arguments(self, _):
@@ -365,6 +366,12 @@ def load_arguments(self, _):
         c.argument('environment_name', options_list=['--name', '-n'], help="The environment name.")
         c.argument('yaml', type=file_type, help='Path to a .yaml file with the configuration of a Dapr component. All other parameters will be ignored. For an example, see https://learn.microsoft.com/en-us/azure/container-apps/dapr-overview?tabs=bicep1%2Cyaml#component-schema')
 
+    with self.argument_context('containerapp arc setup-core-dns') as c:
+        c.argument('distro', arg_type=get_enum_type([AKS_AZURE_LOCAL_DISTRO]), required=True, help="The distro supported to setup CoreDNS.")
+        c.argument('kube_config', help="Path to the kube config file.")
+        c.argument('kube_context', help="Kube context from current machine.")
+        c.argument('skip_ssl_verification', help="Skip SSL verification for any cluster connection.")
+
     with self.argument_context('containerapp github-action add') as c:
         c.argument('build_env_vars', nargs='*', help="A list of environment variable(s) for the build. Space-separated values in 'key=value' format.",
                    validator=validate_build_env_vars, is_preview=True)

src/containerapp/azext_containerapp/_params.py

@@ -13,6 +13,7 @@
 import hashlib
 import re
 import requests
+import shutil
 import packaging.version as SemVer
 
 from enum import Enum

src/containerapp/azext_containerapp/_utils.py

@@ -187,6 +187,9 @@ def load_command_table(self, args):
             g.custom_command('set', 'connected_env_create_or_update_storage', supports_no_wait=True, exception_handler=ex_handler_factory())
             g.custom_command('remove', 'connected_env_remove_storage', supports_no_wait=True, confirmation=True, exception_handler=ex_handler_factory())
 
+    with self.command_group('containerapp arc', is_preview=True) as g:
+        g.custom_command('setup-core-dns', 'setup_core_dns', confirmation=True, exception_handler=ex_handler_factory())
+
     with self.command_group('containerapp env java-component') as g:
         g.custom_command('list', 'list_java_components')
 

src/containerapp/azext_containerapp/commands.py

@@ -9,6 +9,7 @@
 from urllib.parse import urlparse
 import json
 import requests
+import copy
 import subprocess
 from concurrent.futures import ThreadPoolExecutor
 
@@ -127,7 +128,13 @@
 
 from ._ssh_utils import (SSH_DEFAULT_ENCODING, DebugWebSocketConnection, read_debug_ssh)
 
-from ._utils import connected_env_check_cert_name_availability, get_oryx_run_image_tags, patchable_check, get_pack_exec_path, is_docker_running, parse_build_env_vars, env_has_managed_identity
+from ._utils import (connected_env_check_cert_name_availability, get_oryx_run_image_tags, patchable_check,
+                     get_pack_exec_path, is_docker_running, parse_build_env_vars, env_has_managed_identity)
+
+from ._arc_utils import (get_core_dns_deployment, get_core_dns_configmap, backup_custom_core_dns_configmap,
+                         replace_configmap, replace_deployment, delete_configmap, patch_coredns,
+                         create_folder, create_sub_folder,
+                         check_kube_connection, create_kube_client)
 
 from ._constants import (CONTAINER_APPS_RP,
                          NAME_INVALID, NAME_ALREADY_EXISTS, ACR_IMAGE_SUFFIX, DEV_POSTGRES_IMAGE, DEV_POSTGRES_SERVICE_TYPE,
@@ -136,7 +143,8 @@
                          DEV_QDRANT_CONTAINER_NAME, DEV_QDRANT_SERVICE_TYPE, DEV_WEAVIATE_IMAGE, DEV_WEAVIATE_CONTAINER_NAME, DEV_WEAVIATE_SERVICE_TYPE,
                          DEV_MILVUS_IMAGE, DEV_MILVUS_CONTAINER_NAME, DEV_MILVUS_SERVICE_TYPE, DEV_SERVICE_LIST, CONTAINER_APPS_SDK_MODELS, BLOB_STORAGE_TOKEN_STORE_SECRET_SETTING_NAME,
                          DAPR_SUPPORTED_STATESTORE_DEV_SERVICE_LIST, DAPR_SUPPORTED_PUBSUB_DEV_SERVICE_LIST,
-                         JAVA_COMPONENT_CONFIG, JAVA_COMPONENT_EUREKA, JAVA_COMPONENT_ADMIN, JAVA_COMPONENT_NACOS, JAVA_COMPONENT_GATEWAY, DOTNET_COMPONENT_RESOURCE_TYPE)
+                         JAVA_COMPONENT_CONFIG, JAVA_COMPONENT_EUREKA, JAVA_COMPONENT_ADMIN, JAVA_COMPONENT_NACOS, JAVA_COMPONENT_GATEWAY, DOTNET_COMPONENT_RESOURCE_TYPE,
+                         CUSTOM_CORE_DNS, CORE_DNS, KUBE_SYSTEM)
 
 
 logger = get_logger(__name__)
@@ -2064,6 +2072,105 @@ def connected_env_remove_storage(cmd, storage_name, name, resource_group_name, n
         handle_raw_exception(e)
 
 
+def setup_core_dns(cmd, distro=None, kube_config=None, kube_context=None, skip_ssl_verification=False):
+    # Checking the connection to kubernetes cluster.
+    check_kube_connection(kube_config, kube_context, skip_ssl_verification)
+
+    # create a local path to store the original and the changed deployment and core dns configmap.
+    time_stamp = time.strftime("%Y-%m-%d-%H.%M.%S")
+
+    parent_folder, folder_status, error = create_folder("setup-core-dns", time_stamp)
+    if not folder_status:
+        raise ValidationError(error)
+
+    original_folder, folder_status, error = create_sub_folder(parent_folder, "original")
+    if not folder_status:
+        raise ValidationError(error)
+
+    kube_client = create_kube_client(kube_config, kube_context, skip_ssl_verification)
+
+    # backup original deployment and configmap
+    logger.info("Backup existing coredns deployment and configmap")
+    original_coredns_deployment = get_core_dns_deployment(kube_client, original_folder)
+    coredns_deployment = copy.deepcopy(original_coredns_deployment)
+
+    original_coredns_configmap = get_core_dns_configmap(kube_client, original_folder)
+    coredns_configmap = copy.deepcopy(original_coredns_configmap)
+
+    volumes = coredns_deployment.spec.template.spec.volumes
+    if volumes is None:
+        raise ValidationError('Unexpected Volumes in coredns deployment, Volumes not found')
+
+    volume_mounts = coredns_deployment.spec.template.spec.containers[0].volume_mounts
+    if volume_mounts is None:
+        raise ValidationError('Unexpected Volume mounts in coredns deployment, VolumeMounts not found')
+
+    coredns_configmap_volume_set = False
+    custom_coredns_configmap_volume_set = False
+    custom_coredns_configmap_volume_mounted = False
+
+    for volume in volumes:
+        if volume.config_map is not None:
+            if volume.config_map.name == CORE_DNS:
+                for mount in volume_mounts:
+                    if mount.name is not None and mount.name == volume.name:
+                        coredns_configmap_volume_set = True
+                        break
+            elif volume.config_map.name == CUSTOM_CORE_DNS:
+                custom_coredns_configmap_volume_set = True
+                for mount in volume_mounts:
+                    if mount.name is not None and mount.name == volume.name:
+                        custom_coredns_configmap_volume_mounted = True
+                        break
+
+    if not coredns_configmap_volume_set:
+        raise ValidationError("Cannot find volume and volume mounts for core dns config map")
+
+    original_custom_core_dns_configmap = backup_custom_core_dns_configmap(kube_client, original_folder)
+
+    new_filepath_with_timestamp, folder_status, error = create_sub_folder(parent_folder, "new")
+    if not folder_status:
+        raise ValidationError(error)
+
+    try:
+        patch_coredns(kube_client, coredns_configmap, coredns_deployment, new_filepath_with_timestamp,
+                      original_custom_core_dns_configmap is not None, not custom_coredns_configmap_volume_set, not custom_coredns_configmap_volume_mounted)
+    except Exception as e:
+        logger.error(f"Failed to setup custom coredns. {e}")
+        logger.info("Start to reverted coredns")
+        replace_succeeded = False
+        retry_count = 0
+        while not replace_succeeded and retry_count < 10:
+            logger.info(f"Retry the revert operation with retry count {retry_count}")
+
+            try:
+                logger.info("Start to reverted coredns configmap")
+                latest_core_dns_configmap = get_core_dns_configmap(kube_client)
+                latest_core_dns_configmap.data = original_coredns_configmap.data
+
+                replace_configmap(CORE_DNS, KUBE_SYSTEM, kube_client, latest_core_dns_configmap)
+                logger.info("Reverted coredns configmap successfully")
+
+                logger.info("Start to reverted coredns deployment")
+                latest_core_dns_deployment = get_core_dns_deployment(kube_client)
+                latest_core_dns_deployment.spec.template.spec = original_coredns_deployment.spec.template.spec
+
+                replace_deployment(CORE_DNS, KUBE_SYSTEM, kube_client, latest_core_dns_deployment)
+                logger.info("Reverted coredns deployment successfully")
+
+                if original_custom_core_dns_configmap is None:
+                    delete_configmap(CUSTOM_CORE_DNS, KUBE_SYSTEM, kube_client)
+                replace_succeeded = True
+            except Exception as revertEx:
+                logger.warning(f"Failed to revert coredns configmap or deployment {revertEx}")
+                retry_count = retry_count + 1
+                time.sleep(2)
+
+        if not replace_succeeded:
+            logger.error(f"Failed to revert the deployment and configuration. "
+                         f"You can get the original coredns config and deployment from {original_folder}")
+
+
 def init_dapr_components(cmd, resource_group_name, environment_name, statestore="redis", pubsub="redis"):
     _validate_subscription_registered(cmd, CONTAINER_APPS_RP)
 

src/containerapp/azext_containerapp/custom.py

@@ -14,16 +14,18 @@
 
 
 # pylint: disable=too-many-instance-attributes
+
 class ConnectedClusterPreparer(NoTrafficRecordingPreparer, SingleValueReplacer):
     def __init__(self, name_prefix='aks', location='eastus2euap', aks_name='my-aks-cluster', connected_cluster_name='my-connected-cluster',
-                 resource_group_parameter_name='resource_group', skip_delete=False):
+                 resource_group_parameter_name='resource_group', skip_delete=False, skip_connected_cluster=False):
         super(ConnectedClusterPreparer, self).__init__(name_prefix, 15)
         self.cli_ctx = get_dummy_cli()
         self.location = location
         self.infra_cluster = aks_name
         self.connected_cluster_name = connected_cluster_name
         self.resource_group_parameter_name = resource_group_parameter_name
         self.skip_delete = skip_delete
+        self.skip_connected_cluster = skip_connected_cluster
 
     def create_resource(self, name, **kwargs):
         group = self._get_resource_group(**kwargs)
@@ -37,11 +39,12 @@ def create_resource(self, name, **kwargs):
             self.live_only_execute(self.cli_ctx, f'az aks create --resource-group {group} --name {self.infra_cluster} --enable-aad --generate-ssh-keys --enable-cluster-autoscaler --min-count 4 --max-count 10 --node-count 4 --location {aks_location}')
             self.live_only_execute(self.cli_ctx, f'az aks get-credentials --resource-group {group} --name {self.infra_cluster} --overwrite-existing --admin')
 
-            self.live_only_execute(self.cli_ctx, f'az connectedk8s connect --resource-group {group} --name {self.connected_cluster_name} --location {arc_location}')
-            connected_cluster = self.live_only_execute(self.cli_ctx, f'az connectedk8s show --resource-group {group} --name {self.connected_cluster_name}').get_output_in_json()
-            while connected_cluster is not None and connected_cluster["connectivityStatus"] == "Connecting":
-                time.sleep(5)
+            if not self.skip_connected_cluster:
+                self.live_only_execute(self.cli_ctx, f'az connectedk8s connect --resource-group {group} --name {self.connected_cluster_name} --location {arc_location}')
                 connected_cluster = self.live_only_execute(self.cli_ctx, f'az connectedk8s show --resource-group {group} --name {self.connected_cluster_name}').get_output_in_json()
+                while connected_cluster is not None and connected_cluster["connectivityStatus"] == "Connecting":
+                    time.sleep(5)
+                    connected_cluster = self.live_only_execute(self.cli_ctx, f'az connectedk8s show --resource-group {group} --name {self.connected_cluster_name}').get_output_in_json()
 
         except AttributeError:  # live only execute returns None if playing from record
             pass