Import key-values from AKS ConfigMap

Author: RichardChen820

src/azure-cli/azure/cli/command_modules/appconfig/_constants.py

@@ -144,6 +144,7 @@ class CompareFields:
 CompareFieldsMap = {
     "appconfig": (CompareFields.CONTENT_TYPE, CompareFields.VALUE, CompareFields.TAGS),
     "appservice": (CompareFields.VALUE, CompareFields.TAGS),
+    "aks": (CompareFields.VALUE, CompareFields.TAGS),
     "file": (CompareFields.CONTENT_TYPE, CompareFields.VALUE),
     "kvset": (CompareFields.CONTENT_TYPE, CompareFields.VALUE, CompareFields.TAGS),
     "restore": (CompareFields.VALUE, CompareFields.CONTENT_TYPE, CompareFields.LOCKED, CompareFields.TAGS)

src/azure-cli/azure/cli/command_modules/appconfig/_diff_utils.py

@@ -141,7 +141,7 @@ def get_serializer(level):
     '''
     Helper method that returns a serializer method called in formatting a string representation of a key-value.
     '''
-    source_modes = ("appconfig", "appservice", "file")
+    source_modes = ("appconfig", "appservice", "file", "aks")
     kvset_modes = ("kvset", "restore")
 
     if level not in source_modes and level not in kvset_modes:

src/azure-cli/azure/cli/command_modules/appconfig/_kv_import_helpers.py

@@ -549,7 +549,39 @@ def __read_kv_from_file(
     except OSError:
         raise FileOperationError("File is not available.")
 
+    flattened_data = flatten_config_data(
+        config_data=config_data,
+        format_=format_,
+        content_type=content_type,
+        prefix_to_add=prefix_to_add,
+        depth=depth,
+        separator=separator
+    )
+
+    # convert to KeyValue list
+    key_values = []
+    for k, v in flattened_data.items():
+        if validate_import_key(key=k):
+            key_values.append(KeyValue(key=k, value=v))
+    return key_values
+
+def flatten_config_data(config_data, format_, content_type, prefix_to_add="", depth=None, separator=None):
+    """
+    Flatten configuration data into a dictionary of key-value pairs.
+    
+    Args:
+        config_data: The configuration data to flatten (dict or list)
+        format_ (str): The format of the configuration data ('json', 'yaml', 'properties')
+        content_type (str): Content type for JSON validation
+        prefix_to_add (str): Prefix to add to each key
+        depth (int): Maximum depth for flattening hierarchical data
+        separator (str): Separator for hierarchical keys
+        
+    Returns:
+        dict: Flattened key-value pairs
+    """
     flattened_data = {}
+    
     if format_ == "json" and content_type and is_json_content_type(content_type):
         for key in config_data:
             __flatten_json_key_value(
@@ -580,14 +612,8 @@ def __read_kv_from_file(
                     depth=depth,
                     separator=separator,
                 )
-
-    # convert to KeyValue list
-    key_values = []
-    for k, v in flattened_data.items():
-        if validate_import_key(key=k):
-            key_values.append(KeyValue(key=k, value=v))
-    return key_values
-
+    
+    return flattened_data
 
 # App Service <-> List of KeyValue object
 
@@ -713,6 +739,155 @@ def __read_kv_from_app_service(
     except Exception as exception:
         raise CLIError("Failed to read key-values from appservice.\n" + str(exception))
 
+def __read_kv_from_kubernetes_configmap(
+    cmd,
+    aks_cluster,
+    configmap_name,
+    format_,
+    namespace="default",
+    prefix_to_add="",
+    content_type=None,
+    depth=None,
+    separator=None
+):
+    """
+    Read key-value pairs from a Kubernetes ConfigMap using aks_runcommand.
+    
+    Args:
+        cmd: The command context object
+        aks_cluster (str): Name of the AKS cluster
+        configmap_name (str): Name of the ConfigMap to read from
+        format_ (str): Format of the data in the ConfigMap (e.g., "json", "yaml")
+        namespace (str): Kubernetes namespace where the ConfigMap resides (default: "default")
+        prefix_to_add (str): Prefix to add to each key in the ConfigMap
+        content_type (str): Content type to apply to the key-values
+        depth (int): Maximum depth for flattening hierarchical data
+        separator (str): Separator for hierarchical keys
+    
+    Returns:
+        list: List of KeyValue objects
+    """
+    key_values = []
+    from azure.cli.command_modules.acs.custom import aks_runcommand
+    from azure.cli.command_modules.acs._client_factory import cf_managed_clusters
+    
+    # Get the AKS client from the factory
+    cmd.cli_ctx.data['subscription_id'] = aks_cluster["subscription"]
+    cmd.cli_ctx.data['safe_params'] = None
+    aks_client = cf_managed_clusters(cmd.cli_ctx)
+
+    # Command to get the ConfigMap and output it as JSON
+    command = f"kubectl get configmap {configmap_name} -n {namespace} -o json"
+    
+    # Execute the command on the cluster
+    result = aks_runcommand(cmd, aks_client, aks_cluster["resource_group"], aks_cluster["name"], command_string=command)
+
+    if hasattr(result, 'logs') and result.logs: 
+        if not hasattr(result, 'exit_code') or result.exit_code != 0:
+            raise CLIError(
+                f"Failed to get ConfigMap {configmap_name} in namespace {namespace}. {result.logs.strip()}"
+            )
+
+        try:
+            import json
+            configmap_data = json.loads(result.logs)
+            
+            # Extract the data section which contains the key-value pairs
+            kvs = __extract_kv_from_configmap_data(
+                configmap_data, content_type, prefix_to_add, format_, depth, separator)
+            
+            key_values.extend(kvs)
+        except json.JSONDecodeError:
+            raise ValidationError(
+                f"The result from ConfigMap {configmap_name} could not be parsed as valid JSON."
+            )
+    else:
+        raise CLIError(
+            f"Failed to get ConfigMap {configmap_name} in namespace {namespace}."
+        )
+
+    return key_values
+
+def __extract_kv_from_configmap_data(configmap_data, content_type, prefix_to_add="", format_=None, depth=None, separator=None):
+    """
+    Helper function to extract key-value pairs from ConfigMap data.
+    
+    Args:
+        configmap_data (dict): The ConfigMap data as a dictionary
+        prefix_to_add (str): Prefix to add to each key
+        content_type (str): Content type to apply to the key-values
+        format_ (str): Format of the data in the ConfigMap (e.g., "json", "yaml")
+        depth (int): Maximum depth for flattening hierarchical data
+        separator (str): Separator for hierarchical keys
+        
+    Returns:
+        list: List of KeyValue objects
+    """
+    key_values = []
+    import json
+    
+    if 'data' not in configmap_data:
+        logger.warning("ConfigMap exists but has no data")
+        return key_values
+        
+    for key, value in configmap_data['data'].items():
+        if format_ in ("json", "yaml", "properties"):
+            if format_ == "json":
+                try:
+                    value = json.loads(value)
+                except json.JSONDecodeError:
+                    logger.warning(
+                        f'Value "{value}" for key "{key}" is not a well formatted JSON data.'
+                    )
+                    continue
+            elif format_ == "yaml":
+                try:
+                    value = yaml.safe_load(value)
+                except yaml.YAMLError:
+                    logger.warning(
+                        f'Value "{value}" for key "{key}" is not a well formatted YAML data.'
+                    )
+                    continue
+            elif format_ == "properties":
+                try:
+                    import io
+                    value = javaproperties.load(io.StringIO(value))
+                except Exception:
+                    logger.warning(
+                        f'Value "{value}" for key "{key}" is not a well formatted properties data.'
+                    )
+                    continue       
+
+            flattened_data = flatten_config_data(
+                config_data=value,
+                format_=format_,
+                content_type=content_type,
+                prefix_to_add=prefix_to_add,
+                depth=depth,
+                separator=separator
+            )
+
+            key_values = []
+            for k, v in flattened_data.items():
+                if validate_import_key(key=k):
+                    key_values.append(KeyValue(key=k, value=v))
+            return key_values
+        
+        elif validate_import_key(key):
+            # If content_type is JSON, validate the value
+            if content_type and is_json_content_type(content_type):
+                try:
+                    json.loads(value)
+                except json.JSONDecodeError:
+                    logger.warning(
+                        f'Value "{value}" for key "{key}" is not a valid JSON object, which conflicts with the provided content type "{content_type}".'
+                    )
+                    continue
+            
+            kv = KeyValue(key=prefix_to_add + key, value=value)
+            key_values.append(kv)
+            
+    return key_values
 
 def __validate_import_keyvault_ref(kv):
     if kv and validate_import_key(kv.key):

src/azure-cli/azure/cli/command_modules/appconfig/_params.py

@@ -16,7 +16,8 @@
     get_default_location_from_resource_group
 from ._constants import ImportExportProfiles, ImportMode, FeatureFlagConstants, ARMAuthenticationMode
 
-from ._validators import (validate_appservice_name_or_id, validate_sku, validate_snapshot_query_fields,
+from ._validators import (validate_appservice_name_or_id, validate_aks_cluster_name_or_id, 
+                          validate_sku, validate_snapshot_query_fields,
                           validate_connection_string, validate_datetime,
                           validate_export, validate_import,
                           validate_import_depth, validate_query_fields,
@@ -217,7 +218,7 @@ def load_arguments(self, _):
     with self.argument_context('appconfig kv import') as c:
         c.argument('label', help="Imported KVs and feature flags will be assigned with this label. If no label specified, will assign null label.")
         c.argument('prefix', help="This prefix will be appended to the front of imported keys. Prefix will be ignored for feature flags.")
-        c.argument('source', options_list=['--source', '-s'], arg_type=get_enum_type(['file', 'appconfig', 'appservice']), validator=validate_import, help="The source of importing. Note that importing feature flags from appservice is not supported.")
+        c.argument('source', options_list=['--source', '-s'], arg_type=get_enum_type(['file', 'appconfig', 'appservice', 'aks']), validator=validate_import, help="The source of importing. Note that importing feature flags from appservice is not supported.")
         c.argument('yes', help="Do not prompt for preview.")
         c.argument('skip_features', help="Import only key values and exclude all feature flags. By default, all feature flags will be imported from file or appconfig. Not applicable for appservice.", arg_type=get_three_state_flag())
         c.argument('content_type', help='Content type of all imported items.')
@@ -247,6 +248,11 @@ def load_arguments(self, _):
     with self.argument_context('appconfig kv import', arg_group='AppService') as c:
         c.argument('appservice_account', validator=validate_appservice_name_or_id, help='ARM ID for AppService OR the name of the AppService, assuming it is in the same subscription and resource group as the App Configuration store. Required for AppService arguments')
 
+    with self.argument_context('appconfig kv import', arg_group='AKS') as c:
+        c.argument('aks_cluster', validator=validate_aks_cluster_name_or_id, help='ARM ID for AKS OR the name of the AKS, assuming it is in the same subscription and resource group as the App Configuration store. Required for AKS arguments'),
+        c.argument('configmap_name', help='Name of the ConfigMap. Required for AKS arguments.')
+        c.argument('configmap_namespace', help='Namespace of the ConfigMap. default to "default" namespace if not specified.') 
+
     with self.argument_context('appconfig kv export') as c:
         c.argument('label', help="Only keys and feature flags with this label will be exported. If no label specified, export keys and feature flags with null label by default. When export destination is appconfig, or when export destination is file with `appconfig/kvset` profile, this argument supports asterisk and comma signs for label filtering, for instance, * means all labels, abc* means labels with abc as prefix, and abc,xyz means labels with abc or xyz.")
         c.argument('prefix', help="Prefix to be trimmed from keys. Prefix will be ignored for feature flags.")

src/azure-cli/azure/cli/command_modules/appconfig/_validators.py

@@ -105,6 +105,11 @@ def validate_import(namespace):
     elif source == 'appservice':
         if namespace.appservice_account is None:
             raise RequiredArgumentMissingError("Please provide '--appservice-account' argument")
+    elif source == 'aks':
+        if namespace.aks_cluster is None:
+            raise RequiredArgumentMissingError("Please provide '--aks-cluster' argument")
+        if namespace.configmap_name is None:
+            raise RequiredArgumentMissingError("Please provide '--configmap-name' argument")
 
 
 def validate_export(namespace):
@@ -144,6 +149,28 @@ def validate_appservice_name_or_id(cmd, namespace):
         else:
             namespace.appservice_account = parse_resource_id(namespace.appservice_account)
 
+def validate_aks_cluster_name_or_id(cmd, namespace):
+    from azure.cli.core.commands.client_factory import get_subscription_id
+    from azure.mgmt.core.tools import is_valid_resource_id, parse_resource_id
+    if namespace.aks_cluster:
+        if not is_valid_resource_id(namespace.aks_cluster):
+            config_store_name = ""
+            if namespace.name:
+                config_store_name = namespace.name
+            elif namespace.connection_string:
+                config_store_name = get_store_name_from_connection_string(namespace.connection_string)
+            else:
+                raise CLIError("Please provide App Configuration name or connection string for fetching the AKS cluster details. Alternatively, you can provide a valid ARM ID for the AKS cluster.")
+
+            resource_group, _ = resolve_store_metadata(cmd, config_store_name)
+            namespace.aks_cluster = {
+                "subscription": get_subscription_id(cmd.cli_ctx),
+                "resource_group": resource_group,
+                "name": namespace.aks_cluster
+            }
+        else:
+            namespace.aks_cluster = parse_resource_id(namespace.aks_cluster)
+
 
 def validate_query_fields(namespace):
     if namespace.fields:

src/azure-cli/azure/cli/command_modules/appconfig/keyvalue.py

@@ -30,6 +30,7 @@
     __delete_configuration_setting_from_config_store,
     __read_features_from_file,
     __read_kv_from_app_service,
+    __read_kv_from_kubernetes_configmap,
     __read_kv_from_file,
 )
 from knack.log import get_logger
@@ -89,7 +90,11 @@ def import_config(cmd,
                   src_auth_mode="key",
                   src_endpoint=None,
                   # from-appservice parameters
-                  appservice_account=None):
+                  appservice_account=None,
+                  # from-aks parameters
+                  aks_cluster=None,
+                  configmap_namespace="default",
+                  configmap_name=None):
 
     src_features = []
     dest_features = []
@@ -170,6 +175,25 @@ def import_config(cmd,
     elif source == 'appservice':
         src_kvs = __read_kv_from_app_service(
             cmd, appservice_account=appservice_account, prefix_to_add=prefix, content_type=content_type)
+    
+    elif source == 'aks':
+        if separator:
+            # If separator is provided, use max depth by default unless depth is specified.
+            depth = sys.maxsize if depth is None else int(depth)
+        else:
+            if depth and int(depth) != 1:
+                logger.warning("Cannot flatten hierarchical data without a separator. --depth argument will be ignored.")
+            depth = 1
+            
+        src_kvs = __read_kv_from_kubernetes_configmap(cmd,
+                                                      aks_cluster=aks_cluster,
+                                                      configmap_name=configmap_name,
+                                                      namespace=configmap_namespace,
+                                                      prefix_to_add=prefix,
+                                                      content_type=content_type,
+                                                      format_=format_,
+                                                      separator=separator,
+                                                      depth=depth)
 
     if strict or not yes or import_mode == ImportMode.IGNORE_MATCH:
         # fetch key values from user's configstore