diff --git a/src/connectedk8s/HISTORY.rst b/src/connectedk8s/HISTORY.rst index d44cb3899fb..df6bf9db2c6 100644 --- a/src/connectedk8s/HISTORY.rst +++ b/src/connectedk8s/HISTORY.rst @@ -3,6 +3,11 @@ Release History =============== +1.10.8 +++++++ +* Add checks for the Graytown bundle feature flag in the 'connectedk8s connect' and 'connectedk8s update' commands. +* Display a more detailed error message from the agent-update-validator in the 'connectedk8s upgrade' command. + 1.10.7 ++++++ * Added support for discovering additional k8s distributions and Infrastructure. diff --git a/src/connectedk8s/azext_connectedk8s/_constants.py b/src/connectedk8s/azext_connectedk8s/_constants.py index 4f173c668a2..cfe81b795aa 100644 --- a/src/connectedk8s/azext_connectedk8s/_constants.py +++ b/src/connectedk8s/azext_connectedk8s/_constants.py @@ -517,3 +517,15 @@ # "Application code shouldn't block the creation of resources for a resource provider that is in the registering state." # See https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider allowed_rp_registration_states = ["Registering", "Registered"] + +Connected_Cluster_Type = "connectedclusters" + +Arc_Agentry_Bundle_Feature = "extensionSets" +Arc_Agentry_Bundle_Feature_Setting = "versionManagedExtensions" + +Bundle_Feature_Value_List = ["enabled", "disabled", "preview"] +Bundle_Extension_Type_List = ["microsoft.iotoperations", "microsoft.extensiondiagnostics", "microsoft.arc.containerstorage", "microsoft.azure.secretstore"] + +CONST_K8S_EXTENSION_NAME = "k8s-extension" +CONST_K8S_EXTENSION_CLIENT_FACTORY_MOD_NAME = "azext_k8s_extension._client_factory" +CONST_K8S_EXTENSION_CUSTOM_MOD_NAME = "azext_k8s_extension.custom" diff --git a/src/connectedk8s/azext_connectedk8s/clientproxyhelper/_proxylogic.py b/src/connectedk8s/azext_connectedk8s/clientproxyhelper/_proxylogic.py index 71345064af6..f72074d1b6e 100644 --- a/src/connectedk8s/azext_connectedk8s/clientproxyhelper/_proxylogic.py +++ b/src/connectedk8s/azext_connectedk8s/clientproxyhelper/_proxylogic.py @@ -18,7 +18,7 @@ if TYPE_CHECKING: from subprocess import Popen - from knack.commands import CLICommmand + from knack.commands import CLICommand from requests.models import Response from azext_connectedk8s.vendored_sdks.preview_2024_07_01.models import ( @@ -30,7 +30,7 @@ def handle_post_at_to_csp( - cmd: CLICommmand, + cmd: CLICommand, api_server_port: int, tenant_id: str, clientproxy_process: Popen[bytes], diff --git a/src/connectedk8s/azext_connectedk8s/custom.py b/src/connectedk8s/azext_connectedk8s/custom.py index 8f481f1111b..fa2d34f4842 100644 --- a/src/connectedk8s/azext_connectedk8s/custom.py +++ b/src/connectedk8s/azext_connectedk8s/custom.py @@ -81,7 +81,7 @@ from azure.cli.core.commands import AzCliCommand from azure.core.polling import LROPoller from Crypto.PublicKey.RSA import RsaKey - from knack.commands import CLICommmand + from knack.commands import CLICommand from kubernetes.client import V1NodeList from kubernetes.config.kube_config import ConfigNode from requests.models import Response @@ -99,7 +99,7 @@ def create_connectedk8s( - cmd: CLICommmand, + cmd: CLICommand, client: ConnectedClusterOperations, resource_group_name: str, cluster_name: str, @@ -235,6 +235,10 @@ def create_connectedk8s( configuration_settings, configuration_protected_settings, ) + + # Validate and update bundle feature flag value if provided + validate_connect_cluster_bundle_feature_flag_value(configuration_settings, yes) + arc_agentry_configurations = generate_arc_agent_configuration( configuration_settings, configuration_protected_settings ) @@ -1043,7 +1047,7 @@ def validate_existing_provisioned_cluster_for_reput( raise InvalidArgumentValueError(err_msg) -def send_cloud_telemetry(cmd: CLICommmand) -> str: +def send_cloud_telemetry(cmd: CLICommand) -> str: telemetry.add_extension_event( "connectedk8s", {"Context.Default.AzureCLI.AzureCloud": cmd.cli_ctx.cloud.name} ) @@ -1289,7 +1293,7 @@ def connected_cluster_exists( return True -def get_default_config_dp_endpoint(cmd: CLICommmand, location: str) -> str: +def get_default_config_dp_endpoint(cmd: CLICommand, location: str) -> str: cloud_based_domain = cmd.cli_ctx.cloud.endpoints.active_directory.split(".")[2] config_dp_endpoint = ( f"https://{location}.dp.kubernetesconfiguration.azure.{cloud_based_domain}" @@ -1298,7 +1302,7 @@ def get_default_config_dp_endpoint(cmd: CLICommmand, location: str) -> str: def get_config_dp_endpoint( - cmd: CLICommmand, + cmd: CLICommand, location: str, values_file: str | None, arm_metadata: dict[str, Any] | None = None, @@ -1512,6 +1516,169 @@ def set_security_profile(enable_workload_identity: bool) -> SecurityProfile: return security_profile +def get_k8s_extension_module(module_name): + try: + # adding the installed extension in the path + from azure.cli.core.extension.operations import add_extension_to_path + add_extension_to_path(consts.CONST_K8S_EXTENSION_NAME) + # import the extension module + from importlib import import_module + azext_custom = import_module(module_name) + return azext_custom + except ImportError: + raise ImportError( # pylint: disable=raise-missing-from + "Please add CLI extension `k8s-extension` to disable bundle feature flag.\n" + "Run command `az extension add --name k8s-extension`" + ) + + +def get_bundle_feature_flag_from_arc_agentry_config( + current_arc_agentry_config: list[ArcAgentryConfigurations] +) -> str | None: + for agentry_config in current_arc_agentry_config: + if agentry_config.feature == consts.Arc_Agentry_Bundle_Feature and \ + consts.Arc_Agentry_Bundle_Feature_Setting in agentry_config.settings: + return agentry_config.settings[consts.Arc_Agentry_Bundle_Feature_Setting].lower() + return None + + +def get_bundle_feature_flag_from_configuration_settings( + configuration_settings: dict[str, Any] +) -> str | None: + settings = configuration_settings.get(consts.Arc_Agentry_Bundle_Feature, {}) + value = settings.get(consts.Arc_Agentry_Bundle_Feature_Setting) + return value if value is None else value.lower() + + +def validate_bundle_feature_flag_value( + configuration_settings: dict[str, Any] +) -> str | None: + print(f"Step: {utils.get_utctimestring()}: Validating the bundle feature flag value") + value = get_bundle_feature_flag_from_configuration_settings(configuration_settings) + + if value is not None: + # Remove leading and trailing whitespace and quotes + value = value.strip().strip("'\"") + + if value and value not in consts.Bundle_Feature_Value_List: + raise InvalidArgumentValueError( + f"Not supported value for the feature flag '{consts.Arc_Agentry_Bundle_Feature_Setting}': " + f"'{value}'. Please specify a value from the list: {consts.Bundle_Feature_Value_List}." + ) + + configuration_settings[consts.Arc_Agentry_Bundle_Feature][consts.Arc_Agentry_Bundle_Feature_Setting] = value + print(f"Step: {utils.get_utctimestring()}: Setting the bundle feature flag value to '{value}'") + + return value + + +def validate_connect_cluster_bundle_feature_flag_value( + configuration_settings: dict[str, Any], + yes: bool = False, +): + bundle_feature_flag_value = validate_bundle_feature_flag_value(configuration_settings) + + # If the bundle feature flag value is None, skip the validation + if bundle_feature_flag_value is None: + return + + if bundle_feature_flag_value == "preview": + confirmation_message = ( + f"You are about to enter the 'preview' mode for {consts.Arc_Agentry_Bundle_Feature_Setting}. " + "In this mode, all SLA support will be discontinued, and the cluster will remain in 'preview' mode " + "until it is disconnected from Arc. Are you sure you want to proceed? " + ) + + utils.user_confirmation(confirmation_message, yes) + + logger.warning( + "Entered %s 'preview' mode. All SLA support is discontinued, and the cluster will remain in 'preview' mode " + "until it is disconnected from Arc.", + consts.Arc_Agentry_Bundle_Feature_Setting + ) + + elif bundle_feature_flag_value == "disabled": + err_msg = ( + f"{consts.Arc_Agentry_Bundle_Feature_Setting} 'disabled' mode can only be set using 'az connectedk8s update'. " + f"To keep the bundle feature flag off during cluster connection, remove " + f"{consts.Arc_Agentry_Bundle_Feature_Setting} from the --config." + ) + raise ArgumentUsageError(err_msg) + + +def validate_update_cluster_bundle_feature_flag_value( + cmd: CLICommand, + current_arc_agentry_config: list[ArcAgentryConfigurations], + configuration_settings: dict[str, Any], + resource_group_name: str, + cluster_name: str, +): + bundle_feature_flag_value = validate_bundle_feature_flag_value(configuration_settings) + + # If the bundle feature flag value is None, skip the validation + if bundle_feature_flag_value is None: + return + + current_bundle_feature_flag_value = get_bundle_feature_flag_from_arc_agentry_config(current_arc_agentry_config) + + if bundle_feature_flag_value == "preview": + err_msg = ( + f"{consts.Arc_Agentry_Bundle_Feature_Setting} 'preview' mode can only be enabled when a cluster " + "is first connected to Arc with 'az connectedk8s connect'. Updating the preview mode config with " + "'az connectedk8s update' is not allowed." + ) + raise ArgumentUsageError(err_msg) + + if current_bundle_feature_flag_value == "preview": + err_msg = ( + f"The cluster is in {consts.Arc_Agentry_Bundle_Feature_Setting} 'preview' mode, " + "updating the value is not allowed." + ) + raise ArgumentUsageError(err_msg) + + invalid_transition = ( + (current_bundle_feature_flag_value == "enabled" and bundle_feature_flag_value == "") or + (current_bundle_feature_flag_value == "" and bundle_feature_flag_value == "disabled") + ) + + if invalid_transition: + err_msg = ( + f"Could not set {consts.Arc_Agentry_Bundle_Feature}.{consts.Arc_Agentry_Bundle_Feature_Setting} from " + f"'{current_bundle_feature_flag_value}' to '{bundle_feature_flag_value}'." + ) + + raise ArgumentUsageError(err_msg) + + # If the bundle feature flag is set to 'disabled', check if any bundle extensions are installed + if current_bundle_feature_flag_value == "enabled" and bundle_feature_flag_value == "disabled": + client_factory = get_k8s_extension_module(consts.CONST_K8S_EXTENSION_CLIENT_FACTORY_MOD_NAME) + client = client_factory.cf_k8s_extension_operation(cmd.cli_ctx) + k8s_extension_custom_mod = get_k8s_extension_module(consts.CONST_K8S_EXTENSION_CUSTOM_MOD_NAME) + extensions = k8s_extension_custom_mod.list_k8s_extension( + client, + resource_group_name, + cluster_name, + "connectedClusters", + ) + + installed_bundle_extensions = [ + ext.extension_type.lower() + for ext in extensions + if ext.extension_type.lower() in consts.Bundle_Extension_Type_List + ] + + if installed_bundle_extensions: + err_msg = ( + f"Could not set {consts.Arc_Agentry_Bundle_Feature}.{consts.Arc_Agentry_Bundle_Feature_Setting} to " + f"'disabled' - detected the following extension types on the cluster: {installed_bundle_extensions}.\n" + f"Please remove them with 'az k8s-extension delete --cluster-name " + f"--cluster-type --resource-group --name ' " + f"and try turning off the feature again." + ) + + raise ArgumentUsageError(err_msg) + + def generate_arc_agent_configuration( configuration_settings: dict[str, Any], redacted_protected_settings: dict[str, Any], @@ -1733,7 +1900,7 @@ def list_connectedk8s( def delete_connectedk8s( - cmd: CLICommmand, + cmd: CLICommand, client: ConnectedClusterOperations, resource_group_name: str, cluster_name: str, @@ -1995,7 +2162,7 @@ def update_connected_cluster_internal( def update_connected_cluster( - cmd: CLICommmand, + cmd: CLICommand, client: ConnectedClusterOperations, resource_group_name: str, cluster_name: str, @@ -2072,9 +2239,6 @@ def update_connected_cluster( configuration_settings, configuration_protected_settings, ) - arc_agentry_configurations = generate_arc_agent_configuration( - configuration_settings, redacted_protected_values - ) # Fetch Connected Cluster for agent version connected_cluster = client.get(resource_group_name, cluster_name) @@ -2089,6 +2253,15 @@ def update_connected_cluster( ) raise InvalidArgumentValueError(err_msg) + # Validate and update bundle feature flag value if provided + validate_update_cluster_bundle_feature_flag_value( + cmd, connected_cluster.arc_agentry_configurations, configuration_settings, resource_group_name, cluster_name + ) + + arc_agentry_configurations = generate_arc_agent_configuration( + configuration_settings, redacted_protected_values + ) + # Patching the connected cluster ARM resource arm_properties_unset = ( tags is None @@ -2351,7 +2524,7 @@ def update_connected_cluster( def upgrade_agents( - cmd: CLICommmand, + cmd: CLICommand, client: ConnectedClusterOperations, resource_group_name: str, cluster_name: str, @@ -2654,6 +2827,22 @@ def upgrade_agents( for message in consts.Helm_Install_Release_Userfault_Messages ): telemetry.set_user_fault() + + namespace = "azure-arc" + label_selector = "job-name=agent-update-validator" + + # Get the list of pods matching the label + pods = api_instance.list_namespaced_pod(namespace=namespace, label_selector=label_selector) + + # Extract the terminated message from the container + if pods.items: + pod = pods.items[0] + container_statuses = pod.status.container_statuses + if container_statuses: + state = container_statuses[0].state + if state.terminated: + helm_upgrade_error_message = state.terminated.message + telemetry.set_exception( exception=error_helm_upgrade.decode("ascii"), fault_type=consts.Install_HelmRelease_Fault_Type, @@ -2797,7 +2986,7 @@ def get_all_helm_values( def enable_features( - cmd: CLICommmand, + cmd: CLICommand, client: ConnectedClusterOperations, resource_group_name: str, cluster_name: str, @@ -3030,7 +3219,7 @@ def enable_features( def disable_features( - cmd: CLICommmand, + cmd: CLICommand, client: ConnectedClusterOperations, resource_group_name: str, cluster_name: str, @@ -3169,7 +3358,7 @@ def disable_features( def get_chart_and_disable_features( - cmd: CLICommmand, + cmd: CLICommand, connected_cluster: ConnectedCluster, kube_config: str | None, kube_context: str | None, @@ -3260,7 +3449,7 @@ def get_chart_and_disable_features( def disable_cluster_connect( - cmd: CLICommmand, + cmd: CLICommand, client: ConnectedClusterOperations, resource_group_name: str, cluster_name: str, @@ -3467,7 +3656,7 @@ def handle_merge( def client_side_proxy_wrapper( - cmd: CLICommmand, + cmd: CLICommand, client: ConnectedClusterOperations, resource_group_name: str, cluster_name: str, @@ -3638,7 +3827,7 @@ def client_side_proxy_wrapper( def client_side_proxy_main( - cmd: CLICommmand, + cmd: CLICommand, tenant_id: str, client: ConnectedClusterOperations, resource_group_name: str, @@ -3709,7 +3898,7 @@ def client_side_proxy_main( def client_side_proxy( - cmd: CLICommmand, + cmd: CLICommand, tenant_id: str, client: ConnectedClusterOperations, resource_group_name: str, @@ -3842,7 +4031,7 @@ def client_side_proxy( def check_cl_registration_and_get_oid( - cmd: CLICommmand, cl_oid: str | None, subscription_id: str | None + cmd: CLICommand, cl_oid: str | None, subscription_id: str | None ) -> tuple[bool, str]: print( f"Step: {utils.get_utctimestring()}: Checking Custom Location(Microsoft.ExtendedLocation) RP Registration state for this Subscription, and attempt to get the Custom Location Object ID (OID),if registered" @@ -3881,7 +4070,7 @@ def check_cl_registration_and_get_oid( return enable_custom_locations, custom_locations_oid -def get_custom_locations_oid(cmd: CLICommmand, cl_oid: str | None) -> str: +def get_custom_locations_oid(cmd: CLICommand, cl_oid: str | None) -> str: try: graph_client = graph_client_factory(cmd.cli_ctx) app_id = "bc313c14-388c-4e7d-a58e-70017303ee3b" @@ -3942,7 +4131,7 @@ def get_custom_locations_oid(cmd: CLICommmand, cl_oid: str | None) -> str: def troubleshoot( - cmd: CLICommmand, + cmd: CLICommand, client: ConnectedClusterOperations, resource_group_name: str, cluster_name: str, diff --git a/src/connectedk8s/azext_connectedk8s/tests/latest/test_connectedk8s_scenario.py b/src/connectedk8s/azext_connectedk8s/tests/latest/test_connectedk8s_scenario.py index 9d899e786da..9cf949bd827 100644 --- a/src/connectedk8s/azext_connectedk8s/tests/latest/test_connectedk8s_scenario.py +++ b/src/connectedk8s/azext_connectedk8s/tests/latest/test_connectedk8s_scenario.py @@ -12,13 +12,18 @@ import stat import subprocess import time -from subprocess import PIPE +from subprocess import PIPE, Popen import oras.client # type: ignore[import-untyped] import psutil import requests from azure.cli.core import get_default_cli -from azure.cli.core.azclierror import RequiredArgumentMissingError, ValidationError +from azure.cli.core.azclierror import ( + ArgumentUsageError, + RequiredArgumentMissingError, + ValidationError, + InvalidArgumentValueError +) from azure.cli.testsdk import ( # pylint: disable=import-error LiveScenarioTest, ResourceGroupPreparer, @@ -171,6 +176,25 @@ def install_kubectl_client(): logger.warning("Unable to install kubectl. Error: " + str(e)) return +def get_bundle_feature_flag_from_config_map( + kubectl_client_location: str, + kube_config: str | None, + kube_context: str | None +) -> str | None: + cmd = [kubectl_client_location, "get", "configmap", "azure-clusterconfig", "-n", "azure-arc", "-o", "jsonpath={.data.EXTENSION_BUNDLE_ENABLED_FEATURE_FLAG}"] + if kube_config: + cmd.extend(["--kubeconfig", kube_config]) + if kube_context: + cmd.extend(["--context", kube_context]) + + cmd_output = Popen(cmd, stdout=PIPE, stderr=PIPE) + + bundle_feature_flag, stderr = cmd_output.communicate() + if cmd_output.returncode == 0: + return bundle_feature_flag.decode() + else: + logger.warning("Failed to get bundle feature flag from config map: " + str(stderr.decode())) + return None class Connectedk8sScenarioTest(LiveScenarioTest): @live_only() @@ -266,6 +290,235 @@ def test_connect_withoidcandselfhostedissuer(self, resource_group): # delete the kube config os.remove(_get_test_data_file(managed_cluster_name + "-config.yaml")) + @live_only() + @ResourceGroupPreparer( + name_prefix="conk8stest", location=CONFIG["location"], random_name_length=16 + ) + def test_connect_withbundlefeatureflag(self, resource_group): + managed_cluster_name = self.create_random_name(prefix="test-connect", length=24) + kubeconfig = _get_test_data_file(managed_cluster_name + "-config.yaml") + self.kwargs.update( + { + "rg": resource_group, + "name": self.create_random_name(prefix="cc-", length=12), + "kubeconfig": kubeconfig, + "managed_cluster_name": managed_cluster_name, + "location": CONFIG["location"], + } + ) + + self.cmd("aks create -g {rg} -n {managed_cluster_name} --generate-ssh-keys") + self.cmd( + "aks get-credentials -g {rg} -n {managed_cluster_name} -f {kubeconfig} --admin" + ) + + with self.assertRaisesRegex(InvalidArgumentValueError, "Not supported value for the feature flag"): + self.cmd( + "connectedk8s connect -g {rg} -n {name} --location {location} --disable-auto-upgrade --kube-config {kubeconfig} \ + --kube-context {managed_cluster_name}-admin --config extensionSets.versionManagedExtensions='off'") + + with self.assertRaisesRegex(ArgumentUsageError, "versionManagedExtensions 'disabled' mode can only be set using 'az connectedk8s update'."): + self.cmd( + "connectedk8s connect -g {rg} -n {name} --location {location} --disable-auto-upgrade --kube-config {kubeconfig} \ + --kube-context {managed_cluster_name}-admin --config extensionSets.versionManagedExtensions='disabled'" + ) + + with self.assertLogs(level='WARNING') as cm: + self.cmd( + "connectedk8s connect -g {rg} -n {name} -l {location} --disable-auto-upgrade --kube-config {kubeconfig} \ + --kube-context {managed_cluster_name}-admin --config extensionSets.versionManagedExtensions='preview' --yes", + checks=[ + self.check("resourceGroup", "{rg}"), + self.check("name", "{name}"), + self.check("arcAgentryConfigurations[0].settings.versionManagedExtensions", "preview"), + ], + ) + + self.assertIn( + "All SLA support is discontinued, and the cluster will remain in 'preview' mode until it is disconnected from Arc.", + ''.join(cm.output) + ) + + kubectl_client_location = install_kubectl_client() + configmap_bundle_feature_flag = get_bundle_feature_flag_from_config_map(kubectl_client_location, kubeconfig, f"{managed_cluster_name}-admin") + self.assertEqual(configmap_bundle_feature_flag, "preview") + + with self.assertRaisesRegex(ArgumentUsageError, "The cluster is in versionManagedExtensions 'preview' mode, updating the value is not allowed."): + self.cmd( + "connectedk8s update -g {rg} -n {name} --auto-upgrade false --kube-config {kubeconfig} \ + --kube-context {managed_cluster_name}-admin --config extensionSets.versionManagedExtensions='enabled'") + + self.cmd( + "connectedk8s delete -g {rg} -n {name} --kube-config {kubeconfig} --kube-context " \ + "{managed_cluster_name}-admin --yes" + ) + + self.cmd( + "connectedk8s connect -g {rg} -n {name} -l {location} --disable-auto-upgrade --kube-config {kubeconfig} \ + --kube-context {managed_cluster_name}-admin --config extensionSets.versionManagedExtensions='enabled'", + checks=[ + self.check("resourceGroup", "{rg}"), + self.check("name", "{name}"), + self.check("arcAgentryConfigurations[0].settings.versionManagedExtensions", "enabled"), + ], + ) + + configmap_bundle_feature_flag = get_bundle_feature_flag_from_config_map(kubectl_client_location, kubeconfig, f"{managed_cluster_name}-admin") + self.assertEqual(configmap_bundle_feature_flag, "enabled") + + @live_only() + @ResourceGroupPreparer( + name_prefix="conk8stest", location=CONFIG["location"], random_name_length=16 + ) + def test_update_withbundlefeatureflag(self, resource_group): + managed_cluster_name = self.create_random_name(prefix="test-update", length=24) + kubeconfig = _get_test_data_file(managed_cluster_name + "-config.yaml") + self.kwargs.update( + { + "rg": resource_group, + "name": self.create_random_name(prefix="cc-", length=12), + "kubeconfig": kubeconfig, + "managed_cluster_name": managed_cluster_name, + "location": CONFIG["location"], + } + ) + + self.cmd("aks create -g {rg} -n {managed_cluster_name} --generate-ssh-keys") + self.cmd( + "aks get-credentials -g {rg} -n {managed_cluster_name} -f {kubeconfig} --admin" + ) + + self.cmd( + "connectedk8s connect -g {rg} -n {name} -l {location} --disable-auto-upgrade --kube-config {kubeconfig} \ + --kube-context {managed_cluster_name}-admin --config extensionSets.versionManagedExtensions='enabled'", + checks=[ + self.check("resourceGroup", "{rg}"), + self.check("name", "{name}"), + self.check("arcAgentryConfigurations[0].settings.versionManagedExtensions", "enabled"), + ], + ) + + kubectl_client_location = install_kubectl_client() + configmap_bundle_feature_flag = get_bundle_feature_flag_from_config_map(kubectl_client_location, kubeconfig, f"{managed_cluster_name}-admin") + self.assertEqual(configmap_bundle_feature_flag, "enabled") + + with self.assertRaisesRegex(InvalidArgumentValueError, "Not supported value for the feature flag"): + self.cmd( + "connectedk8s update -g {rg} -n {name} --auto-upgrade false --kube-config {kubeconfig} \ + --kube-context {managed_cluster_name}-admin --config extensionSets.versionManagedExtensions='on'") + + with self.assertRaisesRegex(ArgumentUsageError, "Updating the preview mode config with 'az connectedk8s update' is not allowed."): + self.cmd( + "connectedk8s update -g {rg} -n {name} --auto-upgrade false --kube-config {kubeconfig} \ + --kube-context {managed_cluster_name}-admin --config extensionSets.versionManagedExtensions='preview'") + + with self.assertRaisesRegex(ArgumentUsageError, "Could not set extensionSets.versionManagedExtensions from 'enabled' to ''"): + self.cmd( + "connectedk8s update -g {rg} -n {name} --auto-upgrade false --kube-config {kubeconfig} \ + --kube-context {managed_cluster_name}-admin --config extensionSets.versionManagedExtensions=''") + + # Without any bundle extensions installed on the cluster, the bundle feature flag can be set to 'disabled' + # All leading and trailing single and double quotes and whitespaces should be automatically stripped + self.cmd( + "connectedk8s update -g {rg} -n {name} --auto-upgrade false --kube-config {kubeconfig} \ + --kube-context {managed_cluster_name}-admin --config extensionSets.versionManagedExtensions=\" disabled\"", + checks=[ + self.check("arcAgentryConfigurations[0].settings.versionManagedExtensions", "disabled"), + ], + ) + + configmap_bundle_feature_flag = get_bundle_feature_flag_from_config_map(kubectl_client_location, kubeconfig, f"{managed_cluster_name}-admin") + self.assertEqual(configmap_bundle_feature_flag, "disabled") + + self.cmd( + "connectedk8s update -g {rg} -n {name} --auto-upgrade false --kube-config {kubeconfig} \ + --kube-context {managed_cluster_name}-admin --config extensionSets.versionManagedExtensions='enabled'", + checks=[ + self.check("arcAgentryConfigurations[0].settings.versionManagedExtensions", "enabled"), + ], + ) + + self.cmd("k8s-extension create --cluster-name {name} --resource-group {rg} --cluster-type connectedClusters \ + --extension-type microsoft.iotoperations.platform --name azure-iot-operations-platform \ + --release-train preview --auto-upgrade-minor-version False --config installTrustManager=true \ + --config installCertManager=true --version 0.7.6 --release-namespace cert-manager --scope cluster") + + self.cmd("k8s-extension create --cluster-name {name} --resource-group {rg} --cluster-type connectedClusters \ + --extension-type microsoft.azure.secretstore --name azure-secret-store --auto-upgrade-minor-version False \ + --config rotationPollIntervalInSeconds=120 --config validatingAdmissionPolicies.applyPolicies=false \ + --scope cluster") + + # With bundle extensions installed on the cluster, the bundle feature flag cannot be set to 'disabled' + with self.assertRaisesRegex(ArgumentUsageError, "detected the following extension types on the cluster"): + self.cmd( + "connectedk8s update -g {rg} -n {name} --auto-upgrade false --kube-config {kubeconfig} \ + --kube-context {managed_cluster_name}-admin --config extensionSets.versionManagedExtensions='disabled'") + + configmap_bundle_feature_flag = get_bundle_feature_flag_from_config_map(kubectl_client_location, kubeconfig, f"{managed_cluster_name}-admin") + self.assertEqual(configmap_bundle_feature_flag, "enabled") + + @live_only() + @ResourceGroupPreparer( + name_prefix="conk8stest", location=CONFIG["location"], random_name_length=16 + ) + def test_upgrade_with_agentupdatevalidator(self, resource_group): + managed_cluster_name = self.create_random_name(prefix="test-upgrade", length=24) + kubeconfig = _get_test_data_file(managed_cluster_name + "-config.yaml") + self.kwargs.update( + { + "rg": resource_group, + "name": self.create_random_name(prefix="cc-", length=12), + "kubeconfig": kubeconfig, + "managed_cluster_name": managed_cluster_name, + "location": CONFIG["location"], + } + ) + self.cmd("aks create -g {rg} -n {managed_cluster_name} --generate-ssh-keys") + self.cmd( + "aks get-credentials -g {rg} -n {managed_cluster_name} -f {kubeconfig} --admin" + ) + + self.cmd( + "connectedk8s connect -g {rg} -n {name} -l {location} --disable-auto-upgrade --kube-config {kubeconfig} \ + --kube-context {managed_cluster_name}-admin --config extensionSets.versionManagedExtensions='enabled'", + checks=[ + self.check("resourceGroup", "{rg}"), + self.check("name", "{name}"), + self.check("arcAgentryConfigurations[0].settings.versionManagedExtensions", "enabled"), + ], + ) + + self.cmd( + "connectedk8s upgrade -g {rg} -n {name} --agent-version 1.24.4 --kube-config {kubeconfig} \ + --kube-context {managed_cluster_name}-admin", + ) + + self.cmd("k8s-extension create --cluster-name {name} --resource-group {rg} --cluster-type connectedClusters \ + --extension-type microsoft.iotoperations.platform --name azure-iot-operations-platform \ + --release-train preview --auto-upgrade-minor-version False --config installTrustManager=true \ + --config installCertManager=true --version 0.7.6 --release-namespace cert-manager --scope cluster") + + self.cmd("k8s-extension create --cluster-name {name} --resource-group {rg} --cluster-type connectedClusters \ + --extension-type microsoft.azure.secretstore --name azure-secret-store --auto-upgrade-minor-version False \ + --config rotationPollIntervalInSeconds=120 --config validatingAdmissionPolicies.applyPolicies=false \ + --scope cluster") + + self.cmd("k8s-extension create --cluster-name {name} --resource-group {rg} --cluster-type connectedClusters \ + --extension-type microsoft.arc.containerstorage --name azure-arc-containerstorage \ + --auto-upgrade-minor-version False --config feature.diskStorageClass=default,local-path \ + --config edgeStorageConfiguration.create=True --scope cluster") + + self.cmd("k8s-extension create --cluster-name {name} --resource-group {rg} --cluster-type connectedClusters \ + --extension-type microsoft.graytown.testextension --name graytown-test-extension --version 1.302.0 \ + --release-train preview --auto-upgrade-minor-version False --release-namespace graytowntest --scope cluster") + + # The error message should be achieved from the agent-update-validator + with self.assertRaisesRegex(CLIError, "The target agent version 1.26.0 is not compatible."): + self.cmd( + "connectedk8s upgrade -g {rg} -n {name} --agent-version 1.26.0 --kube-config {kubeconfig} \ + --kube-context {managed_cluster_name}-admin", + ) + @live_only() @ResourceGroupPreparer( name_prefix="conk8stest", location=CONFIG["location"], random_name_length=16 diff --git a/src/connectedk8s/setup.py b/src/connectedk8s/setup.py index 7625c142670..4f9959412ad 100644 --- a/src/connectedk8s/setup.py +++ b/src/connectedk8s/setup.py @@ -13,7 +13,7 @@ # TODO: Confirm this is the right version number you want and it matches your # HISTORY.rst entry. -VERSION = "1.10.7" +VERSION = "1.10.8" # The full list of classifiers is available at # https://pypi.python.org/pypi?%3Aaction=list_classifiers