[AKS] Adjust Node Initialization Taints preview CLI to ignore k8s taints with hard effects on system mode node pools (Azure#8626)

Author: UtheMan

src/aks-preview/HISTORY.rst

@@ -12,6 +12,7 @@ To release a new version, please select a new version number (usually plus 1 to
 Pending
 +++++++
 * Vendor new SDK and bump API version to 2025-01-02-preview.
+* Modify behavior for `--nodepool-initialization-taints` to ignore taints with hard effects on node pools with system mode when creating or updating a cluster.
 
 14.0.0b1
 +++++++

src/aks-preview/azext_aks_preview/_helpers.py

@@ -9,7 +9,7 @@
 import stat
 import sys
 import tempfile
-from typing import TypeVar
+from typing import List, TypeVar
 
 import yaml
 from azext_aks_preview._client_factory import (
@@ -374,3 +374,24 @@ def check_is_monitoring_addon_enabled(addons, instance):
     except Exception as ex:  # pylint: disable=broad-except
         logger.debug("failed to check monitoring addon enabled: %s", ex)
     return is_monitoring_addon_enabled
+
+
+def filter_hard_taints(node_initialization_taints: List[str]) -> List[str]:
+    filtered_taints = []
+    for taint in node_initialization_taints:
+        if not taint:
+            continue
+        # Parse the taint to get the effect
+        taint_parts = taint.split(":")
+        if len(taint_parts) == 2:
+            effect = taint_parts[-1].strip()
+            # Keep the taint if it has a soft effect (PreferNoSchedule)
+            # or if it's a CriticalAddonsOnly taint - AKS allows those on system pools
+            if effect.lower() == "prefernoschedule" or taint.lower().startswith("criticaladdonsonly"):
+                filtered_taints.append(taint)
+            else:
+                logger.warning('Taint %s with hard effect will be skipped from system pool', taint)
+        else:
+            # If the taint doesn't have a recognizable format, keep it, if it's incorrect - AKS-RP will return an error
+            filtered_taints.append(taint)
+    return filtered_taints

src/aks-preview/azext_aks_preview/agentpool_decorator.py

@@ -39,7 +39,10 @@
     CONST_DEFAULT_WINDOWS_NODE_VM_SIZE,
     CONST_SSH_ACCESS_LOCALUSER,
 )
-from azext_aks_preview._helpers import get_nodepool_snapshot_by_snapshot_id
+from azext_aks_preview._helpers import (
+    get_nodepool_snapshot_by_snapshot_id,
+    filter_hard_taints,
+)
 
 logger = get_logger(__name__)
 
@@ -870,7 +873,11 @@ def set_up_init_taints(self, agentpool: AgentPool) -> AgentPool:
         :return: the AgentPool object
         """
         self._ensure_agentpool(agentpool)
-        agentpool.node_initialization_taints = self.context.get_node_initialization_taints()
+        nodepool_initialization_taints = self.context.get_node_initialization_taints()
+        # filter out taints with hard effects for System pools
+        if agentpool.mode is None or agentpool.mode.lower() == "system":
+            nodepool_initialization_taints = filter_hard_taints(nodepool_initialization_taints)
+        agentpool.node_initialization_taints = nodepool_initialization_taints
         return agentpool
 
     def set_up_artifact_streaming(self, agentpool: AgentPool) -> AgentPool:

src/aks-preview/azext_aks_preview/managed_cluster_decorator.py

@@ -47,6 +47,7 @@
     check_is_private_cluster,
     get_cluster_snapshot_by_snapshot_id,
     setup_common_safeguards_profile,
+    filter_hard_taints,
 )
 from azext_aks_preview._loadbalancer import create_load_balancer_profile
 from azext_aks_preview._loadbalancer import (
@@ -4954,7 +4955,11 @@ def update_nodepool_initialization_taints_mc(self, mc: ManagedCluster) -> Manage
         nodepool_initialization_taints = self.context.get_nodepool_initialization_taints()
         if nodepool_initialization_taints is not None:
             for agent_profile in mc.agent_pool_profiles:
-                agent_profile.node_initialization_taints = nodepool_initialization_taints
+                if agent_profile.mode is not None and agent_profile.mode.lower() == "user":
+                    agent_profile.node_initialization_taints = nodepool_initialization_taints
+                    continue
+                # Filter out taints with hard effects (NoSchedule and NoExecute) for system pools
+                agent_profile.node_initialization_taints = filter_hard_taints(nodepool_initialization_taints)
         return mc
 
     def update_node_provisioning_mode(self, mc: ManagedCluster) -> ManagedCluster:

src/aks-preview/azext_aks_preview/tests/latest/recordings/test_aks_check_network.yaml

@@ -15063,7 +15063,7 @@ def test_aks_create_cluster_with_taints(self, resource_group, resource_group_loc
             "initTaint1=value1:PreferNoSchedule,initTaint2=value2:PreferNoSchedule"
         )
         nodepool_taints2 = "taint1=value2:PreferNoSchedule"
-        nodepool_init_taints2 = "initTaint1=value2:PreferNoSchedule"
+        nodepool_init_taints2 = "initTaint1=value2:PreferNoSchedule,initTaint2=value2:NoSchedule,CriticalAddonsOnly=true:NoSchedule,CriticalAddonsOnly=true:NoExecute"
         self.kwargs.update(
             {
                 "resource_group": resource_group,
@@ -15116,6 +15116,20 @@ def test_aks_create_cluster_with_taints(self, resource_group, resource_group_loc
             ],
         )
 
+        # add another nodepool with user mode, without init taints for now - AP level operations are blocked for init taints
+        create_ap_cmd = (
+            "aks nodepool add --resource-group={resource_group} --cluster-name={name} "
+            "--name={nodepool2_name} "
+            "--aks-custom-headers AKSHTTPCustomFeatures=Microsoft.ContainerService/NodeInitializationTaintsPreview "
+        )
+        self.cmd(
+            create_ap_cmd,
+            checks=[
+                self.check("provisioningState", "Succeeded"),
+                self.check("mode", "User"),
+            ],
+        )
+
         update_cmd = (
             "aks update --resource-group={resource_group} --name={name} "
             "--nodepool-taints {nodepool_taints2} "
@@ -15130,13 +15144,53 @@ def test_aks_create_cluster_with_taints(self, resource_group, resource_group_loc
                     "agentPoolProfiles[0].nodeTaints[0]",
                     "taint1=value2:PreferNoSchedule",
                 ),
+                self.check(
+                    "agentPoolProfiles[0].nodeInitializationTaints[] | length(@)",
+                    3,
+                ),
                 self.check(
                     "agentPoolProfiles[0].nodeInitializationTaints[0]",
                     "initTaint1=value2:PreferNoSchedule",
                 ),
+                self.check(
+                    "agentPoolProfiles[0].nodeInitializationTaints[1]",
+                    "CriticalAddonsOnly=true:NoSchedule",
+                ),
+                self.check(
+                    "agentPoolProfiles[0].nodeInitializationTaints[2]",
+                    "CriticalAddonsOnly=true:NoExecute",
+                ),
+                self.check(
+                    "agentPoolProfiles[1].nodeInitializationTaints[] | length(@)",
+                    4,
+                ),
+                self.check(
+                    "agentPoolProfiles[1].nodeInitializationTaints[0]",
+                    "initTaint1=value2:PreferNoSchedule",
+                ),
+                self.check(
+                    "agentPoolProfiles[1].nodeInitializationTaints[1]",
+                    "initTaint2=value2:NoSchedule",
+                ),
+                self.check(
+                    "agentPoolProfiles[1].nodeInitializationTaints[2]",
+                    "CriticalAddonsOnly=true:NoSchedule",
+                ),
+                self.check(
+                    "agentPoolProfiles[1].nodeInitializationTaints[3]",
+                    "CriticalAddonsOnly=true:NoExecute",
+                ),
             ],
         )
 
+        # make sure user nodepool cannot be converted to system pool with hard taints present
+        self.cmd(
+            "aks nodepool update -g {resource_group} --cluster-name {name} -n {nodepool2_name} "
+            "--mode System "
+            "--aks-custom-headers AKSHTTPCustomFeatures=Microsoft.ContainerService/NodeInitializationTaintsPreview ",
+            expect_failure=True,
+        )
+
         update_cmd = (
             "aks update --resource-group={resource_group} --name={name} "
             '--nodepool-taints "" '

src/aks-preview/azext_aks_preview/tests/latest/recordings/test_aks_create_cluster_with_taints.yaml

@@ -14,6 +14,7 @@
     get_nodepool_snapshot,
     get_nodepool_snapshot_by_snapshot_id,
     process_message_for_run_command,
+    filter_hard_taints,
 )
 from azext_aks_preview.__init__ import register_aks_preview_resource_type
 from azext_aks_preview._client_factory import CUSTOM_MGMT_AKS_PREVIEW
@@ -159,5 +160,56 @@ def test_process_message_for_run_command(self):
         self.assertEqual(str(cm.exception), err)
 
 
+class FilterHardTaintsTestCase(unittest.TestCase):
+    def test_filter_hard_taints_keeps_only_soft_taints(self):
+        input_taints = ["taint1=val1:NoSchedule", "taint2=val2:NoExecute", "taint3=val3:PreferNoSchedule"]
+        expected_filtered_taints = ["taint3=val3:PreferNoSchedule"]
+        self.assertEqual(filter_hard_taints(input_taints), expected_filtered_taints)
+
+    def test_filter_hard_taints_preserves_critical_addons_only_taints(self):
+        input_taints = [
+            "CriticalAddonsOnly=true:NoSchedule",  
+            "CriticalAddonsOnly=true:NoExecute",
+            "taint1=val1:NoSchedule",
+            "taint2=val2:PreferNoSchedule"
+        ]
+        expected_filtered_taints = [
+            "CriticalAddonsOnly=true:NoSchedule", 
+            "CriticalAddonsOnly=true:NoExecute", 
+            "taint2=val2:PreferNoSchedule"
+        ]
+        self.assertEqual(filter_hard_taints(input_taints), expected_filtered_taints)
+    
+    def test_filter_hard_taints_with_empty_list(self):
+        input_taints = []
+        expected_filtered_taints = []
+        self.assertEqual(filter_hard_taints(input_taints), expected_filtered_taints)
+    
+    def test_filter_hard_taints_with_empty_strings(self):
+        input_taints = ["taint1=val1:NoSchedule", "", "taint3=val3:PreferNoSchedule"]
+        expected_filtered_taints = ["taint3=val3:PreferNoSchedule"]
+        self.assertEqual(filter_hard_taints(input_taints), expected_filtered_taints)
+    
+    def test_filter_hard_taints_with_invalid_format(self):
+        input_taints = ["invalid-format", "taint1=val1:NoSchedule", "another-invalid", "taint2=val2:PreferNoSchedule"]
+        expected_filtered_taints = ["invalid-format", "another-invalid", "taint2=val2:PreferNoSchedule"]
+        self.assertEqual(filter_hard_taints(input_taints), expected_filtered_taints)
+    
+    def test_filter_hard_taints_case_insensitive(self):
+        input_taints = ["taint1=val1:noschedule", "taint2=val2:PREFERNOSCHEDULE", "taint3=val3:prefernoschedule"]
+        expected_filtered_taints = ["taint2=val2:PREFERNOSCHEDULE", "taint3=val3:prefernoschedule"]
+        self.assertEqual(filter_hard_taints(input_taints), expected_filtered_taints)
+    
+    def test_filter_hard_taints_mixed_effects(self):
+        input_taints = [
+            "key1=value1:NoSchedule", 
+            "key2=value2:NoExecute", 
+            "key3=value3:PreferNoSchedule",
+            "key4:NoSchedule",
+            "key5:PreferNoSchedule"
+        ]
+        expected_filtered_taints = ["key3=value3:PreferNoSchedule", "key5:PreferNoSchedule"]
+        self.assertEqual(filter_hard_taints(input_taints), expected_filtered_taints)
+
 if __name__ == "__main__":
     unittest.main()