[AKS] support Virtual Machines node pools in AKS (#31780)

Author: HuShaoRu

src/azure-cli/azure/cli/command_modules/acs/_consts.py

@@ -42,11 +42,15 @@
 # vm set type
 CONST_VIRTUAL_MACHINE_SCALE_SETS = "VirtualMachineScaleSets"
 CONST_AVAILABILITY_SET = "AvailabilitySet"
+CONST_VIRTUAL_MACHINES = "VirtualMachines"
 
 # vm size
 CONST_DEFAULT_NODE_VM_SIZE = ""
 CONST_DEFAULT_WINDOWS_NODE_VM_SIZE = ""
 
+CONST_DEFAULT_VMS_VM_SIZE = "Standard_DS2_v2"
+CONST_DEFAULT_WINDOWS_VMS_VM_SIZE = "Standard_D2s_v3"
+
 # gpu instance
 CONST_GPU_INSTANCE_PROFILE_MIG1_G = "MIG1g"
 CONST_GPU_INSTANCE_PROFILE_MIG2_G = "MIG2g"

src/azure-cli/azure/cli/command_modules/acs/_help.py

@@ -66,6 +66,9 @@
   - name: --node-vm-size -s
     type: string
     short-summary: Size of Virtual Machines to create as Kubernetes nodes. If the user does not specify one, server will select a default VM size for her/him.
+  - name: --vm-sizes
+    type: string
+    short-summary: Comma-separated list of VM sizes. Valid for VirtualMachines node pool only. If `--vm-sizes` not specified but `--node-vm-size` specified, value of `--node-vm-size` will be used. If neither of them specified, defaults to Standard_DS2_v2 for Linux or Standard_D2s_v3 for Windows.
   - name: --dns-name-prefix -p
     type: string
     short-summary: Prefix for hostnames that are created. If not specified, generate a hostname using the managed cluster and resource group names.
@@ -201,7 +204,7 @@
     short-summary: Maximum nodes count used for autoscaler, when "--enable-cluster-autoscaler" specified. Please specify the value in the range of [1, 1000].
   - name: --vm-set-type
     type: string
-    short-summary: Agent pool vm set type. VirtualMachineScaleSets or AvailabilitySet. Defaults to 'VirtualMachineScaleSets'
+    short-summary: Agent pool vm set type. VirtualMachineScaleSets or AvailabilitySet or VirtualMachines. Defaults to 'VirtualMachineScaleSets'
   - name: --enable-addons -a
     type: string
     short-summary: Enable the Kubernetes addons in a comma-separated list.
@@ -301,10 +304,10 @@
     short-summary: The ID of a PPG.
   - name: --enable-node-public-ip
     type: bool
-    short-summary: Enable VMSS node public IP.
+    short-summary: Enable VMSS or VMs node public IP.
   - name: --node-public-ip-prefix-id
     type: string
-    short-summary: Public IP prefix ID used to assign public IPs to VMSS nodes.
+    short-summary: Public IP prefix ID used to assign public IPs to VMSS or VMs nodes.
   - name: --workspace-resource-id
     type: string
     short-summary: The resource ID of an existing Log Analytics Workspace to use for storing monitoring data. If not specified, uses the default Log Analytics Workspace if it exists, otherwise creates one.
@@ -559,10 +562,10 @@
     short-summary: Enable exporting Kubernetes Namespace and Deployment details to the Cost Analysis views in the Azure portal. For more information see aka.ms/aks/docs/cost-analysis.
   - name: --enable-secure-boot
     type: bool
-    short-summary: Enable Secure Boot on all node pools in the cluster. Must use VMSS agent pool type.
+    short-summary: Enable Secure Boot on all node pools in the cluster. Must use VMSS or VMs agent pool type.
   - name: --enable-vtpm
     type: bool
-    short-summary: Enable vTPM on all node pools in the cluster. Must use VMSS agent pool type.
+    short-summary: Enable vTPM on all node pools in the cluster. Must use VMSS or VMs agent pool type.
   - name: --enable-acns
     type: bool
     short-summary: Enable advanced network functionalities on a cluster. Enabling this will incur additional costs. For non-cilium clusters, acns security will be disabled by default until further notice.
@@ -670,6 +673,8 @@
     text: az aks create -g MyResourceGroup -n MyManagedCluster --enable-azure-service-mesh
   - name: Create a kubernetes cluster with a nodepool having ip allocation mode set to "StaticBlock"
     text: az aks create -g MyResourceGroup -n MyManagedCluster --os-sku Ubuntu --max-pods MaxPodsPerNode --network-plugin azure --vnet-subnet-id /subscriptions/SubID/resourceGroups/AnotherResourceGroup/providers/Microsoft.Network/virtualNetworks/MyVnet/subnets/NodeSubnet --pod-subnet-id /subscriptions/SubID/resourceGroups/AnotherResourceGroup/providers/Microsoft.Network/virtualNetworks/MyVnet/subnets/PodSubnet --pod-ip-allocation-mode StaticBlock
+  - name: Create a kubernetes cluster with VirtualMachines vm set type.
+    text: az aks create -g MyResourceGroup -n MyManagedCluster --vm-set-type VirtualMachines --vm-sizes "VMSize1,VMSize2" --node-count 3
 """
 
 helps['aks update'] = """
@@ -1570,6 +1575,12 @@
   - name: --node-vm-size -s
     type: string
     short-summary: Size of Virtual Machines to create as Kubernetes nodes. If the user does not specify one, server will select a default VM size for her/him.
+  - name: --vm-sizes
+    type: string
+    short-summary: Comma-separated list of VM sizes. Valid for VirtualMachines node pool only. If `--vm-sizes` not specified but `--node-vm-size` specified, value of `--node-vm-size` will be used. If neither of them specified, defaults to Standard_DS2_v2 for Linux or Standard_D2s_v3 for Windows.
+  - name: --vm-set-type
+    type: string
+    short-summary: Agent pool vm set type. VirtualMachineScaleSets or AvailabilitySet or VirtualMachines. Defaults to 'VirtualMachineScaleSets'
   - name: --node-count -c
     type: int
     short-summary: Number of nodes in the Kubernetes agent pool. After creating a cluster, you can change the size of its node pool with `az aks scale`.
@@ -1593,10 +1604,10 @@
     short-summary: Availability zones where agent nodes will be placed. Also, to install agent nodes to more than one zone you need to pass zone numbers separated by blanks.  For example -  To have all 3 zones, you are expected to enter `--zones 1 2 3`
   - name: --enable-node-public-ip
     type: bool
-    short-summary: Enable VMSS node public IP.
+    short-summary: Enable VMSS or VMs node public IP.
   - name: --node-public-ip-prefix-id
     type: string
-    short-summary: Public IP prefix ID used to assign public IPs to VMSS nodes.
+    short-summary: Public IP prefix ID used to assign public IPs to VMSS or VMs nodes.
   - name: --vnet-subnet-id
     type: string
     short-summary: The Resource Id of a subnet in an existing VNet into which to deploy the cluster.
@@ -1710,10 +1721,10 @@
     short-summary: Disable Windows OutboundNAT on Windows agent node pool.
   - name: --enable-secure-boot
     type: bool
-    short-summary: Enable Secure Boot on agent node pool. Must use VMSS agent pool type.
+    short-summary: Enable Secure Boot on agent node pool. Must use VMSS or VMs agent pool type.
   - name: --enable-vtpm
     type: bool
-    short-summary: Enable vTPM on agent node pool. Must use VMSS agent pool type.
+    short-summary: Enable vTPM on agent node pool. Must use VMSS or VMs agent pool type.
   - name: --if-match
     type: string
     short-summary: The value provided will be compared to the ETag of the agentpool, if it matches the operation will proceed. If it does not match, the request will be rejected to prevent accidental overwrites. This must not be specified when creating a new agentpool.
@@ -1745,6 +1756,8 @@
     text: az aks nodepool add -g MyResourceGroup -n MyNodePool --cluster-name MyMC --node-vm-size VMSize --crg-id "/subscriptions/SubID/resourceGroups/ResourceGroupName/providers/Microsoft.ContainerService/CapacityReservationGroups/MyCRGID"
   - name: Create a nodepool with ip allocation mode set to "StaticBlock" and using a pod subnet ID
     text: az aks nodepool add -g MyResourceGroup -n nodepool1 --cluster-name MyManagedCluster  --os-sku Ubuntu --pod-subnet-id /subscriptions/SubID/resourceGroups/AnotherResourceGroup/providers/Microsoft.Network/virtualNetworks/MyVnet/subnets/MySubnet --pod-ip-allocation-mode StaticBlock
+  - name: create a nodepool of type VirtualMachines
+    text: az aks nodepool add -g MyResourceGroup -n MyNodePool --cluster-name MyMC --vm-set-type VirtualMachines --vm-sizes "VMSize1,VMSize2" --node-count 3
 """
 
 helps['aks nodepool delete'] = """
@@ -1857,13 +1870,13 @@
     short-summary: Switch to use non-FIPS-enabled OS on agent nodes.
   - name: --enable-secure-boot
     type: bool
-    short-summary: Enable Secure Boot on an existing Trusted Launch enabled agent node pool. Must use VMSS agent pool type.
+    short-summary: Enable Secure Boot on an existing Trusted Launch enabled agent node pool. Must use VMSS or VMs agent pool type.
   - name: --disable-secure-boot
     type: bool
     short-summary: Disable Secure Boot on on an existing Trusted Launch enabled agent node pool.
   - name: --enable-vtpm
     type: bool
-    short-summary: Enable vTPM on an existing Trusted Launch enabled agent node pool. Must use VMSS agent pool type.
+    short-summary: Enable vTPM on an existing Trusted Launch enabled agent node pool. Must use VMSS or VMs agent pool type.
   - name: --disable-vtpm
     type: bool
     short-summary: Disable vTPM on an existing Trusted Launch enabled agent node pool.
@@ -2000,6 +2013,47 @@
     crafted: true
 """
 
+helps['aks nodepool manual-scale'] = """
+    type: group
+    short-summary: Commands to manage nodepool virtualMachineProfile.scale.manual.
+"""
+
+helps['aks nodepool manual-scale add'] = """
+    type: command
+    short-summary: Add a new manual to a VirtualMachines agentpool in the managed Kubernetes cluster.
+    parameters:
+        - name: --vm-sizes
+          type: string
+          short-summary: Comma-separated list of sizes in the manual.
+        - name: --node-count -c
+          type: int
+          short-summary: Number of nodes in the manual.
+"""
+
+helps['aks nodepool manual-scale update'] = """
+    type: command
+    short-summary: Update an existing manual of a VirtualMachines agentpool in the managed Kubernetes cluster.
+    parameters:
+        - name: --current-vm-sizes
+          type: string
+          short-summary: Comma-separated list of sizes in the manual to be updated.
+        - name: --vm-sizes
+          type: string
+          short-summary: Comma-separated list of new sizes.
+        - name: --node-count -c
+          type: int
+          short-summary: Number of nodes in the manual.
+"""
+
+helps['aks nodepool manual-scale delete'] = """
+    type: command
+    short-summary: Delete an existing manual to a VirtualMachines agentpool in the managed Kubernetes cluster.
+    parameters:
+        - name: --current-vm-sizes
+          type: string
+          short-summary: Comma-separated list of sizes in the manual to be deleted.
+"""
+
 helps['aks show'] = """
 type: command
 short-summary: Show the details for a managed Kubernetes cluster.

src/azure-cli/azure/cli/command_modules/acs/_params.py

@@ -445,6 +445,7 @@ def load_arguments(self, _):
         c.argument('nodepool_name', default='nodepool1',
                    help='Node pool name, up to 12 alphanumeric characters', validator=validate_nodepool_name)
         c.argument('node_vm_size', options_list=['--node-vm-size', '-s'], completer=get_vm_size_completion_list)
+        c.argument('vm_sizes')
         c.argument('os_sku', arg_type=get_enum_type(node_os_skus_create), validator=validate_os_sku)
         c.argument('snapshot_id', validator=validate_snapshot_id)
         c.argument('vnet_subnet_id', validator=validate_vnet_subnet_id)
@@ -823,6 +824,8 @@ def load_arguments(self, _):
 
     with self.argument_context('aks nodepool add') as c:
         c.argument('node_vm_size', options_list=['--node-vm-size', '-s'], completer=get_vm_size_completion_list)
+        c.argument('vm_sizes')
+        c.argument('vm_set_type', validator=validate_vm_set_type)
         c.argument('os_type')
         c.argument('os_sku', arg_type=get_enum_type(node_os_skus), validator=validate_os_sku)
         c.argument('snapshot_id', validator=validate_snapshot_id)
@@ -913,6 +916,16 @@ def load_arguments(self, _):
         c.argument('snapshot_id', validator=validate_snapshot_id)
         c.argument('yes', options_list=['--yes', '-y'], help='Do not prompt for confirmation.', action='store_true')
 
+    with self.argument_context("aks nodepool manual-scale add") as c:
+        c.argument("vm_sizes")
+
+    with self.argument_context("aks nodepool manual-scale update") as c:
+        c.argument("current_vm_sizes")
+        c.argument("vm_sizes")
+
+    with self.argument_context("aks nodepool manual-scale delete") as c:
+        c.argument("current_vm_sizes")
+
     with self.argument_context('aks command invoke') as c:
         c.argument('command_string', options_list=[
                    "--command", "-c"], help='the command to run')

src/azure-cli/azure/cli/command_modules/acs/_validators.py

@@ -19,6 +19,9 @@
     CONST_NETWORK_POD_IP_ALLOCATION_MODE_DYNAMIC_INDIVIDUAL,
     CONST_NETWORK_POD_IP_ALLOCATION_MODE_STATIC_BLOCK,
     CONST_NODEPOOL_MODE_GATEWAY,
+    CONST_VIRTUAL_MACHINE_SCALE_SETS,
+    CONST_AVAILABILITY_SET,
+    CONST_VIRTUAL_MACHINES,
 )
 from azure.cli.core import keys
 from azure.cli.core.azclierror import (
@@ -185,9 +188,12 @@ def validate_vm_set_type(namespace):
     if namespace.vm_set_type is not None:
         if namespace.vm_set_type == '':
             return
-        if namespace.vm_set_type.lower() != "availabilityset" and \
-                namespace.vm_set_type.lower() != "virtualmachinescalesets":
-            raise CLIError("--vm-set-type can only be VirtualMachineScaleSets or AvailabilitySet")
+        if (
+            namespace.vm_set_type.lower() != CONST_VIRTUAL_MACHINE_SCALE_SETS.lower() and
+            namespace.vm_set_type.lower() != CONST_AVAILABILITY_SET.lower() and
+            namespace.vm_set_type.lower() != CONST_VIRTUAL_MACHINES.lower()
+        ):
+            raise CLIError("--vm-set-type can only be VirtualMachineScaleSets or AvailabilitySet or VirtualMachines")
 
 
 def validate_load_balancer_sku(namespace):

src/azure-cli/azure/cli/command_modules/acs/agentpool_decorator.py

@@ -15,13 +15,16 @@
     CONST_DEFAULT_NODE_OS_TYPE,
     CONST_DEFAULT_NODE_VM_SIZE,
     CONST_DEFAULT_WINDOWS_NODE_VM_SIZE,
+    CONST_DEFAULT_VMS_VM_SIZE,
+    CONST_DEFAULT_WINDOWS_VMS_VM_SIZE,
     CONST_NODEPOOL_MODE_SYSTEM,
     CONST_NODEPOOL_MODE_USER,
     CONST_SCALE_DOWN_MODE_DELETE,
     CONST_SCALE_SET_PRIORITY_REGULAR,
     CONST_SCALE_SET_PRIORITY_SPOT,
     CONST_SPOT_EVICTION_POLICY_DELETE,
     CONST_VIRTUAL_MACHINE_SCALE_SETS,
+    CONST_VIRTUAL_MACHINES,
     CONST_OS_SKU_WINDOWS2019,
     CONST_OS_SKU_WINDOWS2022,
     AgentPoolDecoratorMode,
@@ -629,6 +632,33 @@ def get_node_vm_size(self) -> str:
         """
         return self._get_node_vm_size(read_only=False)
 
+    def get_vm_sizes(self) -> List[str]:
+        """Obtain the value of vm_sizes.
+
+        :return: list of strings
+        """
+        raw_value = self.raw_param.get("vm_sizes")
+        node_vm_size = self.raw_param.get("node_vm_size")
+        if raw_value is not None:
+            # vm_sizes is a comma-separated string, only used when vm_set_type is VirtualMachines
+            if self.get_vm_set_type() != CONST_VIRTUAL_MACHINES:
+                raise InvalidArgumentValueError("--vm-sizes can only be used with --vm-set-type VirtualMachines.")
+            if node_vm_size:
+                raise MutuallyExclusiveArgumentError("Cannot specify -vm-sizes and --node-vm-size at the same time.")
+            vm_sizes = [x.strip() for x in raw_value.split(",")]
+        else:
+            # when vm_sizes is not specified, try to use the value from node_vm_size (only 1 size)
+            if node_vm_size:
+                vm_sizes = [node_vm_size]
+            else:
+                # use default value
+                if self.get_os_type().lower() == "windows":
+                    vm_sizes = [CONST_DEFAULT_WINDOWS_VMS_VM_SIZE]
+                else:
+                    vm_sizes = [CONST_DEFAULT_VMS_VM_SIZE]
+
+        return vm_sizes
+
     def _get_os_type(self, read_only: bool = False) -> Union[str, None]:
         """Internal function to dynamically obtain the value of os_type according to the context.
 
@@ -1222,8 +1252,12 @@ def get_vm_set_type(self) -> str:
             vm_set_type = CONST_VIRTUAL_MACHINE_SCALE_SETS
         elif vm_set_type.lower() == CONST_AVAILABILITY_SET.lower():
             vm_set_type = CONST_AVAILABILITY_SET
+        elif vm_set_type.lower() == CONST_VIRTUAL_MACHINES.lower():
+            vm_set_type = CONST_VIRTUAL_MACHINES
         else:
-            raise InvalidArgumentValueError("--vm-set-type can only be VirtualMachineScaleSets or AvailabilitySet")
+            raise InvalidArgumentValueError(
+                "--vm-set-type can only be VirtualMachineScaleSets or AvailabilitySet or VirtualMachines"
+            )
         # this parameter does not need validation
         return vm_set_type
 
@@ -2064,6 +2098,37 @@ def set_up_pod_ip_allocation_mode(self, agentpool: AgentPool) -> AgentPool:
 
         return agentpool
 
+    def set_up_virtual_machines_profile(self, agentpool: AgentPool) -> AgentPool:
+        """Set up virtual machines profile for the AgentPool object.
+
+        :return: the AgentPool object
+        """
+        self._ensure_agentpool(agentpool)
+
+        # validate vm_sizes first, then skip if not Virtual Machines
+        sizes = self.context.get_vm_sizes()
+        if len(sizes) != 1:
+            raise InvalidArgumentValueError(f"We only accept single sku size for manual profile. {sizes} is invalid.")
+
+        if self.context.get_vm_set_type() != CONST_VIRTUAL_MACHINES:
+            return agentpool
+
+        count, _, _, _ = self.context.get_node_count_and_enable_cluster_autoscaler_min_max_count()
+        agentpool.virtual_machines_profile = self.models.VirtualMachinesProfile(
+            scale=self.models.ScaleProfile(
+                manual=[
+                    self.models.ManualScaleProfile(
+                        size=sizes[0],
+                        count=count,
+                    )
+                ]
+            )
+        )
+        agentpool.vm_size = None
+        agentpool.count = None
+
+        return agentpool
+
     def construct_agentpool_profile_default(self, bypass_restore_defaults: bool = False) -> AgentPool:
         """The overall controller used to construct the AgentPool profile by default.
 
@@ -2114,6 +2179,8 @@ def construct_agentpool_profile_default(self, bypass_restore_defaults: bool = Fa
         agentpool = self.set_up_gpu_profile(agentpool)
         # set up agentpool gateway profile
         agentpool = self.set_up_agentpool_gateway_profile(agentpool)
+        # set up virtual machines profile
+        agentpool = self.set_up_virtual_machines_profile(agentpool)
         # restore defaults
         if not bypass_restore_defaults:
             agentpool = self._restore_defaults_in_agentpool(agentpool)

src/azure-cli/azure/cli/command_modules/acs/commands.py

@@ -151,6 +151,12 @@ def load_command_table(self, _):
         g.custom_command('operation-abort', 'aks_agentpool_operation_abort', supports_no_wait=True)
         g.custom_command('delete-machines', 'aks_agentpool_delete_machines', supports_no_wait=True)
 
+    # AKS nodepool manual-scale command
+    with self.command_group('aks nodepool manual-scale', agent_pools_sdk, client_factory=cf_agent_pools) as g:
+        g.custom_command("add", "aks_agentpool_manual_scale_add", supports_no_wait=True)
+        g.custom_command("update", "aks_agentpool_manual_scale_update", supports_no_wait=True)
+        g.custom_command("delete", "aks_agentpool_manual_scale_delete", supports_no_wait=True, confirmation=True)
+
     with self.command_group('aks command', managed_clusters_sdk, client_factory=cf_managed_clusters) as g:
         g.custom_command('invoke', 'aks_runcommand', supports_no_wait=True,
                          table_transformer=aks_run_command_result_format)