CKS Enhancements

api/src/main/java/com/cloud/event/EventTypes.java

@@ -289,6 +289,8 @@ public class EventTypes {
 
     //registering userdata events
     public static final String EVENT_REGISTER_USER_DATA = "REGISTER.USER.DATA";
+    public static final String EVENT_REGISTER_CNI_CONFIG = "REGISTER.CNI.CONFIG";
+    public static final String EVENT_DELETE_CNI_CONFIG = "DELETE.CNI.CONFIG";
 
     //register for user API and secret keys
     public static final String EVENT_REGISTER_FOR_SECRET_API_KEY = "REGISTER.USER.KEY";

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java → api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java

@@ -44,6 +44,8 @@ enum Event {
         AutoscaleRequested,
         ScaleUpRequested,
         ScaleDownRequested,
+        AddNodeRequested,
+        RemoveNodeRequested,
         UpgradeRequested,
         OperationSucceeded,
         OperationFailed,
@@ -59,6 +61,8 @@ enum State {
         Stopped("All resources for the Kubernetes cluster are destroyed, Kubernetes cluster may still have ephemeral resource like persistent volumes provisioned"),
         Scaling("Transient state in which resources are either getting scaled up/down"),
         Upgrading("Transient state in which cluster is getting upgraded"),
+        Importing("Transient state in which additional nodes are added as worker nodes to a cluster"),
+        RemovingNodes("Transient state in which additional nodes are removed from a cluster"),
         Alert("State to represent Kubernetes clusters which are not in expected desired state (operationally in active control place, stopped cluster VM's etc)."),
         Recovering("State in which Kubernetes cluster is recovering from alert state"),
         Destroyed("End state of Kubernetes cluster in which all resources are destroyed, cluster will not be usable further"),
@@ -96,6 +100,17 @@ enum State {
             s_fsm.addTransition(State.Upgrading, Event.OperationSucceeded, State.Running);
             s_fsm.addTransition(State.Upgrading, Event.OperationFailed, State.Alert);
 
+            s_fsm.addTransition(State.Running, Event.AddNodeRequested, State.Importing);
+            s_fsm.addTransition(State.Alert, Event.AddNodeRequested, State.Importing);
+            s_fsm.addTransition(State.Importing, Event.OperationSucceeded, State.Running);
+            s_fsm.addTransition(State.Importing, Event.OperationFailed, State.Running);
+            s_fsm.addTransition(State.Alert, Event.OperationSucceeded, State.Running);
+
+            s_fsm.addTransition(State.Running, Event.RemoveNodeRequested, State.RemovingNodes);
+            s_fsm.addTransition(State.Alert, Event.RemoveNodeRequested, State.RemovingNodes);
+            s_fsm.addTransition(State.RemovingNodes, Event.OperationSucceeded, State.Running);
+            s_fsm.addTransition(State.RemovingNodes, Event.OperationFailed, State.Running);
+
             s_fsm.addTransition(State.Alert, Event.RecoveryRequested, State.Recovering);
             s_fsm.addTransition(State.Recovering, Event.OperationSucceeded, State.Running);
             s_fsm.addTransition(State.Recovering, Event.OperationFailed, State.Alert);
@@ -142,4 +157,13 @@ enum State {
     Long getMaxSize();
     Long getSecurityGroupId();
     ClusterType getClusterType();
+    Long getControlServiceOfferingId();
+    Long getWorkerServiceOfferingId();
+    Long getEtcdServiceOfferingId();
+    Long getControlTemplateId();
+    Long getWorkerTemplateId();
+    Long getEtcdTemplateId();
+    Long getEtcdNodeCount();
+    Long getCniConfigId();
+    String getCniConfigDetails();
 }

api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java

@@ -18,14 +18,23 @@
 
 import org.apache.cloudstack.acl.ControlledEntity;
 
+import java.util.Map;
+
 import com.cloud.user.Account;
 import com.cloud.uservm.UserVm;
 import com.cloud.utils.component.Adapter;
 
 public interface KubernetesServiceHelper extends Adapter {
 
+    enum KubernetesClusterNodeType {
+        CONTROL, WORKER, ETCD, DEFAULT
+    }
+
     ControlledEntity findByUuid(String uuid);
     ControlledEntity findByVmId(long vmId);
     void checkVmCanBeDestroyed(UserVm userVm);
+    boolean isValidNodeType(String nodeType);
+    Map<String, Long> getServiceOfferingNodeTypeMap(Map<String, Map<String, String>> serviceOfferingNodeTypeMap);
+    Map<String, Long> getTemplateNodeTypeMap(Map<String, Map<String, String>> templateNodeTypeMap);
     void cleanupForAccount(Account account);
 }

api/src/main/java/com/cloud/network/NetworkService.java

@@ -268,4 +268,6 @@ Network createPrivateNetwork(String networkName, String displayText, long physic
     InternalLoadBalancerElementService getInternalLoadBalancerElementByNetworkServiceProviderId(long networkProviderId);
     InternalLoadBalancerElementService getInternalLoadBalancerElementById(long providerId);
     List<InternalLoadBalancerElementService> getInternalLoadBalancerElements();
+
+    boolean handleCksIsoOnNetworkVirtualRouter(Long virtualRouterId, boolean mount) throws ResourceUnavailableException;
 }

api/src/main/java/com/cloud/network/vpc/Vpc.java

@@ -105,4 +105,6 @@ public enum State {
     String getIp6Dns1();
 
     String getIp6Dns2();
+
+    boolean useRouterIpAsResolver();
 }

api/src/main/java/com/cloud/network/vpc/VpcService.java

@@ -48,17 +48,17 @@ public interface VpcService {
      * @param vpcName
      * @param displayText
      * @param cidr
-     * @param networkDomain TODO
+     * @param networkDomain   TODO
      * @param ip4Dns1
      * @param ip4Dns2
-     * @param displayVpc TODO
+     * @param displayVpc      TODO
+     * @param useVrIpResolver
      * @return
      * @throws ResourceAllocationException TODO
      */
     Vpc createVpc(long zoneId, long vpcOffId, long vpcOwnerId, String vpcName, String displayText, String cidr, String networkDomain,
                   String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Boolean displayVpc, Integer publicMtu, Integer cidrSize,
-                  Long asNumber, List<Long> bgpPeerIds)
-            throws ResourceAllocationException;
+                  Long asNumber, List<Long> bgpPeerIds, Boolean useVrIpResolver) throws ResourceAllocationException;
 
     /**
      * Persists VPC record in the database

api/src/main/java/com/cloud/server/ManagementService.java

@@ -59,8 +59,10 @@
 import org.apache.cloudstack.api.command.user.ssh.DeleteSSHKeyPairCmd;
 import org.apache.cloudstack.api.command.user.ssh.ListSSHKeyPairsCmd;
 import org.apache.cloudstack.api.command.user.ssh.RegisterSSHKeyPairCmd;
+import org.apache.cloudstack.api.command.user.userdata.DeleteCniConfigurationCmd;
 import org.apache.cloudstack.api.command.user.userdata.DeleteUserDataCmd;
 import org.apache.cloudstack.api.command.user.userdata.ListUserDataCmd;
+import org.apache.cloudstack.api.command.user.userdata.RegisterCniConfigurationCmd;
 import org.apache.cloudstack.api.command.user.userdata.RegisterUserDataCmd;
 import org.apache.cloudstack.api.command.user.vm.GetVMPasswordCmd;
 import org.apache.cloudstack.api.command.user.vmgroup.UpdateVMGroupCmd;
@@ -360,17 +362,23 @@ public interface ManagementService {
      *            The api command class.
      * @return The list of userdatas found.
      */
-    Pair<List<? extends UserData>, Integer> listUserDatas(ListUserDataCmd cmd);
+    Pair<List<? extends UserData>, Integer> listUserDatas(ListUserDataCmd cmd, boolean forCks);
+
+    /**
+     * Registers a cni configuration.
+     *
+     * @param cmd    The api command class.
+     * @return A VO with the registered userdata.
+     */
+    UserData registerCniConfiguration(RegisterCniConfigurationCmd cmd);
 
     /**
      * Registers a userdata.
      *
-     * @param cmd
-     *            The api command class.
+     * @param cmd    The api command class.
      * @return A VO with the registered userdata.
      */
     UserData registerUserData(RegisterUserDataCmd cmd);
-
     /**
      * Deletes a userdata.
      *
@@ -380,6 +388,14 @@ public interface ManagementService {
      */
     boolean deleteUserData(DeleteUserDataCmd cmd);
 
+    /**
+     * Deletes a userdata.
+     *
+     * @param cmd
+     *            The api command class.
+     * @return True on success. False otherwise.
+     */
+    boolean deleteCniConfiguration(DeleteCniConfigurationCmd cmd);
     /**
      * Search registered key pairs for the logged in user.
      *

api/src/main/java/com/cloud/template/TemplateApiService.java

@@ -58,10 +58,23 @@ public interface TemplateApiService {
     VirtualMachineTemplate prepareTemplate(long templateId, long zoneId, Long storageId);
 
 
+    /**
+     * Detach ISO from VM
+     * @param vmId id of the VM
+     * @param isoId id of the ISO (when passed). If it is not passed, it will get it from user_vm table
+     * @param extraParams forced, isVirtualRouter
+     * @return true when operation succeeds, false if not
+     */
+    boolean detachIso(long vmId, Long isoId, Boolean... extraParams);
 
-    boolean detachIso(long vmId, boolean forced);
-
-    boolean attachIso(long isoId, long vmId, boolean forced);
+    /**
+     * Attach ISO to a VM
+     * @param isoId id of the ISO to attach
+     * @param vmId id of the VM to attach the ISO to
+     * @param extraParams: forced, isVirtualRouter
+     * @return true when operation succeeds, false if not
+     */
+    boolean attachIso(long isoId, long vmId, Boolean... extraParams);
 
     /**
      * Deletes a template

api/src/main/java/com/cloud/template/VirtualMachineTemplate.java

@@ -145,6 +145,8 @@ public enum TemplateFilter {
 
     boolean isDeployAsIs();
 
+    boolean isForCks();
+
     Long getUserDataId();
 
     UserData.UserDataOverridePolicy getUserDataOverridePolicy();

api/src/main/java/com/cloud/user/UserData.java

@@ -29,4 +29,5 @@ public enum UserDataOverridePolicy {
     String getUserData();
 
     String getParams();
+    boolean isForCks();
 }

api/src/main/java/com/cloud/vm/UserVmService.java

@@ -20,6 +20,7 @@
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.deploy.DeploymentPlan;
 import org.apache.cloudstack.api.BaseCmd.HTTPMethod;
 import org.apache.cloudstack.api.command.admin.vm.AssignVMCmd;
 import org.apache.cloudstack.api.command.admin.vm.RecoverVMCmd;
@@ -111,7 +112,7 @@ UserVm startVirtualMachine(StartVMCmd cmd) throws StorageUnavailableException, E
 
     UserVm rebootVirtualMachine(RebootVMCmd cmd) throws InsufficientCapacityException, ResourceUnavailableException, ResourceAllocationException;
 
-    void startVirtualMachine(UserVm vm) throws OperationTimedoutException, ResourceUnavailableException, InsufficientCapacityException;
+    void startVirtualMachine(UserVm vm, DeploymentPlan plan) throws OperationTimedoutException, ResourceUnavailableException, InsufficientCapacityException;
 
     void startVirtualMachineForHA(VirtualMachine vm, Map<VirtualMachineProfile.Param, Object> params,
           DeploymentPlanner planner) throws InsufficientCapacityException, ResourceUnavailableException,

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -89,6 +89,9 @@ public interface VmDetailConstants {
     String DEPLOY_AS_IS_CONFIGURATION = "configurationId";
     String KEY_PAIR_NAMES = "keypairnames";
     String CKS_CONTROL_NODE_LOGIN_USER = "controlNodeLoginUser";
+    String CKS_NODE_TYPE = "node";
+    String OFFERING = "offering";
+    String TEMPLATE = "template";
 
     // VMware to KVM VM migrations specific
     String VMWARE_TO_KVM_PREFIX = "vmware-to-kvm";

api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java

@@ -84,7 +84,7 @@ public enum ApiCommandResourceType {
     ObjectStore(org.apache.cloudstack.storage.object.ObjectStore.class),
     Bucket(org.apache.cloudstack.storage.object.Bucket.class),
     QuotaTariff(org.apache.cloudstack.quota.QuotaTariff.class),
-    KubernetesCluster(null),
+    KubernetesCluster(com.cloud.kubernetes.cluster.KubernetesCluster.class),
     KubernetesSupportedVersion(null),
     SharedFS(org.apache.cloudstack.storage.sharedfs.SharedFS.class);
 

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -119,6 +119,10 @@ public class ApiConstants {
     public static final String CN = "cn";
     public static final String COMMAND = "command";
     public static final String CMD_EVENT_TYPE = "cmdeventtype";
+    public static final String CNI_CONFIG = "cniconfig";
+    public static final String CNI_CONFIG_ID = "cniconfigurationid";
+    public static final String CNI_CONFIG_DETAILS = "cniconfigdetails";
+    public static final String CNI_CONFIG_NAME = "cniconfigname";
     public static final String COMPONENT = "component";
     public static final String CPU_CORE_PER_SOCKET = "cpucorepersocket";
     public static final String CPU_NUMBER = "cpunumber";
@@ -140,6 +144,7 @@ public class ApiConstants {
     public static final String ENCRYPT_FORMAT = "encryptformat";
     public static final String ENCRYPT_ROOT = "encryptroot";
     public static final String ENCRYPTION_SUPPORTED = "encryptionsupported";
+    public static final String ETCD_IPS = "etcdips";
     public static final String MIN_IOPS = "miniops";
     public static final String MAX_IOPS = "maxiops";
     public static final String HYPERVISOR_SNAPSHOT_RESERVE = "hypervisorsnapshotreserve";
@@ -332,6 +337,7 @@ public class ApiConstants {
     public static final String LBID = "lbruleid";
     public static final String LB_PROVIDER = "lbprovider";
     public static final String MAC_ADDRESS = "macaddress";
+    public static final String MANUAL_UPGRADE = "manualupgrade";
     public static final String MAX = "max";
     public static final String MAX_SNAPS = "maxsnaps";
     public static final String MAX_BACKUPS = "maxbackups";
@@ -343,6 +349,7 @@ public class ApiConstants {
     public static final String MIGRATIONS = "migrations";
     public static final String MEMORY = "memory";
     public static final String MODE = "mode";
+    public static final String MOUNT_CKS_ISO_ON_VR = "mountcksisoonvr";
     public static final String MULTI_ARCH = "ismultiarch";
     public static final String NSX_MODE = "nsxmode";
     public static final String NETWORK_MODE = "networkmode";
@@ -359,6 +366,7 @@ public class ApiConstants {
     public static final String NIC_PACKED_VIRTQUEUES_ENABLED = "nicpackedvirtqueuesenabled";
     public static final String NEW_START_IP = "newstartip";
     public static final String NEW_END_IP = "newendip";
+    public static final String KUBERNETES_NODE_VERSION = "kubernetesnodeversion";
     public static final String NUM_RETRIES = "numretries";
     public static final String OFFER_HA = "offerha";
     public static final String OS_DISTRIBUTION = "osdistribution";
@@ -546,6 +554,7 @@ public class ApiConstants {
     public static final String USER_SECURITY_GROUP_LIST = "usersecuritygrouplist";
     public static final String USER_SECRET_KEY = "usersecretkey";
     public static final String USE_VIRTUAL_NETWORK = "usevirtualnetwork";
+    public static final String USE_VIRTUAL_ROUTER_IP_RESOLVER = "userouteripresolver";
     public static final String UPDATE_IN_SEQUENCE = "updateinsequence";
     public static final String VALUE = "value";
     public static final String VIRTUAL_MACHINE_ID = "virtualmachineid";
@@ -562,6 +571,12 @@ public class ApiConstants {
 
     public static final String VLAN = "vlan";
     public static final String VLAN_RANGE = "vlanrange";
+    public static final String WORKER_SERVICE_OFFERING_ID = "workerofferingid";
+    public static final String WORKER_SERVICE_OFFERING_NAME = "workerofferingname";
+    public static final String CONTROL_SERVICE_OFFERING_ID = "controlofferingid";
+    public static final String CONTROL_SERVICE_OFFERING_NAME = "controlofferingname";
+    public static final String ETCD_SERVICE_OFFERING_ID = "etcdofferingid";
+    public static final String ETCD_SERVICE_OFFERING_NAME = "etcdofferingname";
     public static final String REMOVE_VLAN = "removevlan";
     public static final String VLAN_ID = "vlanid";
     public static final String ISOLATED_PVLAN = "isolatedpvlan";
@@ -912,6 +927,7 @@ public class ApiConstants {
     public static final String SPLIT_CONNECTIONS = "splitconnections";
     public static final String FOR_VPC = "forvpc";
     public static final String FOR_NSX = "fornsx";
+    public static final String FOR_CKS = "forcks";
     public static final String NSX_SUPPORT_LB = "nsxsupportlb";
     public static final String NSX_SUPPORTS_INTERNAL_LB = "nsxsupportsinternallb";
     public static final String FOR_TUNGSTEN = "fortungsten";
@@ -1120,6 +1136,10 @@ public class ApiConstants {
     public static final String MASTER_NODES = "masternodes";
     public static final String NODE_IDS = "nodeids";
     public static final String CONTROL_NODES = "controlnodes";
+    public static final String ETCD_NODES = "etcdnodes";
+    public static final String EXTERNAL_NODES = "externalnodes";
+    public static final String IS_EXTERNAL_NODE = "isexternalnode";
+    public static final String IS_ETCD_NODE = "isetcdnode";
     public static final String MIN_SEMANTIC_VERSION = "minimumsemanticversion";
     public static final String MIN_KUBERNETES_VERSION_ID = "minimumkubernetesversionid";
     public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize";
@@ -1128,6 +1148,8 @@ public class ApiConstants {
     public static final String AUTOSCALING_ENABLED = "autoscalingenabled";
     public static final String MIN_SIZE = "minsize";
     public static final String MAX_SIZE = "maxsize";
+    public static final String NODE_TYPE_OFFERING_MAP = "nodeofferings";
+    public static final String NODE_TYPE_TEMPLATE_MAP = "nodetemplates";
 
     public static final String BOOT_TYPE = "boottype";
     public static final String BOOT_MODE = "bootmode";

api/src/main/java/org/apache/cloudstack/api/command/user/iso/DetachIsoCmd.java

@@ -104,7 +104,7 @@
 
     @Override
     public void execute() {
-        boolean result = _templateService.detachIso(virtualMachineId, isForced());
+        boolean result = _templateService.detachIso(virtualMachineId, null, isForced());
         if (result) {
             UserVm userVm = _entityMgr.findById(UserVm.class, virtualMachineId);
             UserVmResponse response = _responseGenerator.createUserVmResponse(getResponseView(), "virtualmachine", userVm).get(0);

api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java

@@ -99,6 +99,11 @@
             description = "(VMware only) true if VM deployments should preserve all the configurations defined for this template", since = "4.15.1")
     private Boolean deployAsIs;
 
+    @Parameter(name=ApiConstants.FOR_CKS,
+            type = CommandType.BOOLEAN,
+            description = "if true, the templates would be available for deploying CKS clusters", since = "4.21.0")
+    protected Boolean forCks;
+
     public String getDisplayText() {
         return StringUtils.isBlank(displayText) ? getName() : displayText;
     }
@@ -168,6 +173,10 @@
                 Boolean.TRUE.equals(deployAsIs);
     }
 
+    public boolean isForCks() {
+        return Boolean.TRUE.equals(forCks);
+    }
+
     public CPU.CPUArch getArch() {
         return CPU.CPUArch.fromType(arch);
     }

api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java

@@ -106,6 +106,11 @@
             since = "4.19.0")
     private Boolean isVnf;
 
+    @Parameter(name = ApiConstants.FOR_CKS, type = CommandType.BOOLEAN,
+            description = "list templates that can be used to deploy CKS clusters",
+            since = "4.21.0")
+    private Boolean forCks;
+
     @Parameter(name = ApiConstants.ARCH, type = CommandType.STRING,
             description = "the CPU arch of the template. Valid options are: x86_64, aarch64",
             since = "4.20")
@@ -198,6 +203,8 @@
         return isVnf;
     }
 
+    public Boolean getForCks() { return forCks; }
+
     public CPU.CPUArch getArch() {
         if (StringUtils.isBlank(arch)) {
             return null;