cks: create separate service account in project

A separate service account will be created and added in the project, if
not exist already, when a Kubernetes cluster is deployed in a project.
This account will have a role with limited API access.

Cleanup clusters on owner account cleanup, delete service account
if needed

When the owner account of k8s clusters is deleted, while its node VMs
get expunged, the cluster entry in DB remain present. This fixes the
issue by cleaning up all clusters for the account deleted.

Project k8s service account will be deleted on account cleanup or when
there is no active k8s cluster remaining

Signed-off-by: Abhishek Kumar <[email protected]>

Author: shwstppr

api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java

@@ -18,6 +18,7 @@
 
 import org.apache.cloudstack.acl.ControlledEntity;
 
+import com.cloud.user.Account;
 import com.cloud.uservm.UserVm;
 import com.cloud.utils.component.Adapter;
 
@@ -26,4 +27,5 @@ public interface KubernetesServiceHelper extends Adapter {
     ControlledEntity findByUuid(String uuid);
     ControlledEntity findByVmId(long vmId);
     void checkVmCanBeDestroyed(UserVm userVm);
+    void cleanupForAccount(Account account);
 }

engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java

@@ -16,23 +16,25 @@
 // under the License.
 package com.cloud.user.dao;
 
+import java.util.Date;
+import java.util.List;
+
 import com.cloud.user.Account;
 import com.cloud.user.AccountVO;
 import com.cloud.user.User;
 import com.cloud.utils.Pair;
 import com.cloud.utils.db.Filter;
 import com.cloud.utils.db.GenericDao;
 
-import java.util.Date;
-import java.util.List;
-
 public interface AccountDao extends GenericDao<AccountVO, Long> {
     Pair<User, Account> findUserAccountByApiKey(String apiKey);
 
     List<AccountVO> findAccountsLike(String accountName);
 
     Pair<List<AccountVO>, Integer> findAccountsLike(String accountName, Filter filter);
 
+    List<AccountVO> findAccountsByName(String accountName);
+
     List<AccountVO> findActiveAccounts(Long maxAccountId, Filter filter);
 
     List<AccountVO> findRecentlyDeletedAccounts(Long maxAccountId, Date earliestRemovedDate, Filter filter);

engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java

@@ -16,6 +16,14 @@
 // under the License.
 package com.cloud.user.dao;
 
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+
 import com.cloud.user.Account;
 import com.cloud.user.Account.State;
 import com.cloud.user.AccountVO;
@@ -30,14 +38,7 @@
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.db.SearchCriteria.Func;
 import com.cloud.utils.db.SearchCriteria.Op;
-import org.apache.commons.lang3.StringUtils;
 import com.cloud.utils.db.TransactionLegacy;
-import org.springframework.stereotype.Component;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.util.Date;
-import java.util.List;
 
 @Component
 public class AccountDaoImpl extends GenericDaoBase<AccountVO, Long> implements AccountDao {
@@ -190,6 +191,16 @@ public Pair<List<AccountVO>, Integer> findAccountsLike(String accountName, Filte
         return searchAndCount(sc, filter);
     }
 
+    @Override
+    public List<AccountVO> findAccountsByName(String accountName) {
+        SearchBuilder<AccountVO> sb = createSearchBuilder();
+        sb.and("accountName", sb.entity().getAccountName(), SearchCriteria.Op.EQ);
+        sb.done();
+        SearchCriteria<AccountVO> sc = sb.create();
+        sc.setParameters("accountName", accountName);
+        return search(sc, null);
+    }
+
     @Override
     public Account findEnabledAccount(String accountName, Long domainId) {
         SearchCriteria<AccountVO> sc = AllFieldsSearch.create("accountName", accountName);

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.kubernetes.cluster;
 
+import java.util.List;
+
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd;
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd;
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd;
@@ -34,16 +36,16 @@
 import org.apache.cloudstack.framework.config.Configurable;
 
 import com.cloud.network.Network;
+import com.cloud.user.Account;
 import com.cloud.utils.component.PluggableService;
 import com.cloud.utils.exception.CloudRuntimeException;
 
-import java.util.List;
-
 public interface KubernetesClusterService extends PluggableService, Configurable {
     static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16.0";
     static final int MIN_KUBERNETES_CLUSTER_NODE_CPU = 2;
     static final int MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE = 2048;
     static final String KUBEADMIN_ACCOUNT_NAME = "kubeadmin";
+    String PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME = "Project Kubernetes Service Role";
 
     static final ConfigKey<Boolean> KubernetesServiceEnabled = new ConfigKey<Boolean>("Advanced", Boolean.class,
             "cloud.kubernetes.service.enabled",
@@ -125,4 +127,6 @@ public interface KubernetesClusterService extends PluggableService, Configurable
     List<RemoveVirtualMachinesFromKubernetesClusterResponse> removeVmsFromCluster(RemoveVirtualMachinesFromKubernetesClusterCmd cmd);
 
     boolean isDirectAccess(Network network);
+
+    void cleanupForAccount(Account account);
 }

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java

@@ -32,6 +32,7 @@
 import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
 import com.cloud.kubernetes.version.KubernetesSupportedVersion;
 import com.cloud.kubernetes.version.KubernetesVersionEventTypes;
+import com.cloud.user.Account;
 import com.cloud.uservm.UserVm;
 import com.cloud.utils.component.AdapterBase;
 import com.cloud.utils.exception.CloudRuntimeException;
@@ -50,6 +51,8 @@ public class KubernetesServiceHelperImpl extends AdapterBase implements Kubernet
     private KubernetesClusterDao kubernetesClusterDao;
     @Inject
     private KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
+    @Inject
+    KubernetesClusterService kubernetesClusterService;
 
     protected void setEventTypeEntityDetails(Class<?> eventTypeDefinedClass, Class<?> entityClass) {
         Field[] declaredFields = eventTypeDefinedClass.getDeclaredFields();
@@ -106,6 +109,11 @@ public void checkVmCanBeDestroyed(UserVm userVm) {
         throw new CloudRuntimeException(msg);
     }
 
+    @Override
+    public void cleanupForAccount(Account account) {
+        kubernetesClusterService.cleanupForAccount(account);
+    }
+
     @Override
     public String getConfigComponentName() {
         return KubernetesServiceHelper.class.getSimpleName();

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java

@@ -25,8 +25,8 @@
 
 public interface KubernetesClusterDao extends GenericDao<KubernetesClusterVO, Long>,
         StateDao<KubernetesCluster.State, KubernetesCluster.Event, KubernetesCluster> {
-
-    List<KubernetesClusterVO> listByAccount(long accountId);
+    List<KubernetesClusterVO> listForCleanupByAccount(long accountId);
+    int countNotForGCByAccount(long accountId);
     List<KubernetesClusterVO> findKubernetesClustersToGarbageCollect();
     List<KubernetesClusterVO> findManagedKubernetesClustersInState(KubernetesCluster.State state);
     List<KubernetesClusterVO> listByNetworkId(long networkId);

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java

@@ -30,16 +30,25 @@
 @Component
 public class KubernetesClusterDaoImpl extends GenericDaoBase<KubernetesClusterVO, Long> implements KubernetesClusterDao {
 
-    private final SearchBuilder<KubernetesClusterVO> AccountIdSearch;
+    private final SearchBuilder<KubernetesClusterVO> CleanupAccountIdSearch;
+    private final SearchBuilder<KubernetesClusterVO> NotForGCByAccountIDCount;
     private final SearchBuilder<KubernetesClusterVO> GarbageCollectedSearch;
     private final SearchBuilder<KubernetesClusterVO> ManagedStateSearch;
     private final SearchBuilder<KubernetesClusterVO> SameNetworkSearch;
     private final SearchBuilder<KubernetesClusterVO> KubernetesVersionSearch;
 
     public KubernetesClusterDaoImpl() {
-        AccountIdSearch = createSearchBuilder();
-        AccountIdSearch.and("account", AccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
-        AccountIdSearch.done();
+        CleanupAccountIdSearch = createSearchBuilder();
+        CleanupAccountIdSearch.and("account", CleanupAccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
+        CleanupAccountIdSearch.and("cluster_type", CleanupAccountIdSearch.entity().getClusterType(), SearchCriteria.Op.EQ);
+        CleanupAccountIdSearch.done();
+
+        NotForGCByAccountIDCount = createSearchBuilder();
+        NotForGCByAccountIDCount.and("gc", NotForGCByAccountIDCount.entity().isCheckForGc(), SearchCriteria.Op.EQ);
+        NotForGCByAccountIDCount.and("account", NotForGCByAccountIDCount.entity().getAccountId(), SearchCriteria.Op.EQ);
+        NotForGCByAccountIDCount.and("cluster_type", NotForGCByAccountIDCount.entity().getClusterType(), SearchCriteria.Op.EQ);
+        NotForGCByAccountIDCount.select(null, SearchCriteria.Func.COUNT, null);
+        NotForGCByAccountIDCount.done();
 
         GarbageCollectedSearch = createSearchBuilder();
         GarbageCollectedSearch.and("gc", GarbageCollectedSearch.entity().isCheckForGc(), SearchCriteria.Op.EQ);
@@ -62,10 +71,20 @@ public KubernetesClusterDaoImpl() {
     }
 
     @Override
-    public List<KubernetesClusterVO> listByAccount(long accountId) {
-        SearchCriteria<KubernetesClusterVO> sc = AccountIdSearch.create();
+    public List<KubernetesClusterVO> listForCleanupByAccount(long accountId) {
+        SearchCriteria<KubernetesClusterVO> sc = CleanupAccountIdSearch.create();
+        sc.setParameters("cluster_type", KubernetesCluster.ClusterType.CloudManaged);
+        sc.setParameters("account", accountId);
+        return listBy(sc);
+    }
+
+    @Override
+    public int countNotForGCByAccount(long accountId) {
+        SearchCriteria<KubernetesClusterVO> sc = NotForGCByAccountIDCount.create();
+        sc.setParameters("cluster_type", KubernetesCluster.ClusterType.CloudManaged);
         sc.setParameters("account", accountId);
-        return listBy(sc, null);
+        sc.setParameters("gc", false);
+        return getCount(sc);
     }
 
     @Override

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java

@@ -120,6 +120,7 @@
 import com.cloud.exception.PermissionDeniedException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.host.dao.HostDao;
+import com.cloud.kubernetes.cluster.KubernetesServiceHelper;
 import com.cloud.network.IpAddress;
 import com.cloud.network.IpAddressManager;
 import com.cloud.network.Network;
@@ -887,6 +888,16 @@ public boolean deleteAccount(AccountVO account, long callerUserId, Account calle
         return cleanupAccount(account, callerUserId, caller);
     }
 
+    protected void cleanupPluginsResourcesIfNeeded(Account account) {
+        try {
+            KubernetesServiceHelper kubernetesServiceHelper =
+                    ComponentContext.getDelegateComponentOfType(KubernetesServiceHelper.class);
+            kubernetesServiceHelper.cleanupForAccount(account);
+        } catch (NoSuchBeanDefinitionException ignored) {
+            logger.debug("No KubernetesServiceHelper bean found");
+        }
+    }
+
     protected boolean cleanupAccount(AccountVO account, long callerUserId, Account caller) {
         long accountId = account.getId();
         boolean accountCleanupNeeded = false;
@@ -968,6 +979,8 @@ protected boolean cleanupAccount(AccountVO account, long callerUserId, Account c
                 }
             }
 
+            cleanupPluginsResourcesIfNeeded(account);
+
             // Destroy the account's VMs
             List<UserVmVO> vms = _userVmDao.listByAccountId(accountId);
             if (logger.isDebugEnabled()) {