Skip to content

Commit 36a02b8

Browse files
nvazqueznvazquez
committed
Address review comments: plan CKS cluster deployment based on the node type
1 parent 89effc7 commit 36a02b8

File tree

5 files changed

+60
-16
lines changed

5 files changed

+60
-16
lines changed

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1037,7 +1037,8 @@ protected void validateServiceOfferingForNode(Map<String, Long> map,
10371037
Long serviceOfferingId = map.getOrDefault(key, defaultServiceOfferingId);
10381038
ServiceOffering serviceOffering = serviceOfferingId != null ? serviceOfferingDao.findById(serviceOfferingId) : null;
10391039
if (serviceOffering == null) {
1040-
throw new InvalidParameterValueException("No service offering found with ID: " + serviceOfferingId);
1040+
throw new InvalidParameterValueException("When serviceofferingid is not specified, " +
1041+
"service offerings for each node type must be specified in the nodeofferings parameter.");
10411042
}
10421043
try {
10431044
validateServiceOffering(serviceOffering, clusterKubernetesVersion);
@@ -1294,7 +1295,7 @@ protected void checkServiceOfferingForNodesScale(ServiceOffering serviceOffering
12941295

12951296
protected boolean isAnyNodeOfferingEmpty(Map<String, Long> map) {
12961297
if (MapUtils.isEmpty(map)) {
1297-
return false;
1298+
return true;
12981299
}
12991300
return map.values().stream().anyMatch(Objects::isNull);
13001301
}
@@ -1479,13 +1480,12 @@ public KubernetesCluster createManagedKubernetesCluster(CreateKubernetesClusterC
14791480
final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback<KubernetesClusterVO>() {
14801481
@Override
14811482
public KubernetesClusterVO doInTransaction(TransactionStatus status) {
1482-
final ServiceOffering defaultServiceOffering = serviceOfferingDao.findById(defaultServiceOfferingId);
14831483
Pair<Long, Long> capacityPair = calculateClusterCapacity(serviceOfferingNodeTypeMap, nodeTypeCount, defaultServiceOfferingId);
14841484
final long cores = capacityPair.first();
14851485
final long memory = capacityPair.second();
14861486

14871487
KubernetesClusterVO newCluster = new KubernetesClusterVO(cmd.getName(), cmd.getDisplayName(), zone.getId(), clusterKubernetesVersion.getId(),
1488-
defaultServiceOffering.getId(), Objects.nonNull(finalTemplate) ? finalTemplate.getId() : null,
1488+
defaultServiceOfferingId, Objects.nonNull(finalTemplate) ? finalTemplate.getId() : null,
14891489
defaultNetwork.getId(), owner.getDomainId(), owner.getAccountId(), controlNodeCount, clusterSize,
14901490
KubernetesCluster.State.Created, cmd.getSSHKeyPairName(), cores, memory,
14911491
cmd.getNodeRootDiskSize(), "", KubernetesCluster.ClusterType.CloudManaged);

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,10 @@
128128
import com.cloud.vm.dao.UserVmDao;
129129
import com.cloud.vm.dao.UserVmDetailsDao;
130130

131+
import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL;
132+
import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD;
133+
import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER;
134+
131135

132136
public class KubernetesClusterActionWorker {
133137

@@ -144,6 +148,8 @@ public class KubernetesClusterActionWorker {
144148

145149
protected Logger logger = LogManager.getLogger(getClass());
146150

151+
protected final static List<KubernetesClusterNodeType> CLUSTER_NODES_TYPES_LIST = Arrays.asList(WORKER, CONTROL, ETCD);
152+
147153
protected StateMachine2<KubernetesCluster.State, KubernetesCluster.Event, KubernetesCluster> _stateMachine = KubernetesCluster.State.getStateMachine();
148154

149155
@Inject

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java‎

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -259,13 +259,35 @@ protected DeployDestination plan(final long nodesCount, final DataCenter zone, f
259259
throw new InsufficientServerCapacityException(msg, DataCenter.class, zone.getId());
260260
}
261261

262-
protected DeployDestination plan(Long domainId, Long accountId, Hypervisor.HypervisorType hypervisorType) throws InsufficientServerCapacityException {
263-
ServiceOffering offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId());
262+
/**
263+
* Plan Kubernetes Cluster Deployment
264+
* @return a map of DeployDestination per node type
265+
*/
266+
protected Map<String, DeployDestination> planKubernetesCluster(Long domainId, Long accountId, Hypervisor.HypervisorType hypervisorType) throws InsufficientServerCapacityException {
267+
Map<String, DeployDestination> destinationMap = new HashMap<>();
264268
DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId());
265269
if (logger.isDebugEnabled()) {
266270
logger.debug(String.format("Checking deployment destination for Kubernetes cluster : %s in zone : %s", kubernetesCluster.getName(), zone.getName()));
267271
}
268-
return plan(kubernetesCluster.getTotalNodeCount(), zone, offering, domainId, accountId, hypervisorType);
272+
long controlNodeCount = kubernetesCluster.getControlNodeCount();
273+
long clusterSize = kubernetesCluster.getNodeCount();
274+
long etcdNodes = kubernetesCluster.getEtcdNodeCount();
275+
Map<String, Long> nodeTypeCount = Map.of(WORKER.name(), clusterSize,
276+
CONTROL.name(), controlNodeCount, ETCD.name(), etcdNodes);
277+
278+
for (KubernetesClusterNodeType nodeType : CLUSTER_NODES_TYPES_LIST) {
279+
Long nodes = nodeTypeCount.getOrDefault(nodeType.name(), kubernetesCluster.getServiceOfferingId());
280+
if (nodes == null || nodes == 0) {
281+
continue;
282+
}
283+
ServiceOffering nodeOffering = getServiceOfferingForNodeTypeOnCluster(nodeType, kubernetesCluster);
284+
if (logger.isDebugEnabled()) {
285+
logger.debug(String.format("Checking deployment destination for %s nodes on Kubernetes cluster : %s in zone : %s", nodeType.name(), kubernetesCluster.getName(), zone.getName()));
286+
}
287+
DeployDestination planForNodeType = plan(nodes, zone, nodeOffering, domainId, accountId, hypervisorType);
288+
destinationMap.put(nodeType.name(), planForNodeType);
289+
}
290+
return destinationMap;
269291
}
270292

271293
protected void resizeNodeVolume(final UserVm vm) throws ManagementServerException {
@@ -288,7 +310,7 @@ protected void resizeNodeVolume(final UserVm vm) throws ManagementServerExceptio
288310
}
289311
}
290312

291-
protected void startKubernetesVM(final UserVm vm, final Long domainId, final Long accountId) throws ManagementServerException {
313+
protected void startKubernetesVM(final UserVm vm, final Long domainId, final Long accountId, KubernetesClusterNodeType nodeType) throws ManagementServerException {
292314
CallContext vmContext = null;
293315
if (!ApiCommandResourceType.VirtualMachine.equals(CallContext.current().getEventResourceType())); {
294316
vmContext = CallContext.register(CallContext.current(), ApiCommandResourceType.VirtualMachine);
@@ -298,7 +320,8 @@ protected void startKubernetesVM(final UserVm vm, final Long domainId, final Lon
298320
if (Objects.nonNull(domainId) && !listDedicatedHostsInDomain(domainId).isEmpty()) {
299321
DeployDestination dest = null;
300322
try {
301-
dest = plan(domainId, accountId, vm.getHypervisorType());
323+
Map<String, DeployDestination> destinationMap = planKubernetesCluster(domainId, accountId, vm.getHypervisorType());
324+
dest = destinationMap.get(nodeType.name());
302325
} catch (InsufficientCapacityException e) {
303326
logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e);
304327
}
@@ -341,7 +364,7 @@ protected List<UserVm> provisionKubernetesClusterNodeVms(final long nodeCount, f
341364
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
342365
resizeNodeVolume(vm);
343366
}
344-
startKubernetesVM(vm, domainId, accountId);
367+
startKubernetesVM(vm, domainId, accountId, WORKER);
345368
vm = userVmDao.findById(vm.getId());
346369
if (vm == null) {
347370
throw new ManagementServerException(String.format("Failed to provision worker VM for Kubernetes cluster : %s", kubernetesCluster.getName()));

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import com.cloud.exception.InvalidParameterValueException;
3535
import com.cloud.exception.NetworkRuleConflictException;
3636
import com.cloud.exception.PermissionDeniedException;
37+
import com.cloud.kubernetes.cluster.KubernetesServiceHelper;
3738
import com.cloud.network.vpc.NetworkACL;
3839
import com.cloud.storage.VMTemplateVO;
3940
import com.cloud.user.UserDataVO;
@@ -85,6 +86,7 @@
8586

8687
import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL;
8788
import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD;
89+
import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER;
8890

8991
public class KubernetesClusterStartWorker extends KubernetesClusterResourceModifierActionWorker {
9092

@@ -515,7 +517,7 @@ private Pair<UserVm, String> provisionKubernetesClusterControlVm(final Network n
515517
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
516518
resizeNodeVolume(k8sControlVM);
517519
}
518-
startKubernetesVM(k8sControlVM, domainId, accountId);
520+
startKubernetesVM(k8sControlVM, domainId, accountId, CONTROL);
519521
k8sControlVM = userVmDao.findById(k8sControlVM.getId());
520522
if (k8sControlVM == null) {
521523
throw new ManagementServerException(String.format("Failed to provision control VM for Kubernetes cluster : %s" , kubernetesCluster.getName()));
@@ -538,7 +540,7 @@ private List<UserVm> provisionKubernetesClusterAdditionalControlVms(final String
538540
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
539541
resizeNodeVolume(vm);
540542
}
541-
startKubernetesVM(vm, domainId, accountId);
543+
startKubernetesVM(vm, domainId, accountId, CONTROL);
542544
vm = userVmDao.findById(vm.getId());
543545
if (vm == null) {
544546
throw new ManagementServerException(String.format("Failed to provision additional control VM for Kubernetes cluster : %s" , kubernetesCluster.getName()));
@@ -560,7 +562,7 @@ private Pair<List<UserVm>, List<Network.IpAddresses>> provisionEtcdCluster(final
560562
for (int i = 0; i < kubernetesCluster.getEtcdNodeCount(); i++) {
561563
UserVm vm = createEtcdNode(etcdNodeGuestIps, etcdHostnames, i, domainId, accountId);
562564
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false, true, true);
563-
startKubernetesVM(vm, domainId, accountId);
565+
startKubernetesVM(vm, domainId, accountId, ETCD);
564566
vm = userVmDao.findById(vm.getId());
565567
if (vm == null) {
566568
throw new ManagementServerException(String.format("Failed to provision additional control VM for Kubernetes cluster : %s" , kubernetesCluster.getName()));
@@ -669,7 +671,9 @@ private void startKubernetesClusterVMs(Long domainId, Long accountId) {
669671
}
670672
try {
671673
resizeNodeVolume(vm);
672-
startKubernetesVM(vm, domainId, accountId);
674+
KubernetesClusterVmMapVO map = kubernetesClusterVmMapDao.findByVmId(vm.getId());
675+
KubernetesServiceHelper.KubernetesClusterNodeType nodeType = getNodeTypeFromClusterVMMapRecord(map);
676+
startKubernetesVM(vm, domainId, accountId, nodeType);
673677
} catch (ManagementServerException ex) {
674678
logger.warn(String.format("Failed to start VM : %s in Kubernetes cluster : %s due to ", vm.getDisplayName(), kubernetesCluster.getName()) + ex);
675679
// don't bail out here. proceed further to stop the reset of the VM's
@@ -683,6 +687,16 @@ private void startKubernetesClusterVMs(Long domainId, Long accountId) {
683687
}
684688
}
685689

690+
private KubernetesServiceHelper.KubernetesClusterNodeType getNodeTypeFromClusterVMMapRecord(KubernetesClusterVmMapVO map) {
691+
if (map.isControlNode()) {
692+
return CONTROL;
693+
} else if (map.isEtcdNode()) {
694+
return ETCD;
695+
} else {
696+
return WORKER;
697+
}
698+
}
699+
686700
private boolean isKubernetesClusterKubeConfigAvailable(final long timeoutTime) {
687701
if (StringUtils.isEmpty(publicIpAddress)) {
688702
KubernetesClusterDetailsVO kubeConfigDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData");
@@ -733,7 +747,8 @@ public boolean startKubernetesClusterOnCreate(Long domainId, Long accountId, Lon
733747
DeployDestination dest = null;
734748
try {
735749
VMTemplateVO clusterTemplate = templateDao.findById(kubernetesCluster.getTemplateId());
736-
dest = plan(domainId, accountId, clusterTemplate.getHypervisorType());
750+
Map<String, DeployDestination> destinationMap = planKubernetesCluster(domainId, accountId, clusterTemplate.getHypervisorType());
751+
dest = destinationMap.get(WORKER.name());
737752
} catch (InsufficientCapacityException e) {
738753
logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e);
739754
}

‎plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -370,7 +370,7 @@ public void testClusterCapacity() {
370370

371371
@Test
372372
public void testIsAnyNodeOfferingEmptyNullMap() {
373-
Assert.assertFalse(kubernetesClusterManager.isAnyNodeOfferingEmpty(null));
373+
Assert.assertTrue(kubernetesClusterManager.isAnyNodeOfferingEmpty(null));
374374
}
375375

376376
@Test

0 commit comments

Comments
 (0)