Skip to content

Commit 05b9b6e

Browse files
DaanHooglandDaanHoogland
committed
Merge branch '4.18' into main
2 parents cfd6bff + b79e393 commit 05b9b6e

File tree

13 files changed

+188
-86
lines changed

13 files changed

+188
-86
lines changed

‎api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,9 @@ public void setUrl(String url) {
177177
}
178178

179179
public Long getZoneId() {
180+
if (zoneId == null || zoneId == -1) {
181+
return null;
182+
}
180183
return zoneId;
181184
}
182185

@@ -220,6 +223,10 @@ public boolean isDirectDownload() {
220223
return directDownload == null ? false : directDownload;
221224
}
222225

226+
public void setDirectDownload(Boolean directDownload) {
227+
this.directDownload = directDownload;
228+
}
229+
223230
public boolean isPasswordEnabled() {
224231
return passwordEnabled == null ? false : passwordEnabled;
225232
}

‎engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java‎

Lines changed: 26 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,10 @@
2020

2121
import java.util.ArrayList;
2222
import java.util.List;
23+
import java.util.stream.Collectors;
2324

2425
import javax.inject.Inject;
2526

26-
import com.cloud.hypervisor.Hypervisor;
27-
import com.cloud.utils.exception.CloudRuntimeException;
2827
import org.apache.cloudstack.direct.download.DirectDownloadManager;
2928
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
3029
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
@@ -36,18 +35,21 @@
3635
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
3736
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
3837
import org.apache.cloudstack.storage.image.store.TemplateObject;
38+
import org.apache.commons.collections.CollectionUtils;
3939
import org.apache.commons.lang3.StringUtils;
4040
import org.apache.log4j.Logger;
4141
import org.springframework.stereotype.Component;
4242

4343
import com.cloud.host.HostVO;
4444
import com.cloud.host.dao.HostDao;
45+
import com.cloud.hypervisor.Hypervisor;
4546
import com.cloud.storage.DataStoreRole;
4647
import com.cloud.storage.VMTemplateStoragePoolVO;
4748
import com.cloud.storage.VMTemplateStorageResourceAssoc;
4849
import com.cloud.storage.VMTemplateVO;
4950
import com.cloud.storage.dao.VMTemplateDao;
5051
import com.cloud.storage.dao.VMTemplatePoolDao;
52+
import com.cloud.utils.exception.CloudRuntimeException;
5153

5254
@Component
5355
public class TemplateDataFactoryImpl implements TemplateDataFactory {
@@ -203,12 +205,7 @@ public List<TemplateInfo> listTemplateOnCache(long templateId) {
203205
* Given existing spool refs, return one pool id existing on pools and refs
204206
*/
205207
private Long getOneMatchingPoolIdFromRefs(List<VMTemplateStoragePoolVO> existingRefs, List<StoragePoolVO> pools) {
206-
if (pools.isEmpty()) {
207-
throw new CloudRuntimeException("No storage pools found");
208-
}
209-
if (existingRefs.isEmpty()) {
210-
return pools.get(0).getId();
211-
} else {
208+
if (!existingRefs.isEmpty()) {
212209
for (VMTemplateStoragePoolVO ref : existingRefs) {
213210
for (StoragePoolVO p : pools) {
214211
if (ref.getPoolId() == p.getId()) {
@@ -217,45 +214,51 @@ private Long getOneMatchingPoolIdFromRefs(List<VMTemplateStoragePoolVO> existing
217214
}
218215
}
219216
}
220-
return null;
217+
return pools.get(0).getId();
221218
}
222219

223220
/**
224-
* Retrieve storage pools with scope = cluster or zone matching clusterId or dataCenterId depending on their scope
221+
* Retrieve storage pools with scope = cluster or zone or local matching clusterId or dataCenterId or hostId depending on their scope
225222
*/
226-
private List<StoragePoolVO> getStoragePoolsFromClusterOrZone(Long clusterId, long dataCenterId, Hypervisor.HypervisorType hypervisorType) {
223+
private List<StoragePoolVO> getStoragePoolsForScope(long dataCenterId, Long clusterId, long hostId, Hypervisor.HypervisorType hypervisorType) {
227224
List<StoragePoolVO> pools = new ArrayList<>();
228225
if (clusterId != null) {
229226
List<StoragePoolVO> clusterPools = primaryDataStoreDao.listPoolsByCluster(clusterId);
227+
clusterPools = clusterPools.stream().filter(p -> !p.isLocal()).collect(Collectors.toList());
230228
pools.addAll(clusterPools);
231229
}
232230
List<StoragePoolVO> zonePools = primaryDataStoreDao.findZoneWideStoragePoolsByHypervisor(dataCenterId, hypervisorType);
233231
pools.addAll(zonePools);
232+
List<StoragePoolVO> localPools = primaryDataStoreDao.findLocalStoragePoolsByHostAndTags(hostId, null);
233+
pools.addAll(localPools);
234234
return pools;
235235
}
236236

237+
protected Long getBypassedTemplateExistingOrNewPoolId(VMTemplateVO templateVO, Long hostId) {
238+
HostVO host = hostDao.findById(hostId);
239+
List<StoragePoolVO> pools = getStoragePoolsForScope(host.getDataCenterId(), host.getClusterId(), hostId, host.getHypervisorType());
240+
if (CollectionUtils.isEmpty(pools)) {
241+
throw new CloudRuntimeException(String.format("No storage pool found to download template: %s", templateVO.getName()));
242+
}
243+
List<VMTemplateStoragePoolVO> existingRefs = templatePoolDao.listByTemplateId(templateVO.getId());
244+
return getOneMatchingPoolIdFromRefs(existingRefs, pools);
245+
}
246+
237247
@Override
238248
public TemplateInfo getReadyBypassedTemplateOnPrimaryStore(long templateId, Long poolId, Long hostId) {
239249
VMTemplateVO templateVO = imageDataDao.findById(templateId);
240250
if (templateVO == null || !templateVO.isDirectDownload()) {
241251
return null;
242252
}
243-
Long pool = poolId;
253+
Long templatePoolId = poolId;
244254
if (poolId == null) {
245-
//Get ISO from existing pool ref
246-
HostVO host = hostDao.findById(hostId);
247-
List<StoragePoolVO> pools = getStoragePoolsFromClusterOrZone(host.getClusterId(), host.getDataCenterId(), host.getHypervisorType());
248-
List<VMTemplateStoragePoolVO> existingRefs = templatePoolDao.listByTemplateId(templateId);
249-
pool = getOneMatchingPoolIdFromRefs(existingRefs, pools);
250-
}
251-
if (pool == null) {
252-
throw new CloudRuntimeException("No storage pool found where to download template: " + templateId);
255+
templatePoolId = getBypassedTemplateExistingOrNewPoolId(templateVO, hostId);
253256
}
254-
VMTemplateStoragePoolVO spoolRef = templatePoolDao.findByPoolTemplate(pool, templateId, null);
257+
VMTemplateStoragePoolVO spoolRef = templatePoolDao.findByPoolTemplate(templatePoolId, templateId, null);
255258
if (spoolRef == null) {
256-
directDownloadManager.downloadTemplate(templateId, pool, hostId);
259+
directDownloadManager.downloadTemplate(templateId, templatePoolId, hostId);
257260
}
258-
DataStore store = storeMgr.getDataStore(pool, DataStoreRole.Primary);
261+
DataStore store = storeMgr.getDataStore(templatePoolId, DataStoreRole.Primary);
259262
return this.getTemplate(templateId, store);
260263
}
261264

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -294,12 +294,14 @@ public KVMStoragePool getStoragePoolByURI(String uri) {
294294
String uuid = null;
295295
String sourceHost = "";
296296
StoragePoolType protocol = null;
297-
if (storageUri.getScheme().equalsIgnoreCase("nfs") || storageUri.getScheme().equalsIgnoreCase("NetworkFilesystem")) {
297+
final String scheme = storageUri.getScheme().toLowerCase();
298+
List<String> acceptedSchemes = List.of("nfs", "networkfilesystem", "filesystem");
299+
if (acceptedSchemes.contains(scheme)) {
298300
sourcePath = storageUri.getPath();
299301
sourcePath = sourcePath.replace("//", "/");
300302
sourceHost = storageUri.getHost();
301303
uuid = UUID.nameUUIDFromBytes(new String(sourceHost + sourcePath).getBytes()).toString();
302-
protocol = StoragePoolType.NetworkFilesystem;
304+
protocol = scheme.equals("filesystem") ? StoragePoolType.Filesystem: StoragePoolType.NetworkFilesystem;
303305
}
304306

305307
// secondary storage registers itself through here

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java‎

Lines changed: 35 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -25,23 +25,26 @@
2525
import java.io.FileNotFoundException;
2626
import java.io.FileOutputStream;
2727
import java.io.IOException;
28+
import java.nio.file.Files;
29+
import java.nio.file.Paths;
2830
import java.text.DateFormat;
2931
import java.text.SimpleDateFormat;
3032
import java.util.Arrays;
3133
import java.util.Date;
3234
import java.util.HashMap;
35+
import java.util.HashSet;
3336
import java.util.List;
3437
import java.util.Map;
38+
import java.util.Set;
3539
import java.util.UUID;
40+
import java.util.stream.Collectors;
3641

3742
import javax.naming.ConfigurationException;
3843

39-
import org.apache.cloudstack.direct.download.DirectDownloadHelper;
40-
import org.apache.cloudstack.direct.download.DirectTemplateDownloader;
41-
import com.cloud.storage.ScopeType;
42-
import com.cloud.storage.Volume;
4344
import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
4445
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
46+
import org.apache.cloudstack.direct.download.DirectDownloadHelper;
47+
import org.apache.cloudstack.direct.download.DirectTemplateDownloader;
4548
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
4649
import org.apache.cloudstack.storage.command.AttachAnswer;
4750
import org.apache.cloudstack.storage.command.AttachCommand;
@@ -71,7 +74,10 @@
7174
import org.apache.cloudstack.utils.qemu.QemuObject;
7275
import org.apache.commons.collections.MapUtils;
7376
import org.apache.commons.io.FileUtils;
74-
77+
import org.apache.commons.lang3.BooleanUtils;
78+
import org.apache.commons.lang3.StringUtils;
79+
import org.apache.commons.lang3.builder.ToStringBuilder;
80+
import org.apache.commons.lang3.builder.ToStringStyle;
7581
import org.apache.log4j.Logger;
7682
import org.libvirt.Connect;
7783
import org.libvirt.Domain;
@@ -110,9 +116,11 @@
110116
import com.cloud.hypervisor.kvm.resource.wrapper.LibvirtUtilitiesHelper;
111117
import com.cloud.storage.JavaStorageLayer;
112118
import com.cloud.storage.MigrationOptions;
119+
import com.cloud.storage.ScopeType;
113120
import com.cloud.storage.Storage.ImageFormat;
114121
import com.cloud.storage.Storage.StoragePoolType;
115122
import com.cloud.storage.StorageLayer;
123+
import com.cloud.storage.Volume;
116124
import com.cloud.storage.resource.StorageProcessor;
117125
import com.cloud.storage.template.Processor;
118126
import com.cloud.storage.template.Processor.FormatInfo;
@@ -126,16 +134,6 @@
126134
import com.cloud.utils.storage.S3.S3Utils;
127135
import com.cloud.vm.VmDetailConstants;
128136

129-
import java.nio.file.Files;
130-
import java.nio.file.Paths;
131-
import java.util.HashSet;
132-
import java.util.Set;
133-
import java.util.stream.Collectors;
134-
import org.apache.commons.lang3.BooleanUtils;
135-
import org.apache.commons.lang3.StringUtils;
136-
import org.apache.commons.lang3.builder.ToStringBuilder;
137-
import org.apache.commons.lang3.builder.ToStringStyle;
138-
139137
public class KVMStorageProcessor implements StorageProcessor {
140138
private static final Logger s_logger = Logger.getLogger(KVMStorageProcessor.class);
141139
private final KVMStoragePoolManager storagePoolMgr;
@@ -1074,16 +1072,23 @@ private void deleteSnapshotOnPrimary(final CopyCommand cmd, final SnapshotObject
10741072
s_logger.debug(String.format("This backup is temporary, not deleting snapshot [%s] on primary storage [%s]", snapshotPath, primaryPool.getUuid()));
10751073
}
10761074
}
1077-
protected synchronized void attachOrDetachISO(final Connect conn, final String vmName, String isoPath, final boolean isAttach, Map<String, String> params) throws
1075+
1076+
protected synchronized void attachOrDetachISO(final Connect conn, final String vmName, String isoPath, final boolean isAttach, Map<String, String> params, DataStoreTO store) throws
10781077
LibvirtException, InternalErrorException {
10791078
DiskDef iso = new DiskDef();
10801079
boolean isUefiEnabled = MapUtils.isNotEmpty(params) && params.containsKey("UEFI");
10811080
if (isoPath != null && isAttach) {
10821081
final int index = isoPath.lastIndexOf("/");
10831082
final String path = isoPath.substring(0, index);
10841083
final String name = isoPath.substring(index + 1);
1085-
final KVMStoragePool secondaryPool = storagePoolMgr.getStoragePoolByURI(path);
1086-
final KVMPhysicalDisk isoVol = secondaryPool.getPhysicalDisk(name);
1084+
KVMStoragePool storagePool;
1085+
if (store instanceof PrimaryDataStoreTO) {
1086+
PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO)store;
1087+
storagePool = storagePoolMgr.getStoragePool(primaryDataStoreTO.getPoolType(), store.getUuid());
1088+
} else {
1089+
storagePool = storagePoolMgr.getStoragePoolByURI(path);
1090+
}
1091+
final KVMPhysicalDisk isoVol = storagePool.getPhysicalDisk(name);
10871092
isoPath = isoVol.getPath();
10881093

10891094
iso.defISODisk(isoPath, isUefiEnabled);
@@ -1112,7 +1117,7 @@ public Answer attachIso(final AttachCommand cmd) {
11121117
try {
11131118
String dataStoreUrl = getDataStoreUrlFromStore(store);
11141119
final Connect conn = LibvirtConnection.getConnectionByVmName(cmd.getVmName());
1115-
attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), true, cmd.getControllerInfo());
1120+
attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), true, cmd.getControllerInfo(), store);
11161121
} catch (final LibvirtException e) {
11171122
return new Answer(cmd, false, e.toString());
11181123
} catch (final InternalErrorException e) {
@@ -1133,7 +1138,7 @@ public Answer dettachIso(final DettachCommand cmd) {
11331138
try {
11341139
String dataStoreUrl = getDataStoreUrlFromStore(store);
11351140
final Connect conn = LibvirtConnection.getConnectionByVmName(cmd.getVmName());
1136-
attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), false, cmd.getParams());
1141+
attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), false, cmd.getParams(), store);
11371142
} catch (final LibvirtException e) {
11381143
return new Answer(cmd, false, e.toString());
11391144
} catch (final InternalErrorException e) {
@@ -1149,19 +1154,25 @@ public Answer dettachIso(final DettachCommand cmd) {
11491154
* Return data store URL from store
11501155
*/
11511156
private String getDataStoreUrlFromStore(DataStoreTO store) {
1152-
if (!(store instanceof NfsTO) && (!(store instanceof PrimaryDataStoreTO) ||
1153-
store instanceof PrimaryDataStoreTO && !((PrimaryDataStoreTO) store).getPoolType().equals(StoragePoolType.NetworkFilesystem))) {
1157+
List<StoragePoolType> supportedPoolType = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.Filesystem);
1158+
if (!(store instanceof NfsTO) && (!(store instanceof PrimaryDataStoreTO) || !supportedPoolType.contains(((PrimaryDataStoreTO) store).getPoolType()))) {
1159+
s_logger.error(String.format("Unsupported protocol, store: %s", store.getUuid()));
11541160
throw new InvalidParameterValueException("unsupported protocol");
11551161
}
11561162

11571163
if (store instanceof NfsTO) {
11581164
NfsTO nfsStore = (NfsTO)store;
11591165
return nfsStore.getUrl();
1160-
} else if (store instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO) store).getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
1166+
} else if (store instanceof PrimaryDataStoreTO) {
11611167
//In order to support directly downloaded ISOs
1168+
StoragePoolType poolType = ((PrimaryDataStoreTO)store).getPoolType();
11621169
String psHost = ((PrimaryDataStoreTO) store).getHost();
11631170
String psPath = ((PrimaryDataStoreTO) store).getPath();
1164-
return "nfs://" + psHost + File.separator + psPath;
1171+
if (StoragePoolType.NetworkFilesystem.equals(poolType)) {
1172+
return "nfs://" + psHost + File.separator + psPath;
1173+
} else if (StoragePoolType.Filesystem.equals(poolType)) {
1174+
return StoragePoolType.Filesystem.toString().toLowerCase() + "://" + psHost + File.separator + psPath;
1175+
}
11651176
}
11661177
return store.getUrl();
11671178
}

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@
128128
import com.cloud.network.dao.NetworkDao;
129129
import com.cloud.network.dao.NetworkVO;
130130
import com.cloud.network.dao.PhysicalNetworkDao;
131+
import com.cloud.network.router.NetworkHelper;
131132
import com.cloud.network.rules.FirewallRule;
132133
import com.cloud.network.rules.FirewallRuleVO;
133134
import com.cloud.network.security.SecurityGroupManager;
@@ -253,6 +254,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
253254
private SecurityGroupManager securityGroupManager;
254255
@Inject
255256
public SecurityGroupService securityGroupService;
257+
@Inject
258+
public NetworkHelper networkHelper;
256259

257260
@Inject
258261
private UserVmService userVmService;
@@ -360,8 +363,12 @@ private IpAddress getSourceNatIp(Network network) {
360363

361364
public VMTemplateVO getKubernetesServiceTemplate(DataCenter dataCenter, Hypervisor.HypervisorType hypervisorType) {
362365
VMTemplateVO template = templateDao.findSystemVMReadyTemplate(dataCenter.getId(), hypervisorType);
366+
if (DataCenter.Type.Edge.equals(dataCenter.getType()) && template != null && !template.isDirectDownload()) {
367+
LOGGER.debug(String.format("Template %s can not be used for edge zone %s", template, dataCenter));
368+
template = templateDao.findRoutingTemplate(hypervisorType, networkHelper.getHypervisorRouterTemplateConfigMap().get(hypervisorType).valueIn(dataCenter.getId()));
369+
}
363370
if (template == null) {
364-
throw new CloudRuntimeException("Not able to find the System templates or not downloaded in zone " + dataCenter.getId());
371+
throw new CloudRuntimeException("Not able to find the System or Routing template in ready state for the zone " + dataCenter.getUuid());
365372
}
366373
return template;
367374
}
@@ -628,14 +635,17 @@ private void validateEndpointUrl() {
628635
}
629636
}
630637

631-
private DataCenter validateAndGetZoneForKubernetesCreateParameters(Long zoneId) {
638+
private DataCenter validateAndGetZoneForKubernetesCreateParameters(Long zoneId, Long networkId) {
632639
DataCenter zone = dataCenterDao.findById(zoneId);
633640
if (zone == null) {
634641
throw new InvalidParameterValueException("Unable to find zone by ID: " + zoneId);
635642
}
636643
if (zone.getAllocationState() == Grouping.AllocationState.Disabled) {
637644
throw new PermissionDeniedException(String.format("Cannot perform this operation, zone ID: %s is currently disabled", zone.getUuid()));
638645
}
646+
if (DataCenter.Type.Edge.equals(zone.getType()) && networkId == null) {
647+
throw new PermissionDeniedException("Kubernetes clusters cannot be created on an edge zone without an existing network");
648+
}
639649
return zone;
640650
}
641651

@@ -675,7 +685,7 @@ private void validateUnmanagedKubernetesClusterCreateParameters(final CreateKube
675685
throw new InvalidParameterValueException("Invalid name for the Kubernetes cluster name: " + name);
676686
}
677687

678-
validateAndGetZoneForKubernetesCreateParameters(zoneId);
688+
validateAndGetZoneForKubernetesCreateParameters(zoneId, networkId);
679689
validateSshKeyPairForKubernetesCreateParameters(sshKeyPair, owner);
680690

681691
if (nodeRootDiskSize != null && nodeRootDiskSize <= 0) {
@@ -751,7 +761,7 @@ private void validateManagedKubernetesClusterCreateParameters(final CreateKubern
751761
String.format("Maximum cluster size can not exceed %d. Please contact your administrator", maxClusterSize));
752762
}
753763

754-
DataCenter zone = validateAndGetZoneForKubernetesCreateParameters(zoneId);
764+
DataCenter zone = validateAndGetZoneForKubernetesCreateParameters(zoneId, networkId);
755765

756766
if (!isKubernetesServiceConfigured(zone)) {
757767
throw new CloudRuntimeException("Kubernetes service has not been configured properly to provision Kubernetes clusters");

0 commit comments

Comments
 (0)