Skip to content

Commit cb4848b

Browse files
BryanMLimaBryanMLima
authored
Add support to RBD erasure code pools (#9808)
* Readd filename string on qemuimg create * Remove empty object on the data pool details of storage pools with no data pool * Only use the method createPhysicalDiskByLibVirt with RBD when the pool is of erasure code type. Also added javadoc for createPhysicalDisk method * Change literal '/' string to File.separator * Add support for erasure code pools * Fix null on putAll
1 parent 2dfe6a6 commit cb4848b

File tree

21 files changed

+312
-111
lines changed

21 files changed

+312
-111
lines changed

‎api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,10 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations {
149149
@Param(description = "whether this pool is managed or not")
150150
private Boolean managed;
151151

152+
@SerializedName(ApiConstants.DETAILS)
153+
@Param(description = "the storage pool details")
154+
private Map<String, String> details;
155+
152156
public Map<String, String> getCaps() {
153157
return caps;
154158
}
@@ -407,4 +411,12 @@ public Boolean getManaged() {
407411
public void setManaged(Boolean managed) {
408412
this.managed = managed;
409413
}
414+
415+
public Map<String, String> getDetails() {
416+
return details;
417+
}
418+
419+
public void setDetails(Map<String, String> details) {
420+
this.details = details;
421+
}
410422
}

‎engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
import com.cloud.storage.StoragePoolHostVO;
4444
import com.cloud.storage.StorageService;
4545
import com.cloud.storage.dao.StoragePoolHostDao;
46-
import com.cloud.utils.Pair;
4746
import com.cloud.utils.exception.CloudRuntimeException;
4847

4948
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@@ -60,6 +59,7 @@
6059

6160
import java.util.List;
6261
import java.util.Map;
62+
import java.util.Optional;
6363

6464
public class DefaultHostListener implements HypervisorHostListener {
6565
protected Logger logger = LogManager.getLogger(getClass());
@@ -133,9 +133,11 @@ private NicTO createNicTOFromNetworkAndOffering(NetworkVO networkVO, NetworkOffe
133133
@Override
134134
public boolean hostConnect(long hostId, long poolId) throws StorageConflictException {
135135
StoragePool pool = (StoragePool) this.dataStoreMgr.getDataStore(poolId, DataStoreRole.Primary);
136-
Pair<Map<String, String>, Boolean> nfsMountOpts = storageManager.getStoragePoolNFSMountOpts(pool, null);
136+
Map<String, String> detailsMap = storagePoolDetailsDao.listDetailsKeyPairs(poolId);
137+
Map<String, String> nfsMountOpts = storageManager.getStoragePoolNFSMountOpts(pool, null).first();
137138

138-
ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool, nfsMountOpts.first());
139+
Optional.ofNullable(nfsMountOpts).ifPresent(detailsMap::putAll);
140+
ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool, detailsMap);
139141
cmd.setWait(modifyStoragePoolCommandWait);
140142
HostVO host = hostDao.findById(hostId);
141143
logger.debug("Sending modify storage pool command to agent: {} for storage pool: {} with timeout {} seconds", host, pool, cmd.getWait());

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreatePrivateTemplateFromVolumeCommandWrapper.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,9 +108,7 @@ public Answer execute(final CreatePrivateTemplateFromVolumeCommand command, fina
108108
} else {
109109
logger.debug("Converting RBD disk " + disk.getPath() + " into template " + command.getUniqueName());
110110

111-
final QemuImgFile srcFile =
112-
new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primary.getSourceHost(), primary.getSourcePort(), primary.getAuthUserName(),
113-
primary.getAuthSecret(), disk.getPath()));
111+
final QemuImgFile srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primary, disk.getPath()));
114112
srcFile.setFormat(PhysicalDiskFormat.RAW);
115113

116114
final QemuImgFile destFile = new QemuImgFile(tmpltPath + "/" + command.getUniqueName() + ".qcow2");

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -161,11 +161,7 @@ private Map<String, String> getDiskFileInfo(KVMStoragePool pool, KVMPhysicalDisk
161161
QemuImg qemu = new QemuImg(0);
162162
QemuImgFile qemuFile = new QemuImgFile(disk.getPath(), disk.getFormat());
163163
if (StoragePoolType.RBD.equals(pool.getType())) {
164-
String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(pool.getSourceHost(),
165-
pool.getSourcePort(),
166-
pool.getAuthUserName(),
167-
pool.getAuthSecret(),
168-
disk.getPath());
164+
String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(pool, disk.getPath());
169165
qemuFile = new QemuImgFile(rbdDestFile, disk.getFormat());
170166
}
171167
return qemu.info(qemuFile, secure);

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -410,9 +410,7 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk srcDisk, String destVolu
410410
KVMStoragePool srcPool = srcDisk.getPool();
411411

412412
if (srcPool.getType() == StoragePoolType.RBD) {
413-
srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool.getSourceHost(), srcPool.getSourcePort(),
414-
srcPool.getAuthUserName(), srcPool.getAuthSecret(),
415-
srcDisk.getPath()),srcDisk.getFormat());
413+
srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool, srcDisk.getPath()), srcDisk.getFormat());
416414
} else {
417415
srcFile = new QemuImgFile(srcDisk.getPath(), srcDisk.getFormat());
418416
}

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323

2424
import java.util.ArrayList;
2525
import java.util.List;
26+
import java.util.Map;
2627

2728
public class KVMPhysicalDisk {
2829
private String path;
@@ -32,10 +33,17 @@ public class KVMPhysicalDisk {
3233
private String vmName;
3334
private boolean useAsTemplate;
3435

35-
public static String RBDStringBuilder(String monHost, int monPort, String authUserName, String authSecret, String image) {
36-
String rbdOpts;
36+
public static final String RBD_DEFAULT_DATA_POOL = "rbd_default_data_pool";
3737

38-
rbdOpts = "rbd:" + image;
38+
public static String RBDStringBuilder(KVMStoragePool storagePool, String image) {
39+
String monHost = storagePool.getSourceHost();
40+
int monPort = storagePool.getSourcePort();
41+
String authUserName = storagePool.getAuthUserName();
42+
String authSecret = storagePool.getAuthSecret();
43+
Map<String, String> details = storagePool.getDetails();
44+
String dataPool = (details == null) ? null : details.get(RBD_DEFAULT_DATA_POOL);
45+
46+
String rbdOpts = "rbd:" + image;
3947
rbdOpts += ":mon_host=" + composeOptionForMonHosts(monHost, monPort);
4048

4149
if (authUserName == null) {
@@ -46,6 +54,10 @@ public static String RBDStringBuilder(String monHost, int monPort, String authUs
4654
rbdOpts += ":key=" + authSecret;
4755
}
4856

57+
if (dataPool != null) {
58+
rbdOpts += String.format(":rbd_default_data_pool=%s", dataPool);
59+
}
60+
4961
rbdOpts += ":rbd_default_format=2";
5062
rbdOpts += ":client_mount_timeout=30";
5163

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java‎

Lines changed: 20 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -53,28 +53,6 @@
5353
public class KVMStoragePoolManager {
5454
protected Logger logger = LogManager.getLogger(getClass());
5555

56-
private class StoragePoolInformation {
57-
String name;
58-
String host;
59-
int port;
60-
String path;
61-
String userInfo;
62-
boolean type;
63-
StoragePoolType poolType;
64-
Map<String, String> details;
65-
66-
public StoragePoolInformation(String name, String host, int port, String path, String userInfo, StoragePoolType poolType, Map<String, String> details, boolean type) {
67-
this.name = name;
68-
this.host = host;
69-
this.port = port;
70-
this.path = path;
71-
this.userInfo = userInfo;
72-
this.type = type;
73-
this.poolType = poolType;
74-
this.details = details;
75-
}
76-
}
77-
7856
private KVMHAMonitor _haMonitor;
7957
private final Map<String, StoragePoolInformation> _storagePools = new ConcurrentHashMap<String, StoragePoolInformation>();
8058
private final Map<String, StorageAdaptor> _storageMapper = new HashMap<String, StorageAdaptor>();
@@ -303,14 +281,33 @@ public KVMStoragePool getStoragePool(StoragePoolType type, String uuid, boolean
303281
} catch (Exception e) {
304282
StoragePoolInformation info = _storagePools.get(uuid);
305283
if (info != null) {
306-
pool = createStoragePool(info.name, info.host, info.port, info.path, info.userInfo, info.poolType, info.details, info.type);
284+
pool = createStoragePool(info.getName(), info.getHost(), info.getPort(), info.getPath(), info.getUserInfo(), info.getPoolType(), info.getDetails(), info.isType());
307285
} else {
308286
throw new CloudRuntimeException("Could not fetch storage pool " + uuid + " from libvirt due to " + e.getMessage());
309287
}
310288
}
289+
290+
if (pool instanceof LibvirtStoragePool) {
291+
addPoolDetails(uuid, (LibvirtStoragePool) pool);
292+
}
293+
311294
return pool;
312295
}
313296

297+
/**
298+
* As the class {@link LibvirtStoragePool} is constrained to the {@link org.libvirt.StoragePool} class, there is no way of saving a generic parameter such as the details, hence,
299+
* this method was created to always make available the details of libvirt primary storages for when they are needed.
300+
*/
301+
private void addPoolDetails(String uuid, LibvirtStoragePool pool) {
302+
StoragePoolInformation storagePoolInformation = _storagePools.get(uuid);
303+
Map<String, String> details = storagePoolInformation.getDetails();
304+
305+
if (MapUtils.isNotEmpty(details)) {
306+
logger.trace("Adding the details {} to the pool with UUID {}.", details, uuid);
307+
pool.setDetails(details);
308+
}
309+
}
310+
314311
public KVMStoragePool getStoragePoolByURI(String uri) {
315312
URI storageUri = null;
316313

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -667,9 +667,7 @@ public Answer createTemplateFromVolume(final CopyCommand cmd) {
667667
} else {
668668
logger.debug("Converting RBD disk " + disk.getPath() + " into template " + templateName);
669669

670-
final QemuImgFile srcFile =
671-
new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primary.getSourceHost(), primary.getSourcePort(), primary.getAuthUserName(),
672-
primary.getAuthSecret(), disk.getPath()));
670+
final QemuImgFile srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primary, disk.getPath()));
673671
srcFile.setFormat(PhysicalDiskFormat.RAW);
674672

675673
final QemuImgFile destFile = new QemuImgFile(tmpltPath + "/" + templateName + ".qcow2");
@@ -1022,9 +1020,7 @@ public Answer backupSnapshot(final CopyCommand cmd) {
10221020
logger.debug("Attempting to create " + snapDir.getAbsolutePath() + " recursively for snapshot storage");
10231021
FileUtils.forceMkdir(snapDir);
10241022

1025-
final QemuImgFile srcFile =
1026-
new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primaryPool.getSourceHost(), primaryPool.getSourcePort(), primaryPool.getAuthUserName(),
1027-
primaryPool.getAuthSecret(), rbdSnapshot));
1023+
final QemuImgFile srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primaryPool, rbdSnapshot));
10281024
srcFile.setFormat(snapshotDisk.getFormat());
10291025

10301026
final QemuImgFile destFile = new QemuImgFile(snapshotFile);

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java‎

Lines changed: 58 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -960,17 +960,55 @@ public boolean deleteStoragePool(String uuid) {
960960
}
961961
}
962962

963+
/**
964+
* Creates a physical disk depending on the {@link StoragePoolType}:
965+
* <ul>
966+
* <li>
967+
* <b>{@link StoragePoolType#RBD}</b>
968+
* <ul>
969+
* <li>
970+
* If it is an erasure code pool, utilizes QemuImg to create the physical disk through the method
971+
* {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])}
972+
* </li>
973+
* <li>
974+
* Otherwise, utilize Libvirt to create the physical disk through the method
975+
* {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)}
976+
* </li>
977+
* </ul>
978+
* </li>
979+
* <li>
980+
* {@link StoragePoolType#NetworkFilesystem} and {@link StoragePoolType#Filesystem}
981+
* <ul>
982+
* <li>
983+
* If the format is {@link PhysicalDiskFormat#QCOW2} or {@link PhysicalDiskFormat#RAW}, utilizes QemuImg to create the physical disk through the method
984+
* {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])}
985+
* </li>
986+
* <li>
987+
* If the format is {@link PhysicalDiskFormat#DIR} or {@link PhysicalDiskFormat#TAR}, utilize Libvirt to create the physical disk through the method
988+
* {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)}
989+
* </li>
990+
* </ul>
991+
* </li>
992+
* <li>
993+
* For the rest of the {@link StoragePoolType} types, utilizes the Libvirt method
994+
* {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)}
995+
* </li>
996+
* </ul>
997+
*/
963998
@Override
964999
public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool,
9651000
PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) {
9661001

967-
logger.info("Attempting to create volume " + name + " (" + pool.getType().toString() + ") in pool "
968-
+ pool.getUuid() + " with size " + toHumanReadableSize(size));
1002+
logger.info("Attempting to create volume {} ({}) in pool {} with size {}", name, pool.getType().toString(), pool.getUuid(), toHumanReadableSize(size));
9691003

9701004
StoragePoolType poolType = pool.getType();
971-
if (poolType.equals(StoragePoolType.RBD)) {
972-
return createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size);
973-
} else if (poolType.equals(StoragePoolType.NetworkFilesystem) || poolType.equals(StoragePoolType.Filesystem)) {
1005+
if (StoragePoolType.RBD.equals(poolType)) {
1006+
Map<String, String> details = pool.getDetails();
1007+
String dataPool = (details == null) ? null : details.get(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL);
1008+
1009+
return (dataPool == null) ? createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) :
1010+
createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, passphrase);
1011+
} else if (StoragePoolType.NetworkFilesystem.equals(poolType) || StoragePoolType.Filesystem.equals(poolType)) {
9741012
switch (format) {
9751013
case QCOW2:
9761014
case RAW:
@@ -1018,18 +1056,25 @@ private KVMPhysicalDisk createPhysicalDiskByLibVirt(String name, KVMStoragePool
10181056
}
10191057

10201058

1021-
private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool pool,
1022-
PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) {
1023-
String volPath = pool.getLocalPath() + "/" + name;
1059+
private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool pool, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size,
1060+
byte[] passphrase) {
1061+
String volPath;
10241062
String volName = name;
10251063
long virtualSize = 0;
10261064
long actualSize = 0;
10271065
QemuObject.EncryptFormat encryptFormat = null;
10281066
List<QemuObject> passphraseObjects = new ArrayList<>();
1029-
10301067
final int timeout = 0;
1068+
QemuImgFile destFile;
1069+
1070+
if (StoragePoolType.RBD.equals(pool.getType())) {
1071+
volPath = pool.getSourceDir() + File.separator + name;
1072+
destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(pool, volPath));
1073+
} else {
1074+
volPath = pool.getLocalPath() + File.separator + name;
1075+
destFile = new QemuImgFile(volPath);
1076+
}
10311077

1032-
QemuImgFile destFile = new QemuImgFile(volPath);
10331078
destFile.setFormat(format);
10341079
destFile.setSize(size);
10351080
Map<String, String> options = new HashMap<String, String>();
@@ -1312,11 +1357,7 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template,
13121357

13131358

13141359
QemuImgFile srcFile;
1315-
QemuImgFile destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(destPool.getSourceHost(),
1316-
destPool.getSourcePort(),
1317-
destPool.getAuthUserName(),
1318-
destPool.getAuthSecret(),
1319-
disk.getPath()));
1360+
QemuImgFile destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(destPool, disk.getPath()));
13201361
destFile.setFormat(format);
13211362

13221363
if (srcPool.getType() != StoragePoolType.RBD) {
@@ -1591,11 +1632,7 @@ to support snapshots(backuped) as qcow2 files. */
15911632
try {
15921633
srcFile = new QemuImgFile(sourcePath, sourceFormat);
15931634
String rbdDestPath = destPool.getSourceDir() + "/" + name;
1594-
String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(destPool.getSourceHost(),
1595-
destPool.getSourcePort(),
1596-
destPool.getAuthUserName(),
1597-
destPool.getAuthSecret(),
1598-
rbdDestPath);
1635+
String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(destPool, rbdDestPath);
15991636
destFile = new QemuImgFile(rbdDestFile, destFormat);
16001637

16011638
logger.debug("Starting copy from source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath);
@@ -1638,9 +1675,7 @@ to support snapshots(backuped) as qcow2 files. */
16381675
We let Qemu-Img do the work here. Although we could work with librbd and have that do the cloning
16391676
it doesn't benefit us. It's better to keep the current code in place which works
16401677
*/
1641-
srcFile =
1642-
new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool.getSourceHost(), srcPool.getSourcePort(), srcPool.getAuthUserName(), srcPool.getAuthSecret(),
1643-
sourcePath));
1678+
srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool, sourcePath));
16441679
srcFile.setFormat(sourceFormat);
16451680
destFile = new QemuImgFile(destPath);
16461681
destFile.setFormat(destFormat);

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,8 @@ public class LibvirtStoragePool implements KVMStoragePool {
5656
protected String authSecret;
5757
protected String sourceHost;
5858
protected int sourcePort;
59-
6059
protected String sourceDir;
60+
protected Map<String, String> details;
6161

6262
public LibvirtStoragePool(String uuid, String name, StoragePoolType type, StorageAdaptor adaptor, StoragePool pool) {
6363
this.uuid = uuid;
@@ -311,7 +311,11 @@ public boolean supportsConfigDriveIso() {
311311

312312
@Override
313313
public Map<String, String> getDetails() {
314-
return null;
314+
return this.details;
315+
}
316+
317+
public void setDetails(Map<String, String> details) {
318+
this.details = details;
315319
}
316320

317321
@Override

0 commit comments

Comments
 (0)