Skip to content

Commit e17f5fc

Browse files
BryanMLimadhslove
BryanMLima
authored and
dhslove
committed
Add support to RBD erasure code pools (apache#9808)
* Readd filename string on qemuimg create * Remove empty object on the data pool details of storage pools with no data pool * Only use the method createPhysicalDiskByLibVirt with RBD when the pool is of erasure code type. Also added javadoc for createPhysicalDisk method * Change literal '/' string to File.separator * Add support for erasure code pools * Fix null on putAll
1 parent 88e4eb1 commit e17f5fc

File tree

21 files changed

+311
-108
lines changed

21 files changed

+311
-108
lines changed

‎api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,10 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations {
149149
@Param(description = "whether this pool is managed or not")
150150
private Boolean managed;
151151

152+
@SerializedName(ApiConstants.DETAILS)
153+
@Param(description = "the storage pool details")
154+
private Map<String, String> details;
155+
152156
public Map<String, String> getCaps() {
153157
return caps;
154158
}
@@ -407,4 +411,12 @@ public Boolean getManaged() {
407411
public void setManaged(Boolean managed) {
408412
this.managed = managed;
409413
}
414+
415+
public Map<String, String> getDetails() {
416+
return details;
417+
}
418+
419+
public void setDetails(Map<String, String> details) {
420+
this.details = details;
421+
}
410422
}

‎engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
import com.cloud.storage.StoragePoolHostVO;
4444
import com.cloud.storage.StorageService;
4545
import com.cloud.storage.dao.StoragePoolHostDao;
46-
import com.cloud.utils.Pair;
4746
import com.cloud.utils.exception.CloudRuntimeException;
4847

4948
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@@ -60,6 +59,7 @@
6059

6160
import java.util.List;
6261
import java.util.Map;
62+
import java.util.Optional;
6363

6464
public class DefaultHostListener implements HypervisorHostListener {
6565
protected Logger logger = LogManager.getLogger(getClass());
@@ -133,9 +133,11 @@ private NicTO createNicTOFromNetworkAndOffering(NetworkVO networkVO, NetworkOffe
133133
@Override
134134
public boolean hostConnect(long hostId, long poolId) throws StorageConflictException {
135135
StoragePool pool = (StoragePool) this.dataStoreMgr.getDataStore(poolId, DataStoreRole.Primary);
136-
Pair<Map<String, String>, Boolean> nfsMountOpts = storageManager.getStoragePoolNFSMountOpts(pool, null);
136+
Map<String, String> detailsMap = storagePoolDetailsDao.listDetailsKeyPairs(poolId);
137+
Map<String, String> nfsMountOpts = storageManager.getStoragePoolNFSMountOpts(pool, null).first();
137138

138-
ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool, nfsMountOpts.first());
139+
Optional.ofNullable(nfsMountOpts).ifPresent(detailsMap::putAll);
140+
ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool, detailsMap);
139141
cmd.setDetails(primaryStoreDao.getDetails(poolId));
140142
cmd.setWait(modifyStoragePoolCommandWait);
141143
HostVO host = hostDao.findById(hostId);

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreatePrivateTemplateFromVolumeCommandWrapper.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,9 +108,7 @@ public Answer execute(final CreatePrivateTemplateFromVolumeCommand command, fina
108108
} else {
109109
logger.debug("Converting RBD disk " + disk.getPath() + " into template " + command.getUniqueName());
110110

111-
final QemuImgFile srcFile =
112-
new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primary.getSourceHost(), primary.getSourcePort(), primary.getAuthUserName(),
113-
primary.getAuthSecret(), disk.getPath()));
111+
final QemuImgFile srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primary, disk.getPath()));
114112
srcFile.setFormat(PhysicalDiskFormat.RAW);
115113

116114
final QemuImgFile destFile = new QemuImgFile(tmpltPath + "/" + command.getUniqueName() + ".qcow2");

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -161,11 +161,7 @@ private Map<String, String> getDiskFileInfo(KVMStoragePool pool, KVMPhysicalDisk
161161
QemuImg qemu = new QemuImg(0);
162162
QemuImgFile qemuFile = new QemuImgFile(disk.getPath(), disk.getFormat());
163163
if (StoragePoolType.RBD.equals(pool.getType())) {
164-
String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(pool.getSourceHost(),
165-
pool.getSourcePort(),
166-
pool.getAuthUserName(),
167-
pool.getAuthSecret(),
168-
disk.getPath());
164+
String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(pool, disk.getPath());
169165
qemuFile = new QemuImgFile(rbdDestFile, disk.getFormat());
170166
}
171167
return qemu.info(qemuFile, secure);

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -410,9 +410,7 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk srcDisk, String destVolu
410410
KVMStoragePool srcPool = srcDisk.getPool();
411411

412412
if (srcPool.getType() == StoragePoolType.RBD) {
413-
srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool.getSourceHost(), srcPool.getSourcePort(),
414-
srcPool.getAuthUserName(), srcPool.getAuthSecret(),
415-
srcDisk.getPath()),srcDisk.getFormat());
413+
srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool, srcDisk.getPath()), srcDisk.getFormat());
416414
} else {
417415
srcFile = new QemuImgFile(srcDisk.getPath(), srcDisk.getFormat());
418416
}

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323

2424
import java.util.ArrayList;
2525
import java.util.List;
26+
import java.util.Map;
2627

2728
public class KVMPhysicalDisk {
2829
private String path;
@@ -32,10 +33,17 @@ public class KVMPhysicalDisk {
3233
private String vmName;
3334
private boolean useAsTemplate;
3435

35-
public static String RBDStringBuilder(String monHost, int monPort, String authUserName, String authSecret, String image) {
36-
String rbdOpts;
36+
public static final String RBD_DEFAULT_DATA_POOL = "rbd_default_data_pool";
3737

38-
rbdOpts = "rbd:" + image;
38+
public static String RBDStringBuilder(KVMStoragePool storagePool, String image) {
39+
String monHost = storagePool.getSourceHost();
40+
int monPort = storagePool.getSourcePort();
41+
String authUserName = storagePool.getAuthUserName();
42+
String authSecret = storagePool.getAuthSecret();
43+
Map<String, String> details = storagePool.getDetails();
44+
String dataPool = (details == null) ? null : details.get(RBD_DEFAULT_DATA_POOL);
45+
46+
String rbdOpts = "rbd:" + image;
3947
rbdOpts += ":mon_host=" + composeOptionForMonHosts(monHost, monPort);
4048

4149
if (authUserName == null) {
@@ -46,6 +54,10 @@ public static String RBDStringBuilder(String monHost, int monPort, String authUs
4654
rbdOpts += ":key=" + authSecret;
4755
}
4856

57+
if (dataPool != null) {
58+
rbdOpts += String.format(":rbd_default_data_pool=%s", dataPool);
59+
}
60+
4961
rbdOpts += ":rbd_default_format=2";
5062
rbdOpts += ":client_mount_timeout=30";
5163

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java‎

Lines changed: 20 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -53,28 +53,6 @@
5353
public class KVMStoragePoolManager {
5454
protected Logger logger = LogManager.getLogger(getClass());
5555

56-
private class StoragePoolInformation {
57-
String name;
58-
String host;
59-
int port;
60-
String path;
61-
String userInfo;
62-
boolean type;
63-
StoragePoolType poolType;
64-
Map<String, String> details;
65-
66-
public StoragePoolInformation(String name, String host, int port, String path, String userInfo, StoragePoolType poolType, Map<String, String> details, boolean type) {
67-
this.name = name;
68-
this.host = host;
69-
this.port = port;
70-
this.path = path;
71-
this.userInfo = userInfo;
72-
this.type = type;
73-
this.poolType = poolType;
74-
this.details = details;
75-
}
76-
}
77-
7856
private KVMHAMonitor _haMonitor;
7957
private final Map<String, StoragePoolInformation> _storagePools = new ConcurrentHashMap<String, StoragePoolInformation>();
8058
private final Map<String, StorageAdaptor> _storageMapper = new HashMap<String, StorageAdaptor>();
@@ -303,14 +281,33 @@ public KVMStoragePool getStoragePool(StoragePoolType type, String uuid, boolean
303281
} catch (Exception e) {
304282
StoragePoolInformation info = _storagePools.get(uuid);
305283
if (info != null) {
306-
pool = createStoragePool(info.name, info.host, info.port, info.path, info.userInfo, info.poolType, info.details, info.type);
284+
pool = createStoragePool(info.getName(), info.getHost(), info.getPort(), info.getPath(), info.getUserInfo(), info.getPoolType(), info.getDetails(), info.isType());
307285
} else {
308286
throw new CloudRuntimeException("Could not fetch storage pool " + uuid + " from libvirt due to " + e.getMessage());
309287
}
310288
}
289+
290+
if (pool instanceof LibvirtStoragePool) {
291+
addPoolDetails(uuid, (LibvirtStoragePool) pool);
292+
}
293+
311294
return pool;
312295
}
313296

297+
/**
298+
* As the class {@link LibvirtStoragePool} is constrained to the {@link org.libvirt.StoragePool} class, there is no way of saving a generic parameter such as the details, hence,
299+
* this method was created to always make available the details of libvirt primary storages for when they are needed.
300+
*/
301+
private void addPoolDetails(String uuid, LibvirtStoragePool pool) {
302+
StoragePoolInformation storagePoolInformation = _storagePools.get(uuid);
303+
Map<String, String> details = storagePoolInformation.getDetails();
304+
305+
if (MapUtils.isNotEmpty(details)) {
306+
logger.trace("Adding the details {} to the pool with UUID {}.", details, uuid);
307+
pool.setDetails(details);
308+
}
309+
}
310+
314311
public KVMStoragePool getStoragePoolByURI(String uri) {
315312
URI storageUri = null;
316313

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1026,9 +1026,7 @@ public Answer backupSnapshot(final CopyCommand cmd) {
10261026
logger.debug("Attempting to create " + snapDir.getAbsolutePath() + " recursively for snapshot storage");
10271027
FileUtils.forceMkdir(snapDir);
10281028

1029-
final QemuImgFile srcFile =
1030-
new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primaryPool.getSourceHost(), primaryPool.getSourcePort(), primaryPool.getAuthUserName(),
1031-
primaryPool.getAuthSecret(), rbdSnapshot));
1029+
final QemuImgFile srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primaryPool, rbdSnapshot));
10321030
srcFile.setFormat(snapshotDisk.getFormat());
10331031

10341032
final QemuImgFile destFile = new QemuImgFile(snapshotFile);

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java‎

Lines changed: 58 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1014,17 +1014,55 @@ public boolean deleteStoragePool(String uuid) {
10141014
}
10151015
}
10161016

1017+
/**
1018+
* Creates a physical disk depending on the {@link StoragePoolType}:
1019+
* <ul>
1020+
* <li>
1021+
* <b>{@link StoragePoolType#RBD}</b>
1022+
* <ul>
1023+
* <li>
1024+
* If it is an erasure code pool, utilizes QemuImg to create the physical disk through the method
1025+
* {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])}
1026+
* </li>
1027+
* <li>
1028+
* Otherwise, utilize Libvirt to create the physical disk through the method
1029+
* {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)}
1030+
* </li>
1031+
* </ul>
1032+
* </li>
1033+
* <li>
1034+
* {@link StoragePoolType#NetworkFilesystem} and {@link StoragePoolType#Filesystem}
1035+
* <ul>
1036+
* <li>
1037+
* If the format is {@link PhysicalDiskFormat#QCOW2} or {@link PhysicalDiskFormat#RAW}, utilizes QemuImg to create the physical disk through the method
1038+
* {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])}
1039+
* </li>
1040+
* <li>
1041+
* If the format is {@link PhysicalDiskFormat#DIR} or {@link PhysicalDiskFormat#TAR}, utilize Libvirt to create the physical disk through the method
1042+
* {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)}
1043+
* </li>
1044+
* </ul>
1045+
* </li>
1046+
* <li>
1047+
* For the rest of the {@link StoragePoolType} types, utilizes the Libvirt method
1048+
* {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)}
1049+
* </li>
1050+
* </ul>
1051+
*/
10171052
@Override
10181053
public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool,
10191054
PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) {
10201055

1021-
logger.info("Attempting to create volume " + name + " (" + pool.getType().toString() + ") in pool "
1022-
+ pool.getUuid() + " with size " + toHumanReadableSize(size));
1056+
logger.info("Attempting to create volume {} ({}) in pool {} with size {}", name, pool.getType().toString(), pool.getUuid(), toHumanReadableSize(size));
10231057

10241058
StoragePoolType poolType = pool.getType();
1025-
if (poolType.equals(StoragePoolType.RBD)) {
1026-
return createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size);
1027-
} else if (poolType.equals(StoragePoolType.NetworkFilesystem) || poolType.equals(StoragePoolType.Filesystem)) {
1059+
if (StoragePoolType.RBD.equals(poolType)) {
1060+
Map<String, String> details = pool.getDetails();
1061+
String dataPool = (details == null) ? null : details.get(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL);
1062+
1063+
return (dataPool == null) ? createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) :
1064+
createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, passphrase);
1065+
} else if (StoragePoolType.NetworkFilesystem.equals(poolType) || StoragePoolType.Filesystem.equals(poolType)) {
10281066
switch (format) {
10291067
case QCOW2:
10301068
case RAW:
@@ -1072,18 +1110,25 @@ private KVMPhysicalDisk createPhysicalDiskByLibVirt(String name, KVMStoragePool
10721110
}
10731111

10741112

1075-
private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool pool,
1076-
PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) {
1077-
String volPath = pool.getLocalPath() + "/" + name;
1113+
private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool pool, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size,
1114+
byte[] passphrase) {
1115+
String volPath;
10781116
String volName = name;
10791117
long virtualSize = 0;
10801118
long actualSize = 0;
10811119
QemuObject.EncryptFormat encryptFormat = null;
10821120
List<QemuObject> passphraseObjects = new ArrayList<>();
1083-
10841121
final int timeout = 0;
1122+
QemuImgFile destFile;
1123+
1124+
if (StoragePoolType.RBD.equals(pool.getType())) {
1125+
volPath = pool.getSourceDir() + File.separator + name;
1126+
destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(pool, volPath));
1127+
} else {
1128+
volPath = pool.getLocalPath() + File.separator + name;
1129+
destFile = new QemuImgFile(volPath);
1130+
}
10851131

1086-
QemuImgFile destFile = new QemuImgFile(volPath);
10871132
destFile.setFormat(format);
10881133
destFile.setSize(size);
10891134
Map<String, String> options = new HashMap<String, String>();
@@ -1373,11 +1418,7 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template,
13731418

13741419

13751420
QemuImgFile srcFile;
1376-
QemuImgFile destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(destPool.getSourceHost(),
1377-
destPool.getSourcePort(),
1378-
destPool.getAuthUserName(),
1379-
destPool.getAuthSecret(),
1380-
disk.getPath()));
1421+
QemuImgFile destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(destPool, disk.getPath()));
13811422
destFile.setFormat(format);
13821423

13831424
if (srcPool.getType() != StoragePoolType.RBD) {
@@ -1652,11 +1693,7 @@ to support snapshots(backuped) as qcow2 files. */
16521693
try {
16531694
srcFile = new QemuImgFile(sourcePath, sourceFormat);
16541695
String rbdDestPath = destPool.getSourceDir() + "/" + name;
1655-
String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(destPool.getSourceHost(),
1656-
destPool.getSourcePort(),
1657-
destPool.getAuthUserName(),
1658-
destPool.getAuthSecret(),
1659-
rbdDestPath);
1696+
String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(destPool, rbdDestPath);
16601697
destFile = new QemuImgFile(rbdDestFile, destFormat);
16611698

16621699
logger.debug("Starting copy from source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath);
@@ -1699,9 +1736,7 @@ to support snapshots(backuped) as qcow2 files. */
16991736
We let Qemu-Img do the work here. Although we could work with librbd and have that do the cloning
17001737
it doesn't benefit us. It's better to keep the current code in place which works
17011738
*/
1702-
srcFile =
1703-
new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool.getSourceHost(), srcPool.getSourcePort(), srcPool.getAuthUserName(), srcPool.getAuthSecret(),
1704-
sourcePath));
1739+
srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool, sourcePath));
17051740
srcFile.setFormat(sourceFormat);
17061741
destFile = new QemuImgFile(destPath);
17071742
destFile.setFormat(destFormat);

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@ public class LibvirtStoragePool implements KVMStoragePool {
6060
protected String authSecret;
6161
protected String sourceHost;
6262
protected int sourcePort;
63-
6463
protected String sourceDir;
64+
protected Map<String, String> details;
6565

6666
public LibvirtStoragePool(String uuid, String name, StoragePoolType type, StorageAdaptor adaptor, StoragePool pool) {
6767
this.uuid = uuid;
@@ -315,7 +315,11 @@ public boolean supportsConfigDriveIso() {
315315

316316
@Override
317317
public Map<String, String> getDetails() {
318-
return null;
318+
return this.details;
319+
}
320+
321+
public void setDetails(Map<String, String> details) {
322+
this.details = details;
319323
}
320324

321325
@Override

0 commit comments

Comments
 (0)