Skip to content

Commit 685ee9e

Browse files
slavkapslavkap
authored
StorPool: support for direct download (#9833)
1 parent 7f13beb commit 685ee9e

File tree

1 file changed

+203
-19
lines changed

1 file changed

+203
-19
lines changed

‎plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java‎

Lines changed: 203 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,29 @@
1717
package com.cloud.hypervisor.kvm.storage;
1818

1919

20+
import com.cloud.agent.api.to.DiskTO;
21+
import com.cloud.storage.Storage;
22+
import com.cloud.storage.Storage.ImageFormat;
23+
import com.cloud.storage.Storage.ProvisioningType;
24+
import com.cloud.storage.Storage.StoragePoolType;
25+
import com.cloud.utils.exception.CloudRuntimeException;
26+
import com.cloud.utils.script.OutputInterpreter;
27+
import com.cloud.utils.script.Script;
28+
import com.google.gson.Gson;
29+
import com.google.gson.JsonObject;
30+
import com.google.gson.JsonParser;
31+
import com.google.gson.JsonPrimitive;
32+
import org.apache.cloudstack.storage.datastore.util.StorPoolUtil;
33+
import org.apache.cloudstack.utils.qemu.QemuImg;
34+
import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
35+
import org.apache.cloudstack.utils.qemu.QemuImgException;
36+
import org.apache.cloudstack.utils.qemu.QemuImgFile;
37+
import org.apache.commons.lang3.StringUtils;
38+
import org.apache.logging.log4j.LogManager;
39+
import org.apache.logging.log4j.Logger;
40+
import org.jetbrains.annotations.NotNull;
41+
import org.libvirt.LibvirtException;
42+
2043
import java.io.BufferedWriter;
2144
import java.io.File;
2245
import java.io.FileWriter;
@@ -26,19 +49,7 @@
2649
import java.util.HashMap;
2750
import java.util.List;
2851
import java.util.Map;
29-
30-
import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
31-
import org.apache.logging.log4j.Logger;
32-
import org.apache.logging.log4j.LogManager;
33-
34-
import com.cloud.agent.api.to.DiskTO;
35-
import com.cloud.storage.Storage;
36-
import com.cloud.storage.Storage.ImageFormat;
37-
import com.cloud.storage.Storage.ProvisioningType;
38-
import com.cloud.storage.Storage.StoragePoolType;
39-
import com.cloud.utils.exception.CloudRuntimeException;
40-
import com.cloud.utils.script.OutputInterpreter;
41-
import com.cloud.utils.script.Script;
52+
import java.util.UUID;
4253

4354
public class StorPoolStorageAdaptor implements StorageAdaptor {
4455
public static void SP_LOG(String fmt, Object... args) {
@@ -149,6 +160,10 @@ public static String getVolumeNameFromPath(final String volumeUuid, boolean tild
149160
}
150161

151162
public static boolean attachOrDetachVolume(String command, String type, String volumeUuid) {
163+
if (volumeUuid == null) {
164+
LOGGER.debug("Could not attach volume. The volume ID is null");
165+
return false;
166+
}
152167
final String name = getVolumeNameFromPath(volumeUuid, true);
153168
if (name == null) {
154169
return false;
@@ -345,11 +360,85 @@ public boolean createFolder(String uuid, String path) {
345360
throw new UnsupportedOperationException("A folder cannot be created in this configuration.");
346361
}
347362

348-
public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath,
349-
KVMStoragePool destPool, ImageFormat format, int timeout) {
363+
@Override
364+
public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name,
365+
PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) {
350366
return null;
351367
}
352368

369+
@Override
370+
public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath,
371+
KVMStoragePool destPool, ImageFormat format, int timeout) {
372+
if (StringUtils.isEmpty(templateFilePath) || destPool == null) {
373+
throw new CloudRuntimeException(
374+
"Unable to create template from direct download template file due to insufficient data");
375+
}
376+
377+
File sourceFile = new File(templateFilePath);
378+
if (!sourceFile.exists()) {
379+
throw new CloudRuntimeException(
380+
"Direct download template file " + templateFilePath + " does not exist on this host");
381+
}
382+
383+
if (!StoragePoolType.StorPool.equals(destPool.getType())) {
384+
throw new CloudRuntimeException("Unsupported storage pool type: " + destPool.getType().toString());
385+
}
386+
387+
if (!Storage.ImageFormat.QCOW2.equals(format)) {
388+
throw new CloudRuntimeException("Unsupported template format: " + format.toString());
389+
}
390+
391+
String srcTemplateFilePath = templateFilePath;
392+
KVMPhysicalDisk destDisk = null;
393+
QemuImgFile srcFile = null;
394+
QemuImgFile destFile = null;
395+
String templateName = UUID.randomUUID().toString();
396+
String volume = null;
397+
try {
398+
399+
srcTemplateFilePath = extractTemplate(templateFilePath, sourceFile, srcTemplateFilePath, templateName);
400+
401+
QemuImg.PhysicalDiskFormat srcFileFormat = QemuImg.PhysicalDiskFormat.QCOW2;
402+
403+
srcFile = new QemuImgFile(srcTemplateFilePath, srcFileFormat);
404+
405+
String spTemplate = destPool.getUuid().split(";")[0];
406+
407+
QemuImg qemu = new QemuImg(timeout);
408+
OutputInterpreter.AllLinesParser parser = createStorPoolVolume(destPool, srcFile, qemu, spTemplate);
409+
410+
String response = parser.getLines();
411+
412+
LOGGER.debug(response);
413+
volume = StorPoolUtil.devPath(getNameFromResponse(response, false, false));
414+
attachOrDetachVolume("attach", "volume", volume);
415+
destDisk = destPool.getPhysicalDisk(volume);
416+
if (destDisk == null) {
417+
throw new CloudRuntimeException(
418+
"Failed to find the disk: " + volume + " of the storage pool: " + destPool.getUuid());
419+
}
420+
421+
destFile = new QemuImgFile(destDisk.getPath(), QemuImg.PhysicalDiskFormat.RAW);
422+
423+
qemu.convert(srcFile, destFile);
424+
parser = volumeSnapshot(StorPoolStorageAdaptor.getVolumeNameFromPath(volume, true), spTemplate);
425+
response = parser.getLines();
426+
LOGGER.debug(response);
427+
String newPath = StorPoolUtil.devPath(getNameFromResponse(response, false, true));
428+
destDisk = destPool.getPhysicalDisk(newPath);
429+
} catch (QemuImgException | LibvirtException e) {
430+
destDisk = null;
431+
} finally {
432+
if (volume != null) {
433+
attachOrDetachVolume("detach", "volume", volume);
434+
volumeDelete(StorPoolStorageAdaptor.getVolumeNameFromPath(volume, true));
435+
}
436+
Script.runSimpleBashScript("rm -f " + srcTemplateFilePath);
437+
}
438+
439+
return destDisk;
440+
}
441+
353442
@Override
354443
public boolean createFolder(String uuid, String path, String localPath) {
355444
return false;
@@ -367,9 +456,104 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String n
367456
return null;
368457
}
369458

370-
@Override
371-
public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name,
372-
PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) {
373-
return null;
459+
private OutputInterpreter.AllLinesParser createStorPoolVolume(KVMStoragePool destPool, QemuImgFile srcFile,
460+
QemuImg qemu, String templateUuid) throws QemuImgException, LibvirtException {
461+
Map<String, String> info = qemu.info(srcFile);
462+
Map<String, Object> reqParams = new HashMap<>();
463+
reqParams.put("template", templateUuid);
464+
reqParams.put("size", info.get("virtual_size"));
465+
Map<String, String> tags = new HashMap<>();
466+
tags.put("cs", "template");
467+
reqParams.put("tags", tags);
468+
Gson gson = new Gson();
469+
String js = gson.toJson(reqParams);
470+
471+
Script sc = createStorPoolRequest(js, "VolumeCreate", null,true);
472+
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
473+
474+
String res = sc.execute(parser);
475+
if (res != null) {
476+
throw new CloudRuntimeException("Could not create volume due to: " + res);
477+
}
478+
return parser;
479+
}
480+
481+
private OutputInterpreter.AllLinesParser volumeSnapshot(String volumeName, String templateUuid) {
482+
Map<String, String> reqParams = new HashMap<>();
483+
reqParams.put("template", templateUuid);
484+
Gson gson = new Gson();
485+
String js = gson.toJson(reqParams);
486+
487+
Script sc = createStorPoolRequest(js, "VolumeSnapshot", volumeName,true);
488+
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
489+
490+
String res = sc.execute(parser);
491+
if (res != null) {
492+
throw new CloudRuntimeException("Could not snapshot volume due to: " + res);
493+
}
494+
return parser;
495+
}
496+
497+
private OutputInterpreter.AllLinesParser volumeDelete(String volumeName) {
498+
Script sc = createStorPoolRequest(null, "VolumeDelete", volumeName, false);
499+
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
500+
501+
String res = sc.execute(parser);
502+
if (res != null) {
503+
throw new CloudRuntimeException("Could not delete volume due to: " + res);
504+
}
505+
return parser;
506+
}
507+
@NotNull
508+
private static Script createStorPoolRequest(String js, String apiCall, String param, boolean jsonRequired) {
509+
Script sc = new Script("storpool_req", 0, LOGGER);
510+
sc.add("-P");
511+
sc.add("-M");
512+
if (jsonRequired) {
513+
sc.add("--json");
514+
sc.add(js);
515+
}
516+
sc.add(apiCall);
517+
if (param != null) {
518+
sc.add(param);
519+
}
520+
return sc;
521+
}
522+
523+
private String extractTemplate(String templateFilePath, File sourceFile, String srcTemplateFilePath,
524+
String templateName) {
525+
if (isTemplateExtractable(templateFilePath)) {
526+
srcTemplateFilePath = sourceFile.getParent() + "/" + templateName;
527+
String extractCommand = getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath);
528+
Script.runSimpleBashScript(extractCommand);
529+
Script.runSimpleBashScript("rm -f " + templateFilePath);
530+
}
531+
return srcTemplateFilePath;
532+
}
533+
534+
private boolean isTemplateExtractable(String templatePath) {
535+
String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'");
536+
return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
537+
}
538+
539+
private String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) {
540+
if (downloadedTemplateFile.endsWith(".zip")) {
541+
return "unzip -p " + downloadedTemplateFile + " | cat > " + templateFile;
542+
} else if (downloadedTemplateFile.endsWith(".bz2")) {
543+
return "bunzip2 -c " + downloadedTemplateFile + " > " + templateFile;
544+
} else if (downloadedTemplateFile.endsWith(".gz")) {
545+
return "gunzip -c " + downloadedTemplateFile + " > " + templateFile;
546+
} else {
547+
throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile);
548+
}
549+
}
550+
551+
private String getNameFromResponse(String resp, boolean tildeNeeded, boolean isSnapshot) {
552+
JsonParser jsonParser = new JsonParser();
553+
JsonObject respObj = (JsonObject) jsonParser.parse(resp);
554+
JsonPrimitive data = isSnapshot ? respObj.getAsJsonPrimitive("snapshotGlobalId") : respObj.getAsJsonPrimitive("globalId");
555+
String name = data !=null ? data.getAsString() : null;
556+
name = name != null ? name.startsWith("~") && !tildeNeeded ? name.split("~")[1] : name : name;
557+
return name;
374558
}
375559
}

0 commit comments

Comments
 (0)