StorPool: support for direct download

plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java

@@ -17,6 +17,29 @@
 package com.cloud.hypervisor.kvm.storage;
 
 
+import com.cloud.agent.api.to.DiskTO;
+import com.cloud.storage.Storage;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.Storage.ProvisioningType;
+import com.cloud.storage.Storage.StoragePoolType;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.OutputInterpreter;
+import com.cloud.utils.script.Script;
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import org.apache.cloudstack.storage.datastore.util.StorPoolUtil;
+import org.apache.cloudstack.utils.qemu.QemuImg;
+import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
+import org.apache.cloudstack.utils.qemu.QemuImgException;
+import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.NotNull;
+import org.libvirt.LibvirtException;
+
 import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
@@ -26,19 +49,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-
-import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.LogManager;
-
-import com.cloud.agent.api.to.DiskTO;
-import com.cloud.storage.Storage;
-import com.cloud.storage.Storage.ImageFormat;
-import com.cloud.storage.Storage.ProvisioningType;
-import com.cloud.storage.Storage.StoragePoolType;
-import com.cloud.utils.exception.CloudRuntimeException;
-import com.cloud.utils.script.OutputInterpreter;
-import com.cloud.utils.script.Script;
+import java.util.UUID;
 
 public class StorPoolStorageAdaptor implements StorageAdaptor {
     public static void SP_LOG(String fmt, Object... args) {
@@ -149,6 +160,10 @@
     }
 
     public static boolean attachOrDetachVolume(String command, String type, String volumeUuid) {
+        if (volumeUuid == null) {
+            LOGGER.debug("Could not attach volume. The volume ID is null");
+            return false;
+        }
         final String name = getVolumeNameFromPath(volumeUuid, true);
         if (name == null) {
             return false;
@@ -345,11 +360,85 @@
         throw new UnsupportedOperationException("A folder cannot be created in this configuration.");
     }
 
-    public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath,
-            KVMStoragePool destPool, ImageFormat format, int timeout) {
+    @Override
+    public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name,
+                                                         PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) {
         return null;
     }
 
+    @Override
+    public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath,
+                                                                KVMStoragePool destPool, ImageFormat format, int timeout) {
+        if (StringUtils.isEmpty(templateFilePath) || destPool == null) {
+            throw new CloudRuntimeException(
+                    "Unable to create template from direct download template file due to insufficient data");
+        }
+
+        File sourceFile = new File(templateFilePath);
+        if (!sourceFile.exists()) {
+            throw new CloudRuntimeException(
+                    "Direct download template file " + templateFilePath + " does not exist on this host");
+        }
+
+        if (!StoragePoolType.StorPool.equals(destPool.getType())) {
+            throw new CloudRuntimeException("Unsupported storage pool type: " + destPool.getType().toString());
+        }
+
+        if (!Storage.ImageFormat.QCOW2.equals(format)) {
+            throw new CloudRuntimeException("Unsupported template format: " + format.toString());
+        }
+
+        String srcTemplateFilePath = templateFilePath;
+        KVMPhysicalDisk destDisk = null;
+        QemuImgFile srcFile = null;
+        QemuImgFile destFile = null;
+        String templateName = UUID.randomUUID().toString();
+        String volume = null;
+        try {
+
+            srcTemplateFilePath = extractTemplate(templateFilePath, sourceFile, srcTemplateFilePath, templateName);
+
+            QemuImg.PhysicalDiskFormat srcFileFormat = QemuImg.PhysicalDiskFormat.QCOW2;
+
+            srcFile = new QemuImgFile(srcTemplateFilePath, srcFileFormat);
+
+            String spTemplate = destPool.getUuid().split(";")[0];
+
+            QemuImg qemu = new QemuImg(timeout);
+            OutputInterpreter.AllLinesParser parser = createStorPoolVolume(destPool, srcFile, qemu, spTemplate);
+
+            String response = parser.getLines();
+
+            LOGGER.debug(response);
+            volume = StorPoolUtil.devPath(getNameFromResponse(response, false, false));
+            attachOrDetachVolume("attach", "volume", volume);
+            destDisk = destPool.getPhysicalDisk(volume);
+            if (destDisk == null) {
+                throw new CloudRuntimeException(
+                        "Failed to find the disk: " + volume + " of the storage pool: " + destPool.getUuid());
+            }
+
+            destFile = new QemuImgFile(destDisk.getPath(), QemuImg.PhysicalDiskFormat.RAW);
+
+            qemu.convert(srcFile, destFile);
+            parser = volumeSnapshot(StorPoolStorageAdaptor.getVolumeNameFromPath(volume, true), spTemplate);
+            response = parser.getLines();
+            LOGGER.debug(response);
+            String newPath = StorPoolUtil.devPath(getNameFromResponse(response, false, true));
+            destDisk = destPool.getPhysicalDisk(newPath);
+        } catch (QemuImgException | LibvirtException e) {
+            destDisk = null;
+        } finally {
+            if (volume != null) {
+                attachOrDetachVolume("detach", "volume", volume);
+                volumeDelete(StorPoolStorageAdaptor.getVolumeNameFromPath(volume, true));
+            }
+            Script.runSimpleBashScript("rm -f " + srcTemplateFilePath);
+        }
+
+        return destDisk;
+    }
+
     @Override
     public boolean createFolder(String uuid, String path, String localPath) {
         return false;
@@ -367,9 +456,104 @@
         return null;
     }
 
-    @Override
-    public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name,
-            PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) {
-        return null;
+    private OutputInterpreter.AllLinesParser createStorPoolVolume(KVMStoragePool destPool, QemuImgFile srcFile,
+                                                                  QemuImg qemu, String templateUuid) throws QemuImgException, LibvirtException {
+        Map<String, String> info = qemu.info(srcFile);
+        Map<String, Object> reqParams = new HashMap<>();
+        reqParams.put("template", templateUuid);
+        reqParams.put("size", info.get("virtual_size"));
+        Map<String, String> tags = new HashMap<>();
+        tags.put("cs", "template");
+        reqParams.put("tags", tags);
+        Gson gson = new Gson();
+        String js = gson.toJson(reqParams);
+
+        Script sc = createStorPoolRequest(js, "VolumeCreate", null,true);
+        OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
+
+        String res = sc.execute(parser);
+        if (res != null) {
+            throw new CloudRuntimeException("Could not create volume due to: " + res);
+        }
+        return parser;
+    }
+
+    private  OutputInterpreter.AllLinesParser volumeSnapshot(String volumeName, String templateUuid) {
+        Map<String, String> reqParams = new HashMap<>();
+        reqParams.put("template", templateUuid);
+        Gson gson = new Gson();
+        String js = gson.toJson(reqParams);
+
+        Script sc = createStorPoolRequest(js, "VolumeSnapshot", volumeName,true);
+        OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
+
+        String res = sc.execute(parser);
+        if (res != null) {
+            throw new CloudRuntimeException("Could not snapshot volume due to: " + res);
+        }
+        return parser;
+    }
+
+    private OutputInterpreter.AllLinesParser volumeDelete(String volumeName) {
+        Script sc = createStorPoolRequest(null, "VolumeDelete", volumeName, false);
+        OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
+
+        String res = sc.execute(parser);
+        if (res != null) {
+            throw new CloudRuntimeException("Could not delete volume due to: " + res);
+        }
+        return parser;
+    }
+    @NotNull
+    private static Script createStorPoolRequest(String js, String apiCall, String param, boolean jsonRequired) {
+        Script sc = new Script("storpool_req", 0, LOGGER);
+        sc.add("-P");
+        sc.add("-M");
+        if (jsonRequired) {
+            sc.add("--json");
+            sc.add(js);
+        }
+        sc.add(apiCall);
+        if (param != null) {
+            sc.add(param);
+        }
+        return sc;
+    }
+
+    private String extractTemplate(String templateFilePath, File sourceFile, String srcTemplateFilePath,
+                                   String templateName) {
+        if (isTemplateExtractable(templateFilePath)) {
+            srcTemplateFilePath = sourceFile.getParent() + "/" + templateName;
+            String extractCommand = getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath);
+            Script.runSimpleBashScript(extractCommand);
+            Script.runSimpleBashScript("rm -f " + templateFilePath);
+        }
+        return srcTemplateFilePath;
+    }
+
+    private boolean isTemplateExtractable(String templatePath) {
+        String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'");
+        return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
+    }
+
+    private String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) {
+        if (downloadedTemplateFile.endsWith(".zip")) {
+            return "unzip -p " + downloadedTemplateFile + " | cat > " + templateFile;
+        } else if (downloadedTemplateFile.endsWith(".bz2")) {
+            return "bunzip2 -c " + downloadedTemplateFile + " > " + templateFile;
+        } else if (downloadedTemplateFile.endsWith(".gz")) {
+            return "gunzip -c " + downloadedTemplateFile + " > " + templateFile;
+        } else {
+            throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile);
+        }
+    }
+
+    private String getNameFromResponse(String resp, boolean tildeNeeded, boolean isSnapshot) {
+        JsonParser jsonParser = new JsonParser();
+        JsonObject respObj = (JsonObject) jsonParser.parse(resp);
+        JsonPrimitive data = isSnapshot ? respObj.getAsJsonPrimitive("snapshotGlobalId") : respObj.getAsJsonPrimitive("globalId");
+        String name = data !=null ? data.getAsString() : null;
+        name = name != null ? name.startsWith("~") && !tildeNeeded ? name.split("~")[1] : name : name;
+        return name;
     }
 }