Primera pure patches & various small fixes

api/src/main/java/com/cloud/exception/StorageAccessException.java

@@ -26,7 +26,7 @@
 public class StorageAccessException extends RuntimeException {
     private static final long serialVersionUID = SerialVersionUID.StorageAccessException;
 
-    public StorageAccessException(String message) {
-        super(message);
+    public StorageAccessException(String message, Exception causer) {
+        super(message, causer);
     }
 }

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java

@@ -1827,7 +1827,7 @@ private Pair<VolumeVO, DataStore> recreateVolume(VolumeVO vol, VirtualMachinePro
                     try {
                         volService.grantAccess(volFactory.getVolume(newVol.getId()), host, destPool);
                     } catch (Exception e) {
-                        throw new StorageAccessException(String.format("Unable to grant access to the volume [%s] on host [%s].", newVolToString, host));
+                        throw new StorageAccessException(String.format("Unable to grant access to the volume [%s] on host [%s].", newVolToString, host), e);
                     }
                 }
 
@@ -1867,7 +1867,7 @@ protected void grantVolumeAccessToHostIfNeeded(PrimaryDataStore volumeStore, lon
         try {
             volService.grantAccess(volFactory.getVolume(volumeId), host, volumeStore);
         } catch (Exception e) {
-            throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host));
+            throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host), e);
         }
     }
 
@@ -1915,7 +1915,7 @@ public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws Sto
                         try {
                             volService.grantAccess(volFactory.getVolume(vol.getId()), host, store);
                         } catch (Exception e) {
-                            throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host));
+                            throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host), e);
                         }
                     } else {
                         grantVolumeAccessToHostIfNeeded(store, vol.getId(), host, volToString);

engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java

@@ -40,6 +40,7 @@
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
@@ -1533,6 +1534,16 @@ private void handleCreateVolumeFromTemplateBothOnStorageSystem(TemplateInfo temp
                 verifyFormat(templateInfo.getFormat());
             }
 
+            // this blurb handles the case where the storage system can clone a volume from a template
+            String canCloneVolumeFromTemplate = templateInfo.getDataStore().getDriver().getCapabilities().get("CAN_CLONE_VOLUME_FROM_TEMPLATE");
+            if (canCloneVolumeFromTemplate != null && canCloneVolumeFromTemplate.toLowerCase().equals("true")) {
+                DataStoreDriver driver = templateInfo.getDataStore().getDriver();
+                driver.createAsync(volumeInfo.getDataStore(), volumeInfo, null);
+                volumeInfo = _volumeDataFactory.getVolume(volumeInfo.getId(), volumeInfo.getDataStore());
+                driver.copyAsync(templateInfo, volumeInfo, null);
+                return;
+            }
+
             HostVO hostVO = null;
 
             final boolean computeClusterSupportsVolumeClone;
@@ -1640,7 +1651,7 @@ else if (volumeInfo.getFormat() == ImageFormat.OVA) {
                 errMsg = "Create volume from template failed: " + ex.getMessage();
             }
 
-            throw new CloudRuntimeException(errMsg);
+            throw new CloudRuntimeException(errMsg, ex);
         }
         finally {
             if (copyCmdAnswer == null) {
@@ -2633,7 +2644,7 @@ private void handleCreateTemplateFromManagedVolume(VolumeInfo volumeInfo, Templa
         catch (Exception ex) {
             errMsg = ex.getMessage();
 
-            throw new CloudRuntimeException(errMsg);
+            throw new CloudRuntimeException(errMsg, ex);
         }
         finally {
             if (copyCmdAnswer == null) {

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java

@@ -1035,7 +1035,7 @@ private void copyTemplateToManagedTemplateVolume(TemplateInfo srcTemplateInfo, T
             try {
                 grantAccess(templateOnPrimary, destHost, destPrimaryDataStore);
             } catch (Exception e) {
-                throw new StorageAccessException("Unable to grant access to template: " + templateOnPrimary.getId() + " on host: " + destHost.getId());
+                throw new StorageAccessException("Unable to grant access to template: " + templateOnPrimary.getId() + " on host: " + destHost.getId(), e);
             }
 
             templateOnPrimary.processEvent(Event.CopyingRequested);
@@ -1161,7 +1161,7 @@ private void createManagedVolumeCopyManagedTemplateAsync(VolumeInfo volumeInfo,
             try {
                 grantAccess(srcTemplateOnPrimary, destHost, destPrimaryDataStore);
             } catch (Exception e) {
-                throw new StorageAccessException("Unable to grant access to src template: " + srcTemplateOnPrimary.getId() + " on host: " + destHost.getId());
+                throw new StorageAccessException("Unable to grant access to src template: " + srcTemplateOnPrimary.getId() + " on host: " + destHost.getId(), e);
             }
 
             _volumeDetailsDao.addDetail(volumeInfo.getId(), volumeDetailKey, String.valueOf(templatePoolRef.getId()), false);
@@ -1406,7 +1406,7 @@ public TemplateInfo createManagedStorageTemplate(long srcTemplateId, long destDa
                 try {
                     grantAccess(templateOnPrimary, destHost, destPrimaryDataStore);
                 } catch (Exception e) {
-                    throw new StorageAccessException("Unable to grant access to template: " + templateOnPrimary.getId() + " on host: " + destHost.getId());
+                    throw new StorageAccessException("Unable to grant access to template: " + templateOnPrimary.getId() + " on host: " + destHost.getId(), e);
                 }
 
                 templateOnPrimary.processEvent(Event.CopyingRequested);

framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/CloudStackExtendedLifeCycle.java

@@ -71,7 +71,11 @@ public void startBeans() {
         with(new WithComponentLifeCycle() {
             @Override
             public void with(ComponentLifecycle lifecycle) {
-                lifecycle.start();
+                try {
+                    lifecycle.start();
+                } catch (Throwable e) {
+                    log.warn("Unable to start component: " + lifecycle.getName(), e);
+                }
 
                 if (lifecycle instanceof ManagementBean) {
                     ManagementBean mbean = (ManagementBean)lifecycle;
@@ -115,6 +119,9 @@ public void with(ComponentLifecycle lifecycle) {
                 } catch (ConfigurationException e) {
                     log.error("Failed to configure " +  lifecycle.getName(), e);
                     throw new CloudRuntimeException(e);
+                } catch (Throwable e) {
+                    log.error("Failed to configure " +  lifecycle.getName(), e);
+                    throw new CloudRuntimeException(e);
                 }
             }
         });

framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/registry/RegistryLifecycle.java

@@ -108,10 +108,15 @@ public void start() {
 
         while (iter.hasNext()) {
             Object next = iter.next();
-            if (registry.register(next)) {
-                log.debug("Registered " + next);
-            } else {
-                iter.remove();
+            try {
+                if (registry.register(next)) {
+                    log.debug("Registered " + next);
+                } else {
+                    log.warn("Bean registration failed for " + next.toString());
+                    iter.remove();
+                }
+            } catch (Throwable e) {
+                log.warn("Bean registration attempt resulted in an exception for " + next.toString(), e);
             }
         }
     }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java

@@ -184,6 +184,7 @@ public String toString() {
                 guestDef.append("<entry name='manufacturer'>Apache Software Foundation</entry>\n");
                 guestDef.append("<entry name='product'>CloudStack " + _type.toString() + " Hypervisor</entry>\n");
                 guestDef.append("<entry name='uuid'>" + _uuid + "</entry>\n");
+                guestDef.append("<entry name='serial'>" + _uuid + "</entry>\n");
                 guestDef.append("</system>\n");
                 guestDef.append("</sysinfo>\n");
 
@@ -222,7 +223,9 @@ public String toString() {
                         guestDef.append("<boot dev='" + bo + "'/>\n");
                     }
                 }
-                guestDef.append("<smbios mode='sysinfo'/>\n");
+                if (_arch == null || !_arch.equals("aarch64")) {
+                    guestDef.append("<smbios mode='sysinfo'/>\n");
+                }
                 guestDef.append("</os>\n");
                 if (iothreads) {
                     guestDef.append(String.format("<iothreads>%s</iothreads>", NUMBER_OF_IOTHREADS));

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetUnmanagedInstancesCommandWrapper.java

@@ -124,7 +124,10 @@ private UnmanagedInstanceTO getUnmanagedInstance(LibvirtComputingResource libvir
             instance.setName(domain.getName());
 
             instance.setCpuCores((int) LibvirtComputingResource.countDomainRunningVcpus(domain));
-            instance.setCpuSpeed(parser.getCpuTuneDef().getShares()/instance.getCpuCores());
+
+            if (parser.getCpuTuneDef() != null && instance.getCpuCores() != null) {
+                instance.setCpuSpeed(parser.getCpuTuneDef().getShares()/instance.getCpuCores());
+            }
 
             if (parser.getCpuModeDef() != null) {
                 instance.setCpuCoresPerSocket(parser.getCpuModeDef().getCoresPerSocket());

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java

@@ -272,8 +272,17 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) {
 
                 String path = derivePath(primaryStore, destData, details);
 
-                if (!storagePoolMgr.connectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path, details)) {
-                    s_logger.warn("Failed to connect physical disk at path: " + path + ", in storage pool id: " + primaryStore.getUuid());
+                if (path == null) {
+                    path = destTempl.getUuid();
+                }
+
+                if (path != null && !storagePoolMgr.connectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path, details)) {
+                    if (primaryStore.getPoolType() == StoragePoolType.FiberChannel) {
+                        s_logger.warn("Failed to connect physical disk at path: " + path + ", in storage pool id: " + primaryStore.getUuid());
+                        return new PrimaryStorageDownloadAnswer("Failed to spool template disk at path: " + path + ", in storage pool id: " + primaryStore.getUuid());
+                    } else {
+                        s_logger.warn("Failed to connect physical disk at path: " + path + ", in storage pool id: " + primaryStore.getUuid());
+                    }
                 }
 
                 primaryVol = storagePoolMgr.copyPhysicalDisk(tmplVol, path != null ? path : destTempl.getUuid(), primaryPool, cmd.getWaitInMillSeconds());
@@ -337,6 +346,7 @@ private String derivePath(PrimaryDataStoreTO primaryStore, DataTO destData, Map<
         } else {
             path = details != null ? details.get("managedStoreTarget") : null;
         }
+
         return path;
     }
 
@@ -417,7 +427,7 @@ public Answer cloneVolumeFromBaseTemplate(final CopyCommand cmd) {
             if (primaryPool.getType() == StoragePoolType.CLVM) {
                 templatePath = ((NfsTO)imageStore).getUrl() + File.separator + templatePath;
                 vol = templateToPrimaryDownload(templatePath, primaryPool, volume.getUuid(), volume.getSize(), cmd.getWaitInMillSeconds());
-            } if (primaryPool.getType() == StoragePoolType.PowerFlex) {
+            } if (primaryPool.getType() == StoragePoolType.PowerFlex || primaryPool.getType() == StoragePoolType.FiberChannel) {
                 Map<String, String> details = primaryStore.getDetails();
                 String path = derivePath(primaryStore, destData, details);
 
@@ -771,15 +781,19 @@ else if (srcData instanceof SnapshotObjectTO) {
 
         KVMStoragePool secondaryStorage = null;
 
+        String path = null;
         try {
             // look for options indicating an overridden path or IQN.  Used when snapshots have to be
             // temporarily copied on the manaaged storage device before the actual copy to target object
             Map<String, String> details = cmd.getOptions();
-            String path = details != null ? details.get(DiskTO.PATH) : null;
+            path = details != null ? details.get(DiskTO.PATH) : null;
             if (path == null) {
                 path = details != null ? details.get(DiskTO.IQN) : null;
                 if (path == null) {
-                    new CloudRuntimeException("The 'path' or 'iqn' field must be specified.");
+                    path = srcData.getPath();
+                    if (path == null) {
+                        new CloudRuntimeException("The 'path' or 'iqn' field must be specified.");
+                    }
                 }
             }
 
@@ -842,8 +856,6 @@ else if (srcData instanceof SnapshotObjectTO) {
             loc.addFormat(info);
             loc.save();
 
-            storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path);
-
             TemplateObjectTO newTemplate = new TemplateObjectTO();
 
             newTemplate.setPath(templateFolder + File.separator + templateName + ".qcow2");
@@ -863,6 +875,10 @@ else if (srcData instanceof SnapshotObjectTO) {
 
             return new CopyCmdAnswer(ex.toString());
         } finally {
+            if (path != null) {
+                storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path);
+            }
+
             if (secondaryStorage != null) {
                 secondaryStorage.delete();
             }
@@ -1038,7 +1054,9 @@ public Answer backupSnapshot(final CopyCommand cmd) {
                 command.add(NAME_OPTION, snapshotName);
                 command.add("-p", snapshotDestPath);
 
-                descName = UUID.randomUUID().toString();
+                if (isCreatedFromVmSnapshot) {
+                    descName = UUID.randomUUID().toString();
+                }
 
                 command.add("-t", descName);
                 final String result = command.execute();

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java

@@ -160,6 +160,13 @@ private KVMPhysicalDisk getPhysicalDisk(AddressInfo address, KVMStoragePool pool
         KVMPhysicalDisk disk = new KVMPhysicalDisk(address.getPath(), address.toString(), pool);
         disk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
 
+        // validate we have a connection, if not we need to connect first.
+        if (!isConnected(address.getPath())) {
+            if (!connectPhysicalDisk(address, pool, null)) {
+                throw new CloudRuntimeException("Unable to connect to volume " + address.getPath());
+            }
+        }
+
         long diskSize = getPhysicalDiskSize(address.getPath());
         disk.setSize(diskSize);
         disk.setVirtualSize(diskSize);
@@ -197,6 +204,10 @@ public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool, Map<S
         // we expect WWN values in the volumePath so need to convert it to an actual physical path
         AddressInfo address = this.parseAndValidatePath(volumePath);
 
+        return connectPhysicalDisk(address, pool, details);
+    }
+
+    private boolean connectPhysicalDisk(AddressInfo address, KVMStoragePool pool, Map<String, String> details) {
         // validate we have a connection id - we can't proceed without that
         if (address.getConnectionId() == null) {
             LOGGER.error("Unable to connect volume with address [" + address.getPath() + "] of the storage pool: " + pool.getUuid() + " - connection id is not set in provided path");
@@ -508,6 +519,18 @@ boolean waitForDiskToBecomeAvailable(AddressInfo address, KVMStoragePool pool, l
         return false;
     }
 
+    boolean isConnected(String path) {
+        // run a command to test if this is a binary device at this path
+        Script blockTest = new Script("/bin/test", LOGGER);
+        blockTest.add("-b", path);
+        blockTest.execute();
+        int rc = blockTest.getExitValue();
+        if (rc == 0) {
+            return true;
+        }
+        return false;
+    }
+
     long getPhysicalDiskSize(String diskPath) {
         if (StringUtils.isEmpty(diskPath)) {
             return 0;

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java

@@ -362,6 +362,11 @@ protected IpAddress getVpcTierKubernetesPublicIp(Network network) {
         IpAddress address = ipAddressDao.findByUuid(detailsVO.getValue());
         if (address == null || network.getVpcId() != address.getVpcId()) {
             LOGGER.warn(String.format("Public IP with ID: %s linked to the Kubernetes cluster: %s is not usable", detailsVO.getValue(), kubernetesCluster.getName()));
+            if (address == null) {
+                LOGGER.warn(String.format("Public IP with ID: %s was not found by uuid", detailsVO.getValue()));
+            } else {
+                LOGGER.warn(String.format("Public IP with ID: %s was associated with vpc %d instead of %d", detailsVO.getValue(), address.getVpcId().longValue(), network.getVpcId().longValue()));
+            }
             return null;
         }
         return address;

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java

@@ -192,7 +192,7 @@ public static String getKubernetesClusterConfig(final KubernetesCluster kubernet
         while (System.currentTimeMillis() < timeoutTime) {
             try {
                 Pair<Boolean, String> result = SshHelper.sshExecute(ipAddress, port, user,
-                        sshKeyFile, null, "sudo cat /etc/kubernetes/admin.conf",
+                        sshKeyFile, null, "sudo cat /etc/kubernetes/user.conf 2>/dev/null || sudo cat /etc/kubernetes/admin.conf",
                         10000, 10000, 10000);
 
                 if (result.first() && StringUtils.isNotEmpty(result.second())) {