Merge branch '4.20' into fix-vmware-ip-loss

Author: DaanHoogland

core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java

@@ -27,6 +27,7 @@
 import java.io.RandomAccessFile;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
 
@@ -80,6 +81,18 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
     private ResourceType resourceType = ResourceType.TEMPLATE;
     private final HttpMethodRetryHandler myretryhandler;
     private boolean followRedirects = false;
+    private boolean isChunkedTransfer;
+
+    protected static final List<String> CUSTOM_HEADERS_FOR_CHUNKED_TRANSFER_SIZE = Arrays.asList(
+            "x-goog-stored-content-length",
+            "x-goog-meta-size",
+            "x-amz-meta-size",
+            "x-amz-meta-content-length",
+            "x-object-meta-size",
+            "x-original-content-length",
+            "x-oss-meta-content-length",
+            "x-file-size");
+    private static final long MIN_FORMAT_VERIFICATION_SIZE = 1024 * 1024;
 
     public HttpTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes,
             String user, String password, Proxy proxy, ResourceType resourceType) {
@@ -205,13 +218,11 @@ public long download(boolean resume, DownloadCompleteCallback callback) {
                  RandomAccessFile out = new RandomAccessFile(file, "rw");
             ) {
                 out.seek(localFileSize);
-
-                logger.info("Starting download from " + downloadUrl + " to " + toFile + " remoteSize=" + toHumanReadableSize(remoteSize) + " , max size=" + toHumanReadableSize(maxTemplateSizeInBytes));
-
-                if (copyBytes(file, in, out)) return 0;
-
+                logger.info("Starting download from {} to {} remoteSize={} , max size={}",downloadUrl, toFile,
+                        toHumanReadableSize(remoteSize), toHumanReadableSize(maxTemplateSizeInBytes));
+                boolean eof = copyBytes(file, in, out);
                 Date finish = new Date();
-                checkDowloadCompletion();
+                checkDownloadCompletion(eof);
                 downloadTime += finish.getTime() - start.getTime();
             } finally { /* in.close() and out.close() */ }
             return totalBytes;
@@ -237,28 +248,32 @@ public long download(boolean resume, DownloadCompleteCallback callback) {
     }
 
     private boolean copyBytes(File file, InputStream in, RandomAccessFile out) throws IOException {
-        int bytes;
-        byte[] block = new byte[CHUNK_SIZE];
+        byte[] buffer = new byte[CHUNK_SIZE];
         long offset = 0;
-        boolean done = false;
         VerifyFormat verifyFormat = new VerifyFormat(file);
         status = Status.IN_PROGRESS;
-        while (!done && status != Status.ABORTED && offset <= remoteSize) {
-            if ((bytes = in.read(block, 0, CHUNK_SIZE)) > -1) {
-                offset = writeBlock(bytes, out, block, offset);
-                if (!ResourceType.SNAPSHOT.equals(resourceType) &&
-                        !verifyFormat.isVerifiedFormat() &&
-                        (offset >= 1048576 || offset >= remoteSize)) { //let's check format after we get 1MB or full file
-                    verifyFormat.invoke();
-                }
-            } else {
-                done = true;
+        while (status != Status.ABORTED) {
+            int bytesRead = in.read(buffer, 0, CHUNK_SIZE);
+            if (bytesRead == -1) {
+                logger.debug("Reached EOF on input stream");
+                break;
+            }
+            offset = writeBlock(bytesRead, out, buffer, offset);
+            if (!ResourceType.SNAPSHOT.equals(resourceType)
+                    && !verifyFormat.isVerifiedFormat()
+                    && (offset >= MIN_FORMAT_VERIFICATION_SIZE || offset >= remoteSize)) {
+                verifyFormat.invoke();
+            }
+            if (offset >= remoteSize) {
+                logger.debug("Reached expected remote size limit: {} bytes", remoteSize);
+                break;
             }
         }
         out.getFD().sync();
-        return false;
+        return !Status.ABORTED.equals(status);
     }
 
+
     private long writeBlock(int bytes, RandomAccessFile out, byte[] block, long offset) throws IOException {
         out.write(block, 0, bytes);
         offset += bytes;
@@ -267,11 +282,13 @@ private long writeBlock(int bytes, RandomAccessFile out, byte[] block, long offs
         return offset;
     }
 
-    private void checkDowloadCompletion() {
+    private void checkDownloadCompletion(boolean eof) {
         String downloaded = "(incomplete download)";
-        if (totalBytes >= remoteSize) {
+        if (eof && ((totalBytes >= remoteSize) || (isChunkedTransfer && remoteSize == maxTemplateSizeInBytes))) {
             status = Status.DOWNLOAD_FINISHED;
-            downloaded = "(download complete remote=" + toHumanReadableSize(remoteSize) + " bytes)";
+            downloaded = "(download complete remote=" +
+                    (remoteSize == maxTemplateSizeInBytes ? toHumanReadableSize(remoteSize) : "unknown") +
+                    " bytes)";
         }
         errorString = "Downloaded " + toHumanReadableSize(totalBytes) + " bytes " + downloaded;
     }
@@ -293,18 +310,42 @@ private void checkAndSetDownloadSize() {
         }
     }
 
+    protected long getRemoteSizeForChunkedTransfer() {
+        for (String headerKey : CUSTOM_HEADERS_FOR_CHUNKED_TRANSFER_SIZE) {
+            Header header = request.getResponseHeader(headerKey);
+            if (header == null) {
+                continue;
+            }
+            try {
+                return Long.parseLong(header.getValue());
+            } catch (NumberFormatException ignored) {}
+        }
+        Header contentRangeHeader = request.getResponseHeader("Content-Range");
+        if (contentRangeHeader != null) {
+            String contentRange = contentRangeHeader.getValue();
+            if (contentRange != null && contentRange.contains("/")) {
+                String totalSize = contentRange.substring(contentRange.indexOf('/') + 1).trim();
+                return Long.parseLong(totalSize);
+            }
+        }
+        return 0;
+    }
+
     private boolean tryAndGetRemoteSize() {
         Header contentLengthHeader = request.getResponseHeader("content-length");
-        boolean chunked = false;
+        isChunkedTransfer = false;
         long reportedRemoteSize = 0;
         if (contentLengthHeader == null) {
             Header chunkedHeader = request.getResponseHeader("Transfer-Encoding");
-            if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
+            if (chunkedHeader != null && "chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
+                isChunkedTransfer = true;
+                reportedRemoteSize = getRemoteSizeForChunkedTransfer();
+                logger.debug("{} is using chunked transfer encoding, possible remote size: {}", downloadUrl,
+                        reportedRemoteSize);
+            } else {
                 status = Status.UNRECOVERABLE_ERROR;
                 errorString = " Failed to receive length of download ";
                 return false;
-            } else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
-                chunked = true;
             }
         } else {
             reportedRemoteSize = Long.parseLong(contentLengthHeader.getValue());
@@ -316,9 +357,11 @@ private boolean tryAndGetRemoteSize() {
                 return false;
             }
         }
-
         if (remoteSize == 0) {
             remoteSize = reportedRemoteSize;
+            if (remoteSize != 0) {
+                logger.debug("Remote size for {} found to be {}", downloadUrl, toHumanReadableSize(remoteSize));
+            }
         }
         return true;
     }

engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql

@@ -54,6 +54,59 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.storage_pool', 'used_iops', 'bigint
 -- Add reason column for op_ha_work
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.op_ha_work', 'reason', 'varchar(32) DEFAULT NULL COMMENT "Reason for the HA work"');
 
+-- Support for XCP-ng 8.3.0 and XenServer 8.4 by adding hypervisor capabilities
+-- https://docs.xenserver.com/en-us/xenserver/8/system-requirements/configuration-limits.html
+-- https://docs.xenserver.com/en-us/citrix-hypervisor/system-requirements/configuration-limits.html
+INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported) VALUES (UUID(), 'XenServer', '8.3.0', 1000, 254, 64, 1);
+INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported) VALUES (UUID(), 'XenServer', '8.4.0', 1000, 240, 64, 1);
+
+-- Add missing and new Guest OS mappings
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (2, 'Debian GNU/Linux 10 (64-bit)', 'XenServer', '8.2.1', 'Debian Buster 10');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (5, 'SUSE Linux Enterprise Server 15 (64-bit)', 'XenServer', '8.2.1', 'SUSE Linux Enterprise 15 (64-bit)');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'XenServer', '8.2.1', 'Windows Server 2022 (64-bit)');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows 11 (64-bit)', 'XenServer', '8.2.1', 'Windows 11');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (10, 'Ubuntu 20.04 LTS', 'XenServer', '8.2.1', 'Ubuntu Focal Fossa 20.04');
+
+-- Copy XS 8.2.1 hypervisor guest OS mappings to XS 8.3 and 8.3 mappings to 8.4
+INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', '8.3.0', guest_os_name, guest_os_id, utc_timestamp(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='8.2.1';
+
+-- Add new and missing guest os mappings for XS 8.3
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (1, 'Rocky Linux 9', 'XenServer', '8.3.0', 'Rocky Linux 9');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (1, 'Rocky Linux 8', 'XenServer', '8.3.0', 'Rocky Linux 8');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (1, 'AlmaLinux 9', 'XenServer', '8.3.0', 'AlmaLinux 9');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (1, 'AlmaLinux 8', 'XenServer', '8.3.0', 'AlmaLinux 8');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (2, 'Debian GNU/Linux 12 (64-bit)', 'XenServer', '8.3.0', 'Debian Bookworm 12');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (3, 'Oracle Linux 9', 'XenServer', '8.3.0', 'Oracle Linux 9');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (3, 'Oracle Linux 8', 'XenServer', '8.3.0', 'Oracle Linux 8');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (4, 'Red Hat Enterprise Linux 8.0', 'XenServer', '8.3.0', 'Red Hat Enterprise Linux 8');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (4, 'Red Hat Enterprise Linux 9.0', 'XenServer', '8.3.0', 'Red Hat Enterprise Linux 9');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (10, 'Ubuntu 22.04 LTS', 'XenServer', '8.3.0', 'Ubuntu Jammy Jellyfish 22.04');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (5, 'SUSE Linux Enterprise Server 12 SP5 (64-bit)', 'XenServer', '8.3.0', 'SUSE Linux Enterprise Server 12 SP5 (64-bit');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (4, 'NeoKylin Linux Server 7', 'XenServer', '8.3.0', 'NeoKylin Linux Server 7');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (1, 'CentOS Stream 9', 'XenServer', '8.3.0', 'CentOS Stream 9');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (4, 'Scientific Linux 7', 'XenServer', '8.3.0', 'Scientific Linux 7');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (7, 'Generic Linux UEFI', 'XenServer', '8.3.0', 'Generic Linux UEFI');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (7, 'Generic Linux BIOS', 'XenServer', '8.3.0', 'Generic Linux BIOS');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (2, 'Gooroom Platform 2.0', 'XenServer', '8.3.0', 'Gooroom Platform 2.0');
+
+INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', '8.4.0', guest_os_name, guest_os_id, utc_timestamp(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='8.3.0';
+
+-- Add new guest os mappings for XS 8.4 and KVM
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2025', 'XenServer', '8.4.0', 'Windows Server 2025');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (10, 'Ubuntu 24.04 LTS', 'XenServer', '8.4.0', 'Ubuntu Noble Numbat 24.04');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (2, 'Debian GNU/Linux 10 (64-bit)', 'KVM', 'default', 'Debian GNU/Linux 10 (64-bit)');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (2, 'Debian GNU/Linux 11 (64-bit)', 'KVM', 'default', 'Debian GNU/Linux 11 (64-bit)');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (2, 'Debian GNU/Linux 12 (64-bit)', 'KVM', 'default', 'Debian GNU/Linux 12 (64-bit)');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows 11 (64-bit)', 'KVM', 'default', 'Windows 11');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2025', 'KVM', 'default', 'Windows Server 2025');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (10, 'Ubuntu 24.04 LTS', 'KVM', 'default', 'Ubuntu 24.04 LTS');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (1, 'CentOS Stream 10 (preview)', 'XenServer', '8.4.0', 'CentOS Stream 10 (preview)');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (1, 'CentOS Stream 9', 'XenServer', '8.4.0', 'CentOS Stream 9');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (4, 'Scientific Linux 7', 'XenServer', '8.4.0', 'Scientific Linux 7');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (4, 'NeoKylin Linux Server 7', 'XenServer', '8.4.0', 'NeoKylin Linux Server 7');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (5, 'SUSE Linux Enterprise Server 12 SP5 (64-bit)', 'XenServer', '8.4.0', 'SUSE Linux Enterprise Server 12 SP5 (64-bit');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (2, 'Gooroom Platform 2.0', 'XenServer', '8.4.0', 'Gooroom Platform 2.0');
+
 -- Grant access to 2FA APIs for the "Read-Only User - Default" role
 
 CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Read-Only User - Default', 'setupUserTwoFactorAuthentication', 'ALLOW');

engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java

@@ -126,7 +126,26 @@ public static String buildConfigDrive(List<NicProfile> nics, List<String[]> vmDa
 
             File openStackFolder = new File(tempDirName + ConfigDrive.openStackConfigDriveName);
 
-            writeVendorEmptyJsonFile(openStackFolder);
+            /*
+            Try to find VM password in the vmData.
+            If it is found, then write it into vendor-data.json
+            */
+            String vmPassword = "";
+            for (String[] item : vmData) {
+                String dataType = item[CONFIGDATA_DIR];
+                String fileName = item[CONFIGDATA_FILE];
+                String content = item[CONFIGDATA_CONTENT];
+                if (PASSWORD_FILE.equals(fileName)) {
+                    vmPassword = content;
+                    break;
+                }
+            }
+            if (vmPassword.equals("")) {
+                writeVendorDataJsonFile(openStackFolder);
+            } else {
+                writeVendorDataJsonFile(openStackFolder, vmPassword);
+            }
+
             writeNetworkData(nics, supportedServices, openStackFolder);
             for (NicProfile nic: nics) {
                 if (supportedServices.get(nic.getId()).contains(Network.Service.UserData)) {
@@ -253,14 +272,34 @@ static boolean needForGeneratingNetworkData(Map<Long, List<Network.Service>> sup
      *
      *  If the folder does not exist, and we cannot create it, we throw a {@link CloudRuntimeException}.
      */
-    static void writeVendorEmptyJsonFile(File openStackFolder) {
+    static void writeVendorDataJsonFile(File openStackFolder) {
         if (openStackFolder.exists() || openStackFolder.mkdirs()) {
             writeFile(openStackFolder, "vendor_data.json", "{}");
         } else {
             throw new CloudRuntimeException("Failed to create folder " + openStackFolder);
         }
     }
 
+    /**
+     *  Writes vendor data containing Cloudstack-generated password into vendor-data.json
+     *
+     *  If the folder does not exist, and we cannot create it, we throw a {@link CloudRuntimeException}.
+     */
+    static void writeVendorDataJsonFile(File openStackFolder, String password) {
+        if (openStackFolder.exists() || openStackFolder.mkdirs()) {
+            writeFile(
+                openStackFolder,
+                "vendor_data.json",
+                String.format(
+                    "{\"cloud-init\": \"#cloud-config\\npassword: %s\\nchpasswd:\\n  expire: False\"}",
+                    password
+                )
+            );
+        } else {
+            throw new CloudRuntimeException("Failed to create folder " + openStackFolder);
+        }
+    }
+
     /**
      * Creates the {@link JsonObject} with VM's metadata. The vmData is a list of arrays; we expect this list to have the following entries:
      * <ul>