Merge branch '4.20' into ghi9800-updateVrOnSettingChange

Author: DaanHoogland

.github/workflows/main-sonar-check.yml

@@ -54,7 +54,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-m2
 

.github/workflows/sonar-check.yml

@@ -56,7 +56,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-m2
 

api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java

@@ -18,6 +18,7 @@
 
 import org.apache.cloudstack.acl.ControlledEntity;
 
+import com.cloud.user.Account;
 import com.cloud.uservm.UserVm;
 import com.cloud.utils.component.Adapter;
 
@@ -26,4 +27,5 @@ public interface KubernetesServiceHelper extends Adapter {
     ControlledEntity findByUuid(String uuid);
     ControlledEntity findByVmId(long vmId);
     void checkVmCanBeDestroyed(UserVm userVm);
+    void cleanupForAccount(Account account);
 }

api/src/main/java/com/cloud/user/Account.java

@@ -71,6 +71,7 @@ public static Type getFromValue(Integer type){
     }
 
     public static final long ACCOUNT_ID_SYSTEM = 1;
+    public static final long ACCOUNT_ID_ADMIN = 2;
 
     public String getAccountName();
 

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -448,6 +448,7 @@ public class ApiConstants {
     public static final String SENT = "sent";
     public static final String SENT_BYTES = "sentbytes";
     public static final String SERIAL = "serial";
+    public static final String SERVICE_IP = "serviceip";
     public static final String SERVICE_OFFERING_ID = "serviceofferingid";
     public static final String SESSIONKEY = "sessionkey";
     public static final String SHOW_CAPACITIES = "showcapacities";

api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java

@@ -74,6 +74,11 @@ public class ManagementServerResponse extends BaseResponse {
     @Param(description = "the running OS kernel version for this Management Server")
     private String kernelVersion;
 
+    @Deprecated
+    @SerializedName(ApiConstants.SERVICE_IP)
+    @Param(description = "the IP Address for this Management Server. This is deprecated, please use 'ipaddress' instead.")
+    private String serviceIp;
+
     @SerializedName(ApiConstants.IP_ADDRESS)
     @Param(description = "the IP Address for this Management Server")
     private String ipAddress;
@@ -122,6 +127,10 @@ public Date getLastBoot() {
         return lastBoot;
     }
 
+    public String getServiceIp() {
+        return serviceIp;
+    }
+
     public String getIpAddress() {
         return ipAddress;
     }
@@ -170,6 +179,10 @@ public void setKernelVersion(String kernelVersion) {
         this.kernelVersion = kernelVersion;
     }
 
+    public void setServiceIp(String serviceIp) {
+        this.serviceIp = serviceIp;
+    }
+
     public void setIpAddress(String ipAddress) {
         this.ipAddress = ipAddress;
     }

api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java

@@ -79,6 +79,14 @@ public class UnmanagedInstanceResponse extends BaseResponse {
     @Param(description = "the operating system of the virtual machine")
     private String operatingSystem;
 
+    @SerializedName(ApiConstants.BOOT_MODE)
+    @Param(description = "indicates the boot mode")
+    private String bootMode;
+
+    @SerializedName(ApiConstants.BOOT_TYPE)
+    @Param(description = "indicates the boot type")
+    private String bootType;
+
     @SerializedName(ApiConstants.DISK)
     @Param(description = "the list of disks associated with the virtual machine", responseObject = UnmanagedInstanceDiskResponse.class)
     private Set<UnmanagedInstanceDiskResponse> disks;
@@ -211,4 +219,20 @@ public void setNics(Set<NicResponse> nics) {
     public void addNic(NicResponse nic) {
         this.nics.add(nic);
     }
+
+    public String getBootMode() {
+        return bootMode;
+    }
+
+    public void setBootMode(String bootMode) {
+        this.bootMode = bootMode;
+    }
+
+    public String getBootType() {
+        return bootType;
+    }
+
+    public void setBootType(String bootType) {
+        this.bootType = bootType;
+    }
 }

api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java

@@ -61,6 +61,9 @@ public enum PowerState {
 
     private String vncPassword;
 
+    private String bootType;
+    private String bootMode;
+
     public String getName() {
         return name;
     }
@@ -196,6 +199,22 @@ public String toString() {
                         this, "name", "internalCSName", "hostName", "clusterName"));
     }
 
+    public String getBootType() {
+        return bootType;
+    }
+
+    public void setBootType(String bootType) {
+        this.bootType = bootType;
+    }
+
+    public String getBootMode() {
+        return bootMode;
+    }
+
+    public void setBootMode(String bootMode) {
+        this.bootMode = bootMode;
+    }
+
     public static class Disk {
         private String diskId;
 

core/src/main/java/com/cloud/agent/api/GetVmIpAddressCommand.java

@@ -24,11 +24,13 @@ public class GetVmIpAddressCommand extends Command {
     String vmName;
     String vmNetworkCidr;
     boolean windows = false;
+    String macAddress;
 
-    public GetVmIpAddressCommand(String vmName, String vmNetworkCidr, boolean windows) {
+    public GetVmIpAddressCommand(String vmName, String vmNetworkCidr, boolean windows, String macAddress) {
         this.vmName = vmName;
         this.windows = windows;
         this.vmNetworkCidr = vmNetworkCidr;
+        this.macAddress = macAddress;
     }
 
     @Override
@@ -47,4 +49,8 @@ public boolean isWindows(){
     public String getVmNetworkCidr() {
         return vmNetworkCidr;
     }
+
+    public String getMacAddress() {
+        return macAddress;
+    }
 }

core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java

@@ -27,6 +27,7 @@
 import java.io.RandomAccessFile;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
 
@@ -80,6 +81,18 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
     private ResourceType resourceType = ResourceType.TEMPLATE;
     private final HttpMethodRetryHandler myretryhandler;
     private boolean followRedirects = false;
+    private boolean isChunkedTransfer;
+
+    protected static final List<String> CUSTOM_HEADERS_FOR_CHUNKED_TRANSFER_SIZE = Arrays.asList(
+            "x-goog-stored-content-length",
+            "x-goog-meta-size",
+            "x-amz-meta-size",
+            "x-amz-meta-content-length",
+            "x-object-meta-size",
+            "x-original-content-length",
+            "x-oss-meta-content-length",
+            "x-file-size");
+    private static final long MIN_FORMAT_VERIFICATION_SIZE = 1024 * 1024;
 
     public HttpTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes,
             String user, String password, Proxy proxy, ResourceType resourceType) {
@@ -205,13 +218,11 @@ public long download(boolean resume, DownloadCompleteCallback callback) {
                  RandomAccessFile out = new RandomAccessFile(file, "rw");
             ) {
                 out.seek(localFileSize);
-
-                logger.info("Starting download from " + downloadUrl + " to " + toFile + " remoteSize=" + toHumanReadableSize(remoteSize) + " , max size=" + toHumanReadableSize(maxTemplateSizeInBytes));
-
-                if (copyBytes(file, in, out)) return 0;
-
+                logger.info("Starting download from {} to {} remoteSize={} , max size={}",downloadUrl, toFile,
+                        toHumanReadableSize(remoteSize), toHumanReadableSize(maxTemplateSizeInBytes));
+                boolean eof = copyBytes(file, in, out);
                 Date finish = new Date();
-                checkDowloadCompletion();
+                checkDownloadCompletion(eof);
                 downloadTime += finish.getTime() - start.getTime();
             } finally { /* in.close() and out.close() */ }
             return totalBytes;
@@ -237,28 +248,32 @@ public long download(boolean resume, DownloadCompleteCallback callback) {
     }
 
     private boolean copyBytes(File file, InputStream in, RandomAccessFile out) throws IOException {
-        int bytes;
-        byte[] block = new byte[CHUNK_SIZE];
+        byte[] buffer = new byte[CHUNK_SIZE];
         long offset = 0;
-        boolean done = false;
         VerifyFormat verifyFormat = new VerifyFormat(file);
         status = Status.IN_PROGRESS;
-        while (!done && status != Status.ABORTED && offset <= remoteSize) {
-            if ((bytes = in.read(block, 0, CHUNK_SIZE)) > -1) {
-                offset = writeBlock(bytes, out, block, offset);
-                if (!ResourceType.SNAPSHOT.equals(resourceType) &&
-                        !verifyFormat.isVerifiedFormat() &&
-                        (offset >= 1048576 || offset >= remoteSize)) { //let's check format after we get 1MB or full file
-                    verifyFormat.invoke();
-                }
-            } else {
-                done = true;
+        while (status != Status.ABORTED) {
+            int bytesRead = in.read(buffer, 0, CHUNK_SIZE);
+            if (bytesRead == -1) {
+                logger.debug("Reached EOF on input stream");
+                break;
+            }
+            offset = writeBlock(bytesRead, out, buffer, offset);
+            if (!ResourceType.SNAPSHOT.equals(resourceType)
+                    && !verifyFormat.isVerifiedFormat()
+                    && (offset >= MIN_FORMAT_VERIFICATION_SIZE || offset >= remoteSize)) {
+                verifyFormat.invoke();
+            }
+            if (offset >= remoteSize) {
+                logger.debug("Reached expected remote size limit: {} bytes", remoteSize);
+                break;
             }
         }
         out.getFD().sync();
-        return false;
+        return !Status.ABORTED.equals(status);
     }
 
+
     private long writeBlock(int bytes, RandomAccessFile out, byte[] block, long offset) throws IOException {
         out.write(block, 0, bytes);
         offset += bytes;
@@ -267,11 +282,13 @@ private long writeBlock(int bytes, RandomAccessFile out, byte[] block, long offs
         return offset;
     }
 
-    private void checkDowloadCompletion() {
+    private void checkDownloadCompletion(boolean eof) {
         String downloaded = "(incomplete download)";
-        if (totalBytes >= remoteSize) {
+        if (eof && ((totalBytes >= remoteSize) || (isChunkedTransfer && remoteSize == maxTemplateSizeInBytes))) {
             status = Status.DOWNLOAD_FINISHED;
-            downloaded = "(download complete remote=" + toHumanReadableSize(remoteSize) + " bytes)";
+            downloaded = "(download complete remote=" +
+                    (remoteSize == maxTemplateSizeInBytes ? toHumanReadableSize(remoteSize) : "unknown") +
+                    " bytes)";
         }
         errorString = "Downloaded " + toHumanReadableSize(totalBytes) + " bytes " + downloaded;
     }
@@ -293,18 +310,42 @@ private void checkAndSetDownloadSize() {
         }
     }
 
+    protected long getRemoteSizeForChunkedTransfer() {
+        for (String headerKey : CUSTOM_HEADERS_FOR_CHUNKED_TRANSFER_SIZE) {
+            Header header = request.getResponseHeader(headerKey);
+            if (header == null) {
+                continue;
+            }
+            try {
+                return Long.parseLong(header.getValue());
+            } catch (NumberFormatException ignored) {}
+        }
+        Header contentRangeHeader = request.getResponseHeader("Content-Range");
+        if (contentRangeHeader != null) {
+            String contentRange = contentRangeHeader.getValue();
+            if (contentRange != null && contentRange.contains("/")) {
+                String totalSize = contentRange.substring(contentRange.indexOf('/') + 1).trim();
+                return Long.parseLong(totalSize);
+            }
+        }
+        return 0;
+    }
+
     private boolean tryAndGetRemoteSize() {
         Header contentLengthHeader = request.getResponseHeader("content-length");
-        boolean chunked = false;
+        isChunkedTransfer = false;
         long reportedRemoteSize = 0;
         if (contentLengthHeader == null) {
             Header chunkedHeader = request.getResponseHeader("Transfer-Encoding");
-            if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
+            if (chunkedHeader != null && "chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
+                isChunkedTransfer = true;
+                reportedRemoteSize = getRemoteSizeForChunkedTransfer();
+                logger.debug("{} is using chunked transfer encoding, possible remote size: {}", downloadUrl,
+                        reportedRemoteSize);
+            } else {
                 status = Status.UNRECOVERABLE_ERROR;
                 errorString = " Failed to receive length of download ";
                 return false;
-            } else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
-                chunked = true;
             }
         } else {
             reportedRemoteSize = Long.parseLong(contentLengthHeader.getValue());
@@ -316,9 +357,11 @@ private boolean tryAndGetRemoteSize() {
                 return false;
             }
         }
-
         if (remoteSize == 0) {
             remoteSize = reportedRemoteSize;
+            if (remoteSize != 0) {
+                logger.debug("Remote size for {} found to be {}", downloadUrl, toHumanReadableSize(remoteSize));
+            }
         }
         return true;
     }