systemvm-template: support on-demand download during setup and registration

client/bindir/cloud-setup-management.in

@@ -36,6 +36,106 @@ from cloudutils.cloudException import CloudRuntimeException, CloudInternalExcept
 from cloudutils.globalEnv import globalEnv
 from cloudutils.serviceConfigServer import cloudManagementConfig
 from optparse import OptionParser
+import urllib.request
+import configparser
+import hashlib
+
+SYSTEMVM_TEMPLATES_PATH = "/usr/share/cloudstack-management/templates/systemvm"
+SYSTEMVM_TEMPLATES_METADATA_FILE = SYSTEMVM_TEMPLATES_PATH + "/metadata.ini"
+
+def verify_sha512_checksum(file_path, expected_checksum):
+    sha512 = hashlib.sha512()
+    try:
+        with open(file_path, "rb") as f:
+            for chunk in iter(lambda: f.read(8192), b""):
+                sha512.update(chunk)
+        return sha512.hexdigest().lower() == expected_checksum.lower()
+    except Exception as e:
+        print(f"Failed to verify checksum for {file_path}: {e}")
+        return False
+
+def download_file(url, dest_path, chunk_size=8 * 1024 * 1024):
+    """
+    Downloads a file from the given URL to the specified destination path in chunks.
+    """
+    try:
+        with urllib.request.urlopen(url) as response:
+            total_size = response.length if response.length else None
+            downloaded = 0
+            try:
+                with open(dest_path, 'wb') as out_file:
+                    while True:
+                        chunk = response.read(chunk_size)
+                        if not chunk:
+                            break
+                        out_file.write(chunk)
+                        downloaded += len(chunk)
+                        if total_size:
+                            print(f"Downloaded {downloaded / (1024 * 1024):.2f}MB of {total_size / (1024 * 1024):.2f}MB", end='\r')
+            except PermissionError as pe:
+                print(f"Permission denied: {dest_path}")
+                raise
+        print(f"\nDownloaded file from {url} to {dest_path}")
+    except Exception as e:
+        print(f"Failed to download file: {e}")
+        raise
+
+def download_template_if_needed(template, url, filename, checksum):
+    dest_path = os.path.join(SYSTEMVM_TEMPLATES_PATH, filename)
+    if os.path.exists(dest_path):
+        if checksum and verify_sha512_checksum(dest_path, checksum):
+            print(f"{template} System VM template already exists at {dest_path} with valid checksum, skipping download.")
+            return
+        else:
+            print(f"{template} System VM template at {dest_path} has invalid or missing checksum, re-downloading...")
+    else:
+        print(f"Downloading {template} System VM template from {url} to {dest_path}...")
+    try:
+        download_file(url, dest_path)
+        #After download, verify checksum if provided
+        if checksum:
+            if verify_sha512_checksum(dest_path, checksum):
+                print(f"{template} System VM template downloaded and verified successfully.")
+            else:
+                print(f"ERROR: Checksum verification failed for {template} System VM template after download.")
+    except Exception as e:
+        print(f"ERROR: Failed to download {template} System VM template: {e}")
+
+def collect_template_metadata(selected_templates, options):
+    template_metadata_list = []
+    if not os.path.exists(SYSTEMVM_TEMPLATES_METADATA_FILE):
+        print(f"ERROR: System VM templates metadata file not found at {SYSTEMVM_TEMPLATES_METADATA_FILE}, cannot download templates.")
+        sys.exit(1)
+    config = configparser.ConfigParser()
+    config.read(SYSTEMVM_TEMPLATES_METADATA_FILE)
+    template_repo_url = None
+    if options.systemvm_templates_repository:
+        if "default" in config and "downloadrepository" in config["default"]:
+            template_repo_url = config["default"]["downloadrepository"].strip()
+        if not template_repo_url:
+            print("ERROR: downloadrepository value is empty in metadata file, cannot use --systemvm-template-repository option.")
+            sys.exit(1)
+    for template in selected_templates:
+        if template in config:
+            url = config[template].get("downloadurl")
+            filename = config[template].get("filename")
+            checksum = config[template].get("checksum")
+            if url and filename:
+                if template_repo_url:
+                    url = url.replace(template_repo_url, options.systemvm_templates_repository)
+                template_metadata_list.append({
+                    "template": template,
+                    "url": url,
+                    "filename": filename,
+                    "checksum": checksum
+                })
+            else:
+                print(f"ERROR: URL or filename not found for {template} System VM template in metadata.")
+                sys.exit(1)
+        else:
+            print(f"ERROR: No metadata found for {template} System VM template.")
+            sys.exit(1)
+    return template_metadata_list
 
 if __name__ == '__main__':
     initLoging("@MSLOGDIR@/setupManagement.log")
@@ -45,6 +145,16 @@ if __name__ == '__main__':
     parser.add_option("--https", action="store_true", dest="https", help="Enable HTTPs connection of management server")
     parser.add_option("--tomcat7", action="store_true", dest="tomcat7", help="Depreciated option, don't use it")
     parser.add_option("--no-start", action="store_true", dest="nostart", help="Do not start management server after successful configuration")
+    parser.add_option(
+        "--systemvm-templates",
+        dest="systemvm_templates",
+        help="Specify System VM templates to download: all, kvm-aarch64, kvm-x86_64, xenserver, vmware or comma-separated list of hypervisor combinations (e.g., kvm-x86_64,xenserver). Default is kvm-x86_64.",
+    )
+    parser.add_option(
+        "--systemvm-templates-repository",
+        dest="systemvm_templates_repository",
+        help="Specify the URL to download System VM templates from."
+    )
     (options, args) = parser.parse_args()
     if options.https:
         glbEnv.svrMode = "HttpsServer"
@@ -53,6 +163,34 @@ if __name__ == '__main__':
     if options.nostart:
         glbEnv.noStart = True
 
+    available_templates = ["kvm-aarch64", "kvm-x86_64", "xenserver", "vmware"]
+    templates_arg = options.systemvm_templates
+
+    selected_templates = ["kvm-x86_64"]
+    if templates_arg:
+        templates_list = [t.strip().lower() for t in templates_arg.split(",")]
+        if "all" in templates_list:
+            if len(templates_list) > 1:
+                print("WARNING: 'all' specified for System VM templates, ignoring other specified templates.")
+            selected_templates = available_templates
+        else:
+            invalid_templates = []
+            for t in templates_list:
+                if t in available_templates:
+                    if t not in selected_templates:
+                        selected_templates.append(t)
+                else:
+                    if t not in invalid_templates:
+                        invalid_templates.append(t)
+            if invalid_templates:
+                print(f"ERROR: Invalid System VM template names provided: {', '.join(invalid_templates)}")
+                sys.exit(1)
+    print(f"Selected systemvm templates to download: {', '.join(selected_templates) if selected_templates else 'None'}")
+
+    template_metadata_list = []
+    if selected_templates:
+        template_metadata_list = collect_template_metadata(selected_templates, options)
+
     glbEnv.mode = "Server"
 
     print("Starting to configure CloudStack Management Server:")
@@ -74,3 +212,6 @@ if __name__ == '__main__':
             syscfg.restore()
         except:
             pass
+
+    for meta in template_metadata_list:
+        download_template_if_needed(meta["template"], meta["url"], meta["filename"], meta["checksum"])

client/conf/server.properties.in

@@ -62,3 +62,8 @@ extensions.deployment.mode=@EXTENSIONSDEPLOYMENTMODE@
 # Thread pool configuration
 #threads.min=10
 #threads.max=500
+
+# The URL prefix for the system VM templates repository. When downloading system VM templates, the server replaces the
+# `downloadrepository` key value from the metadata file in template URLs. If not specified, the original template URL
+# will be used for download.
+# system.vm.templates.download.repository=http://download.cloudstack.org/systemvm/

engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java

@@ -51,6 +51,7 @@
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 import org.apache.cloudstack.utils.security.DigestHelper;
+import org.apache.cloudstack.utils.server.ServerPropertiesUtil;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -108,7 +109,11 @@ public class SystemVmTemplateRegistration {
     private static Integer LINUX_12_ID = 363;
     private static final Integer SCRIPT_TIMEOUT = 1800000;
     private static final Integer LOCK_WAIT_TIMEOUT = 1200;
+    protected static final String TEMPLATE_DOWNLOAD_URL_KEY = "downloadurl";
+    protected static final String TEMPLATES_DOWNLOAD_REPOSITORY_KEY = "downloadrepository";
+    protected static final String TEMPLATES_CUSTOM_DOWNLOAD_REPOSITORY_KEY = "system.vm.templates.download.repository";
     protected static final List<CPU.CPUArch> DOWNLOADABLE_TEMPLATE_ARCH_TYPES = Arrays.asList(
+            CPU.CPUArch.amd64,
             CPU.CPUArch.arm64
     );
 
@@ -820,6 +825,11 @@ public static String parseMetadataFile() {
             LOGGER.error(errMsg);
             throw new CloudRuntimeException(errMsg);
         }
+        Ini.Section defaultSection = ini.get("default");
+        String defaultDownloadRepository = defaultSection.get(TEMPLATES_DOWNLOAD_REPOSITORY_KEY);
+        String customDownloadRepository = ServerPropertiesUtil.getProperty(TEMPLATES_CUSTOM_DOWNLOAD_REPOSITORY_KEY);
+        boolean updateCustomDownloadRepository = StringUtils.isNotBlank(customDownloadRepository) &&
+                StringUtils.isNotBlank(defaultDownloadRepository);
         for (Pair<Hypervisor.HypervisorType, CPU.CPUArch> hypervisorType : hypervisorList) {
             String key = getHypervisorArchKey(hypervisorType.first(), hypervisorType.second());
             Ini.Section section = ini.get(key);
@@ -828,16 +838,21 @@ public static String parseMetadataFile() {
                         key, metadataFilePath);
                 continue;
             }
+            String url = section.get(TEMPLATE_DOWNLOAD_URL_KEY);
+            if (StringUtils.isNotBlank(url) && updateCustomDownloadRepository) {
+                url = url.replaceFirst(defaultDownloadRepository.trim(),
+                        customDownloadRepository.trim());
+                LOGGER.debug("Updated download URL for {} using custom repository to {}", key, url);
+            }
             NewTemplateMap.put(key, new MetadataTemplateDetails(
                     hypervisorType.first(),
                     section.get("templatename"),
                     section.get("filename"),
-                    section.get("downloadurl"),
+                    url,
                     section.get("checksum"),
                     hypervisorType.second(),
                     section.get("guestos")));
         }
-        Ini.Section defaultSection = ini.get("default");
         return defaultSection.get("version").trim();
     }
 

engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java

@@ -344,7 +344,7 @@ public void testIsTemplateFileChecksumDifferent_mismatch() {
     @Test(expected = CloudRuntimeException.class)
     public void testValidateTemplates_metadataTemplateFailure() {
         List<Pair<Hypervisor.HypervisorType, CPU.CPUArch>> list = new ArrayList<>();
-        list.add(new Pair<>(Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64));
+        list.add(new Pair<>(Hypervisor.HypervisorType.VMware, CPU.CPUArch.arm64));
         systemVmTemplateRegistration.validateTemplates(list);
     }
 

engine/schema/templateConfig.sh

@@ -27,6 +27,7 @@ function getTemplateVersion() {
   export CS_VERSION="${subversion1}"."${subversion2}"
   export CS_MINOR_VERSION="${minorversion}"
   export VERSION="${CS_VERSION}.${CS_MINOR_VERSION}"
+  export CS_SYSTEMTEMPLATE_REPO="https://download.cloudstack.org/systemvm/"
 }
 
 function getGenericName() {
@@ -63,7 +64,7 @@ function getChecksum() {
 
 function createMetadataFile() {
   local fileData=$(cat $SOURCEFILE)
-  echo -e "["default"]\nversion = $VERSION.${securityversion}\n" >> $METADATAFILE
+  echo -e "["default"]\nversion = $VERSION.${securityversion}\ndownloadrepository = $CS_SYSTEMTEMPLATE_REPO\n" >> $METADATAFILE
   for template in "${templates[@]}"
   do
     section="${template%%:*}"
@@ -82,13 +83,21 @@ function createMetadataFile() {
 
 declare -a templates
 getTemplateVersion $1
-templates=( "kvm-x86_64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2"
-            "kvm-aarch64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-aarch64-kvm.qcow2.bz2"
-            "vmware:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-vmware.ova"
-            "xenserver:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-xen.vhd.bz2"
-            "hyperv:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-hyperv.vhd.zip"
-            "lxc:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2"
-            "ovm3:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-ovm.raw.bz2" )
+declare -A template_specs=(
+  [kvm-x86_64]="x86_64-kvm.qcow2.bz2"
+  [kvm-aarch64]="aarch64-kvm.qcow2.bz2"
+  [vmware]="x86_64-vmware.ova"
+  [xenserver]="x86_64-xen.vhd.bz2"
+  [hyperv]="x86_64-hyperv.vhd.zip"
+  [lxc]="x86_64-kvm.qcow2.bz2"
+  [ovm3]="x86_64-ovm.raw.bz2"
+)
+
+templates=()
+for key in "${!template_specs[@]}"; do
+  url="${CS_SYSTEMTEMPLATE_REPO}/${CS_VERSION}/systemvmtemplate-$VERSION-${template_specs[$key]}"
+  templates+=("$key:$url")
+done
 
 PARENTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/dist/systemvm-templates/"
 mkdir -p $PARENTPATH

utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java

@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.utils.server;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.cloud.utils.PropertiesUtil;
+
+public class ServerPropertiesUtil {
+    private static final Logger logger = LoggerFactory.getLogger(ServerPropertiesUtil.class);
+    protected static final String PROPERTIES_FILE = "server.properties";
+    protected static final AtomicReference<Properties> propertiesRef = new AtomicReference<>();
+
+    public static String getProperty(String name) {
+        Properties props = propertiesRef.get();
+        if (props != null) {
+            return props.getProperty(name);
+        }
+        File propsFile = PropertiesUtil.findConfigFile(PROPERTIES_FILE);
+        if (propsFile == null) {
+            logger.error("{} file not found", PROPERTIES_FILE);
+            return null;
+        }
+        Properties tempProps = new Properties();
+        try (FileInputStream is = new FileInputStream(propsFile)) {
+            tempProps.load(is);
+        } catch (IOException e) {
+            logger.error("Error loading {}: {}", PROPERTIES_FILE, e.getMessage(), e);
+            return null;
+        }
+        if (!propertiesRef.compareAndSet(null, tempProps)) {
+            tempProps = propertiesRef.get();
+        }
+        return tempProps.getProperty(name);
+    }
+}