Merge remote-tracking branch 'apache/main' into 4.21-fix-vpc-conserve-mode

Author: weizhouapache

.asf.yaml

@@ -53,13 +53,12 @@ github:
     - acs-robot
     - gpordeus
     - hsato03
-    - bernardodemarco
     - FelipeM525
     - lucas-a-martins
     - nicoschmdt
     - abh1sar
-    - sudo87
     - rosi-shapeblue
+    - sudo87
 
   protected_branches: ~
 

.github/linters/.markdown-lint.yml

@@ -86,3 +86,6 @@ MD046: false
 
 # MD052/reference-links-images Reference links and images should use a label that is defined
 MD052: false
+
+# MD059/descriptive-link-text Link text should be descriptive
+MD059: false

.github/workflows/ci.yml

@@ -89,7 +89,10 @@ jobs:
                   smoke/test_nested_virtualization
                   smoke/test_set_sourcenat
                   smoke/test_webhook_lifecycle
-                  smoke/test_purge_expunged_vms",
+                  smoke/test_purge_expunged_vms
+                  smoke/test_extension_lifecycle
+                  smoke/test_extension_custom_action_lifecycle
+                  smoke/test_extension_custom",
                 "smoke/test_network
                   smoke/test_network_acl
                   smoke/test_network_ipv6
@@ -137,6 +140,7 @@ jobs:
                   smoke/test_vm_deployment_planner
                   smoke/test_vm_strict_host_tags
                   smoke/test_vm_schedule
+                  smoke/test_deploy_vgpu_enabled_vm
                   smoke/test_vm_life_cycle
                   smoke/test_vm_lifecycle_unmanage_import
                   smoke/test_vm_snapshot_kvm

.github/workflows/ui.yml

@@ -56,6 +56,7 @@ jobs:
           npm run test:unit
 
       - uses: codecov/codecov-action@v4
+        if: github.repository == 'apache/cloudstack'
         with:
           working-directory: ui
           files: ./coverage/lcov.info

.pre-commit-config.yaml

@@ -15,18 +15,24 @@
 # specific language governing permissions and limitations
 # under the License.
 ---
-default_stages: [commit, push]
+default_stages: [pre-commit, pre-push]
 default_language_version:
   # force all unspecified Python hooks to run python3
   python: python3
-minimum_pre_commit_version: "2.17.0"
+minimum_pre_commit_version: "3.2.0"
 repos:
   - repo: meta
     hooks:
       - id: identity
       - id: check-hooks-apply
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.27.2
+    hooks:
+      - id: gitleaks
+        name: run gitleaks
+        description: detect hardcoded secrets
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       #- id: check-added-large-files
       - id: check-case-conflict
@@ -69,7 +75,7 @@ repos:
         name: run codespell
         description: Check spelling with codespell
         args: [--ignore-words=.github/linters/codespell.txt]
-        exclude: ^ui/package\.json$|^ui/package-lock\.json$|^ui/public/js/less\.min\.js$|^ui/public/locales/.*[^n].*\.json$
+        exclude: ^systemvm/agent/noVNC/|^ui/package\.json$|^ui/package-lock\.json$|^ui/public/js/less\.min\.js$|^ui/public/locales/.*[^n].*\.json$
   - repo: https://github.com/pycqa/flake8
     rev: 7.0.0
     hooks:
@@ -87,7 +93,7 @@ repos:
         ^setup/bindir/cloud-setup-encryption\.in$|
         ^venv/.*$
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.43.0
+    rev: v0.45.0
     hooks:
       - id: markdownlint
         name: run markdownlint

README.md

@@ -160,3 +160,11 @@ The following provides more details on the included cryptographic software:
 * CloudStack makes use of the Bouncy Castle general-purpose encryption library.
 * CloudStack can optionally interact with and control OpenSwan-based VPNs.
 * CloudStack has a dependency on and makes use of JSch - a java SSH2 implementation.
+
+## Star History
+
+[![Apache CloudStack Star History](https://api.star-history.com/svg?repos=apache/cloudstack&type=Date)](https://www.star-history.com/#apache/cloudstack&Date)
+
+## Contributors
+
+[![Apache CloudStack Contributors](https://contrib.rocks/image?repo=apache/cloudstack&anon=0&max=500)](https://github.com/apache/cloudstack/graphs/contributors)

agent/conf/agent.properties

@@ -447,3 +447,6 @@ iscsi.session.cleanup.enabled=false
 
 # Timeout (in seconds) to wait for the incremental snapshot to complete.
 # incremental.snapshot.timeout=10800
+
+# If set to true, creates VMs as full clones of their templates on KVM hypervisor. Creates as linked clones otherwise.
+# create.full.clone=false

agent/src/main/java/com/cloud/agent/Agent.java

@@ -97,7 +97,6 @@
 import com.cloud.utils.nio.NioClient;
 import com.cloud.utils.nio.NioConnection;
 import com.cloud.utils.nio.Task;
-import com.cloud.utils.script.OutputInterpreter;
 import com.cloud.utils.script.Script;
 
 /**
@@ -476,7 +475,7 @@ private void scheduleHostLBCheckerTask(final String lbAlgorithm, final long chec
                 return;
             }
 
-            logger.info("Scheduling a recurring preferred host checker task with lb algorithm '{}' and host.lb.interval={} ms", lbAlgorithm, checkInterval);
+            logger.info("Scheduling a recurring preferred host checker task with host.lb.interval={} ms", checkInterval);
             hostLbCheckExecutor = Executors.newSingleThreadScheduledExecutor((new NamedThreadFactory(name)));
             hostLbCheckExecutor.scheduleAtFixedRate(new PreferredHostCheckerTask(), checkInterval, checkInterval,
                     TimeUnit.MILLISECONDS);
@@ -614,9 +613,9 @@ protected void setupStartupCommand(final StartupCommand startup) {
     }
 
     protected String getAgentArch() {
-        final Script command = new Script("/usr/bin/arch", 500, logger);
-        final OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser();
-        return command.execute(parser);
+        String arch = Script.runSimpleBashScript(Script.getExecutableAbsolutePath("arch"), 1000);
+        logger.debug("Arch for agent: {} found: {}", _name, arch);
+        return arch;
     }
 
     @Override
@@ -968,9 +967,11 @@ private Answer migrateAgentToOtherMS(final MigrateAgentConnectionCommand cmd) {
             if (CollectionUtils.isNotEmpty(cmd.getMsList())) {
                 processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval(), false);
             }
-            Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("MigrateAgentConnection-Job")).schedule(() -> {
+            ScheduledExecutorService migrateAgentConnectionService = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("MigrateAgentConnection-Job"));
+            migrateAgentConnectionService.schedule(() -> {
                 migrateAgentConnection(cmd.getAvoidMsList());
             }, 3, TimeUnit.SECONDS);
+            migrateAgentConnectionService.shutdown();
         } catch (Exception e) {
             String errMsg = "Migrate agent connection failed, due to " + e.getMessage();
             logger.debug(errMsg, e);

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -155,6 +155,14 @@ public class AgentProperties{
      */
     public static final Property<Integer> CMDS_TIMEOUT = new Property<>("cmds.timeout", 7200);
 
+    /**
+     * The timeout (in seconds) for the snapshot merge operation, mainly used for classic volume snapshots and disk-only VM snapshots on file-based storage.<br>
+     * This configuration is only considered if libvirt.events.enabled is also true. <br>
+     * Data type: Integer.<br>
+     * Default value: <code>259200</code>
+     */
+    public static final Property<Integer> QCOW2_DELTA_MERGE_TIMEOUT = new Property<>("qcow2.delta.merge.timeout", 60 * 60 * 72);
+
     /**
      * This parameter sets the VM migration speed (in mbps). The default value is -1,<br>
      * which means that the agent will try to guess the speed of the guest network and consume all possible bandwidth.<br>
@@ -213,6 +221,15 @@ public class AgentProperties{
      */
     public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_XML_TRANSFORMER_SCRIPT = new Property<>("agent.hooks.libvirt_vm_xml_transformer.script", "libvirt-vm-xml-transformer.groovy");
 
+    /**
+     * This property is used with the agent.hooks.basedir property to define the Libvirt VM XML transformer shell script.<br>
+     * The shell script is used to execute the Libvirt VM XML transformer script.<br>
+     * For more information see the agent.properties file.<br>
+     * Data type: String.<br>
+     * Default value: <code>libvirt-vm-xml-transformer.sh</code>
+     */
+    public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_XML_TRANSFORMER_SHELL_SCRIPT = new Property<>("agent.hooks.libvirt_vm_xml_transformer.shell_script", "libvirt-vm-xml-transformer.sh");
+
     /**
      * This property is used with the agent.hooks.basedir and agent.hooks.libvirt_vm_xml_transformer.script properties to define the Libvirt VM XML transformer method.<br>
      * Libvirt XML transformer hook does XML-to-XML transformation.<br>
@@ -233,6 +250,15 @@ public class AgentProperties{
      */
     public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_START_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_start.script", "libvirt-vm-state-change.groovy");
 
+    /**
+     * This property is used with the agent.hooks.basedir property to define the Libvirt VM on start shell script.<br>
+     * The shell script is used to execute the Libvirt VM on start script.<br>
+     * For more information see the agent.properties file.<br>
+     * Data type: String.<br>
+     * Default value: <code>libvirt-vm-state-change.sh</code>
+     */
+    public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_START_SHELL_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_start.shell_script", "libvirt-vm-state-change.sh");
+
     /**
      * This property is used with the agent.hooks.basedir and agent.hooks.libvirt_vm_on_start.script properties to define the Libvirt VM on start method.<br>
      * The hook is called right after Libvirt successfully launched the VM.<br>
@@ -252,6 +278,15 @@ public class AgentProperties{
      */
     public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_STOP_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_stop.script", "libvirt-vm-state-change.groovy");
 
+    /**
+     * This property is used with the agent.hooks.basedir property to define the Libvirt VM on stop shell script.<br>
+     * The shell script is used to execute the Libvirt VM on stop script.<br>
+     * For more information see the agent.properties file.<br>
+     * Data type: String.<br>
+     * Default value: <code>libvirt-vm-state-change.sh</code>
+     */
+    public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_STOP_SHELL_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_stop.shell_script", "libvirt-vm-state-change.sh");
+
     /**
      * This property is used with the agent.hooks.basedir and agent.hooks.libvirt_vm_on_stop.script properties to define the Libvirt VM on stop method.<br>
      * The hook is called right after libvirt successfully stopped the VM.<br>
@@ -828,12 +863,20 @@ public Property<Integer> getWorkers() {
      * */
     public static final Property<Integer> REVERT_SNAPSHOT_TIMEOUT = new Property<>("revert.snapshot.timeout", 10800);
 
+    /**
+     *  If set to true, creates VMs as full clones of their templates on KVM hypervisor. Creates as linked clones otherwise. <br>
+     * Data type: Boolean. <br>
+     * Default value: <code>false</code>
+     */
+    public static final Property<Boolean> CREATE_FULL_CLONE = new Property<>("create.full.clone", false);
+
+
     public static class Property <T>{
         private String name;
         private T defaultValue;
         private Class<T> typeClass;
 
-        Property(String name, T value) {
+        public Property(String name, T value) {
             init(name, value);
         }
 

api/src/main/java/com/cloud/agent/api/Command.java

@@ -19,9 +19,10 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import com.cloud.agent.api.LogLevel.Log4jLevel;
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import com.cloud.agent.api.LogLevel.Log4jLevel;
 
 /**
  * implemented by classes that extends the Command class. Command specifies
@@ -60,6 +61,7 @@ public enum State {
     private int wait;  //in second
     private boolean bypassHostMaintenance = false;
     private transient long requestSequence = 0L;
+    protected Map<String, Map<String, String>> externalDetails;
 
     protected Command() {
         this.wait = 0;
@@ -128,6 +130,14 @@ public void setRequestSequence(long requestSequence) {
         this.requestSequence = requestSequence;
     }
 
+    public void setExternalDetails(Map<String, Map<String, String>> externalDetails) {
+        this.externalDetails = externalDetails;
+    }
+
+    public Map<String, Map<String, String>> getExternalDetails() {
+        return externalDetails;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;